diff --git a/.changeset/exact-audiences-authorize.md b/.changeset/exact-audiences-authorize.md deleted file mode 100644 index 38d486b..0000000 --- a/.changeset/exact-audiences-authorize.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": minor ---- - -Correct service auth to use an exact fragmented DID service audience and deterministic least-privilege OAuth RPC scope. Discovery, provider locks, generated clients, and DID documents now distinguish the base service DID from the JWT audience and pin the protected method set. Existing consumers must reconnect, regenerate, and reauthorize; old plain-DID/wildcard OAuth grants are not compatible. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 1a89c1f..3dd7217 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.19.0 + +### Minor Changes + +- 2e01ffe: Correct service auth to use an exact fragmented DID service audience and deterministic least-privilege OAuth RPC scope. Discovery, provider locks, generated clients, and DID documents now distinguish the base service DID from the JWT audience and pin the protected method set. Existing consumers must reconnect, regenerate, and reauthorize; old plain-DID/wildcard OAuth grants are not compatible. + ## 0.18.0 ### Minor Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index d695f92..6392eba 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.18.0", + "version": "0.19.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From bc2f4c1b9eb6bb15867efdf6e8ac5256c912fa52 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:45:48 +0200 Subject: [PATCH 02/34] Add transactional projection change log foundation --- .changeset/durable-projection-log.md | 7 + packages/contrail/README.md | 26 + packages/contrail/src/adapters/postgres.ts | 15 + packages/contrail/src/adapters/sqlite.ts | 1 + packages/contrail/src/core/backfill.ts | 1 + packages/contrail/src/core/bootstrap.ts | 1 + packages/contrail/src/core/change-log.ts | 520 ++++++++++++++++++ packages/contrail/src/core/constellation.ts | 1 + packages/contrail/src/core/db/records.ts | 157 +++++- packages/contrail/src/core/db/schema.ts | 62 ++- packages/contrail/src/core/ingest.ts | 198 ++++--- packages/contrail/src/core/jetstream.ts | 1 + packages/contrail/src/core/persistent.ts | 1 + packages/contrail/src/core/router/notify.ts | 2 +- packages/contrail/src/core/router/profiles.ts | 4 +- packages/contrail/src/core/types.ts | 135 +++++ packages/contrail/src/index.ts | 6 + packages/contrail/tests/change-log.test.ts | 493 +++++++++++++++++ packages/contrail/tests/ingest.test.ts | 4 +- .../tests/postgres-concurrent-init.test.ts | 2 +- packages/contrail/tests/postgres-e2e.test.ts | 79 ++- packages/contrail/tests/postgres.test.ts | 2 +- packages/contrail/tests/schema.test.ts | 53 +- turbo.json | 2 +- 24 files changed, 1655 insertions(+), 118 deletions(-) create mode 100644 .changeset/durable-projection-log.md create mode 100644 packages/contrail/src/core/change-log.ts create mode 100644 packages/contrail/tests/change-log.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md new file mode 100644 index 0000000..6a3e1e5 --- /dev/null +++ b/.changeset/durable-projection-log.md @@ -0,0 +1,7 @@ +--- +"@atmo-dev/contrail": minor +--- + +Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. + +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 8a1e40d..99614f1 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -72,6 +72,32 @@ After a write to a user's PDS, `contrail.notify(uri)` can fetch the authoritativ Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact yielded cursor in the same transaction. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. +Projection winner selection is guarded again inside the write transaction. Overlapping cron, persistent, notify, or backfill writers cannot commit a stale canonical row, derived projection, tombstone, or source checkpoint; a changed predecessor rolls the complete attempt back and retries from fresh durable state. + +## Transactional change log (experimental) + +Fresh empty generations may opt into a compact transactional projection change log: + +```ts +const config = { + // ... + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "current", + requiredForActivation: true, + }, + }, + }, +} satisfies ContrailConfig; +``` + +Static definitions contain no handlers, URLs, clients, credentials, or secrets. Contrail registers them with a random database-generation ID and collection/phase coverage ledger. A winning logical put/delete appends one compact URI/version reference in the same transaction as canonical and derived state plus the source checkpoint. Duplicate, stale, same-CID, absent-delete, rejected, and rolled-back mutations append nothing. Record bodies are hydrated from current state by the later delivery layer rather than copied into the log. + +This milestone intentionally exposes only the atomic log foundation; leased claim/hydrate/ack delivery and current-state bootstrap APIs follow separately. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. + ## Local development A project containing only `contrail.config.ts` can start a complete local service: diff --git a/packages/contrail/src/adapters/postgres.ts b/packages/contrail/src/adapters/postgres.ts index 310be5d..32586e2 100644 --- a/packages/contrail/src/adapters/postgres.ts +++ b/packages/contrail/src/adapters/postgres.ts @@ -16,6 +16,21 @@ const BIGINT_COLUMNS = new Set([ "last_seen_at", "resolved_at", "total", + "revision", + "head_position", + "retained_floor_position", + "position", + "acknowledged_position", + "bootstrap_anchor_position", + "bootstrap_target_position", + "from_position", + "through_position", + "lease_expires_at", + "next_attempt_at", + "last_success_at", + "last_error_at", + "created_at", + "updated_at", ]); function normalizeRow(row: any): any { diff --git a/packages/contrail/src/adapters/sqlite.ts b/packages/contrail/src/adapters/sqlite.ts index 34b2511..57dee1e 100644 --- a/packages/contrail/src/adapters/sqlite.ts +++ b/packages/contrail/src/adapters/sqlite.ts @@ -9,6 +9,7 @@ interface SqliteStatement extends Statement { export function createSqliteDatabase(path: string): Database { const raw = new DatabaseSync(path); raw.exec("PRAGMA journal_mode = WAL"); + raw.exec("PRAGMA busy_timeout = 5000"); function wrapStatement( sql: string, diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index f4d799f..e84ad58 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -429,6 +429,7 @@ async function backfillUserAttempt( ) .bind(nextCursor ?? null, pageDone ? 1 : 0, now, did, collection); const result = await ingestRecords(db, events, config, { + phase: "historical", skipReplayDetection: options?.skipReplayDetection, skipFeedFanout: true, knownDids: options?.knownDids, diff --git a/packages/contrail/src/core/bootstrap.ts b/packages/contrail/src/core/bootstrap.ts index 7071cbe..e4e5757 100644 --- a/packages/contrail/src/core/bootstrap.ts +++ b/packages/contrail/src/core/bootstrap.ts @@ -496,6 +496,7 @@ export class DatabaseBootstrapTarget implements BootstrapTarget { ): Promise { const knownDids = await this.getKnownDids(); const result = await ingestRecords(this.db, events, this.config, { + phase: "historical", knownDids, skipDerivedProjections: this.options.deferDerivedProjections === true, authoritativeSourceObservation, diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts new file mode 100644 index 0000000..6505ce9 --- /dev/null +++ b/packages/contrail/src/core/change-log.ts @@ -0,0 +1,520 @@ +import type { SqlDialect } from "./dialect"; +import type { + ContrailConfig, + Database, + IngestEvent, + ProjectionPhase, + Statement, +} from "./types"; +import { + canonicalChangeDefinitions, + changeConsumerPhases, + changeLogCoverage, + changesEnabled, +} from "./types"; + +export const MAX_CHANGE_BATCH_CHANGES = 500; +export const MAX_CHANGE_BATCH_BYTES = 512_000; + +export interface RecordChange { + id: string; + kind: "record"; + operation: "put" | "delete"; + uri: string; + did: string; + collection: string; + rkey: string; + cid: string | null; + version: { + sourceId: string; + sourceEpoch: string | null; + sourceRevision: string | null; + sourceTimeUs: number; + sourceCursor: string | null; + }; +} + +type StoredRecordChange = Omit; + +export interface ChangeLogState { + generation: string; + head: string; + retainedFloor: string; + createdAt: number; +} + +interface ChangeLogStateRow { + generation_id: string; + head_position: number | string; + retained_floor_position: number | string; + definitions_json: string; + created_at: number | string; +} + +interface ChangeConsumerRow { + consumer_id: string; + generation_id: string; + acknowledged_position: number | string; + configured_collections_json: string; + configured_phases_json: string; + initial_mode: string; + required_for_activation: number | string; + bootstrap_state: string; + bootstrap_anchor_position: number | string | null; +} + +interface ChangeCoverageRow { + generation_id: string; + collection: string; + phase: ProjectionPhase; + from_position: number | string; + through_position: number | string | null; +} + +export interface ChangeLogSchemaProbe { + exists: boolean; + state: ChangeLogStateRow | null; +} + +/** Optional physical schema. It is absent when no consumers are configured. */ +export function buildChangeLogSchema( + config: ContrailConfig, + dialect: SqlDialect, +): string[] { + if (!changesEnabled(config)) return []; + const bigint = dialect.bigintType; + return [ + `CREATE TABLE IF NOT EXISTS change_log_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + generation_id TEXT NOT NULL, + head_position ${bigint} NOT NULL, + retained_floor_position ${bigint} NOT NULL, + definitions_json TEXT NOT NULL, + created_at ${bigint} NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS change_batches ( + generation_id TEXT NOT NULL, + position ${bigint} NOT NULL, + projection_transaction_id TEXT NOT NULL, + source_id TEXT NOT NULL, + source_epoch TEXT, + source_cursor TEXT, + phase TEXT NOT NULL CHECK (phase IN ('historical', 'live')), + changes_json TEXT NOT NULL, + change_count INTEGER NOT NULL, + created_at ${bigint} NOT NULL, + PRIMARY KEY (generation_id, position) + )`, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_change_batches_transaction ON change_batches(generation_id, projection_transaction_id)", + "CREATE INDEX IF NOT EXISTS idx_change_batches_created ON change_batches(generation_id, created_at)", + `CREATE TABLE IF NOT EXISTS change_consumers ( + consumer_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + acknowledged_position ${bigint} NOT NULL, + configured_collections_json TEXT NOT NULL, + configured_phases_json TEXT NOT NULL, + initial_mode TEXT NOT NULL CHECK (initial_mode IN ('current', 'future', 'history')), + required_for_activation INTEGER NOT NULL DEFAULT 0, + bootstrap_state TEXT NOT NULL CHECK (bootstrap_state IN ('pending', 'scanning', 'catching-up', 'activating', 'ready', 'error', 'reset-required')), + bootstrap_anchor_position ${bigint}, + bootstrap_scan_collection TEXT, + bootstrap_scan_cursor TEXT, + bootstrap_target_position ${bigint}, + bootstrap_token TEXT, + lease_owner TEXT, + lease_expires_at ${bigint}, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at ${bigint}, + last_success_at ${bigint}, + last_error_code TEXT, + last_error_at ${bigint}, + updated_at ${bigint} NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS change_log_coverage ( + generation_id TEXT NOT NULL, + collection TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('historical', 'live')), + from_position ${bigint} NOT NULL, + through_position ${bigint}, + PRIMARY KEY (generation_id, collection, phase, from_position) + )`, + ]; +} + +function missingTable(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + if ((error as { code?: unknown }).code === "42P01") return true; + return /no such table|does not exist/i.test( + String((error as { message?: unknown }).message ?? ""), + ); +} + +export async function probeChangeLogSchema( + db: Database, +): Promise { + try { + const state = await db + .prepare( + `SELECT generation_id, head_position, retained_floor_position, + definitions_json, created_at + FROM change_log_state WHERE id = 1`, + ) + .first(); + return { exists: true, state }; + } catch (error) { + if (missingTable(error)) return { exists: false, state: null }; + throw error; + } +} + +/** Enabling the first log without an old-writer quiet boundary is supported + * only on an empty projection generation in this milestone. */ +export async function assertFreshChangeLogGeneration( + db: Database, +): Promise { + const row = await db + .prepare( + `SELECT CASE WHEN + EXISTS (SELECT 1 FROM record_versions LIMIT 1) OR + EXISTS (SELECT 1 FROM cursor LIMIT 1) OR + EXISTS (SELECT 1 FROM source_position LIMIT 1) OR + EXISTS (SELECT 1 FROM bootstrap_state LIMIT 1) OR + EXISTS (SELECT 1 FROM backfills LIMIT 1) OR + EXISTS (SELECT 1 FROM discovery LIMIT 1) OR + EXISTS (SELECT 1 FROM identities LIMIT 1) + THEN 1 ELSE 0 END AS active`, + ) + .first<{ active: number | string }>(); + if (Number(row?.active ?? 0) !== 0) { + throw new Error( + "Transactional change logging can currently be enabled only on a fresh empty generation; build a fresh generation instead of racing existing projection writers", + ); + } +} + +function canonicalCollections(collections: string[]): string { + return JSON.stringify([...collections].sort()); +} + +function canonicalPhases(phases: ProjectionPhase[]): string { + return JSON.stringify([...phases].sort()); +} + +/** Initialize one immutable milestone-1 logging definition. Later milestones + * add explicit quiet-boundary operations for changing this durable definition. */ +export async function initializeChangeLog( + db: Database, + config: ContrailConfig, +): Promise { + if (!changesEnabled(config)) return; + + const definitions = canonicalChangeDefinitions(config); + const now = Date.now(); + const candidateGeneration = crypto.randomUUID(); + const statements: Statement[] = [ + db + .prepare( + `INSERT INTO change_log_state + (id, generation_id, head_position, retained_floor_position, + definitions_json, created_at) + VALUES (1, ?, 0, 0, ?, ?) + ON CONFLICT(id) DO NOTHING`, + ) + .bind(candidateGeneration, definitions, now), + ]; + + for (const [consumerId, consumer] of Object.entries( + config.changes?.consumers ?? {}, + ).sort(([left], [right]) => left.localeCompare(right))) { + const initialReady = consumer.initial === "current" ? "pending" : "ready"; + statements.push( + db + .prepare( + `INSERT INTO change_consumers + (consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + bootstrap_anchor_position, attempts, updated_at) + SELECT ?, generation_id, head_position, ?, ?, ?, ?, ?, + CASE WHEN ? = 'current' THEN head_position ELSE NULL END, + 0, ? + FROM change_log_state + WHERE id = 1 AND definitions_json = ? + ON CONFLICT(consumer_id) DO NOTHING`, + ) + .bind( + consumerId, + canonicalCollections(consumer.collections), + canonicalPhases(changeConsumerPhases(consumer)), + consumer.initial, + consumer.requiredForActivation === true ? 1 : 0, + initialReady, + consumer.initial, + now, + definitions, + ), + ); + } + + for (const item of changeLogCoverage(config)) { + statements.push( + db + .prepare( + `INSERT INTO change_log_coverage + (generation_id, collection, phase, from_position, through_position) + SELECT generation_id, ?, ?, head_position, NULL + FROM change_log_state + WHERE id = 1 AND definitions_json = ? + ON CONFLICT(generation_id, collection, phase, from_position) + DO NOTHING`, + ) + .bind(item.collection, item.phase, definitions), + ); + } + + // Keep initialization under conservative D1 statement limits. The durable + // definitions_json winner makes these resumable chunks safe under concurrent + // initialization; init does not return until the complete set verifies. + await db.batch(statements.slice(0, 1)); + for (let index = 1; index < statements.length; index += 50) { + await db.batch(statements.slice(index, index + 50)); + } + await assertChangeLogDefinition(db, config); +} + +async function assertChangeLogDefinition( + db: Database, + config: ContrailConfig, +): Promise { + const state = (await probeChangeLogSchema(db)).state; + const definitions = canonicalChangeDefinitions(config); + if (!state || state.definitions_json !== definitions) { + throw new Error( + "Durable change consumer definitions differ from configuration; changing consumers or coverage requires a fresh generation in this milestone", + ); + } + + const consumers = await db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + bootstrap_anchor_position + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + const expectedConsumers = Object.entries(config.changes?.consumers ?? {}).sort( + ([left], [right]) => left.localeCompare(right), + ); + if (consumers.results.length !== expectedConsumers.length) { + throw new Error("Durable change consumer registration is incomplete"); + } + for (let index = 0; index < expectedConsumers.length; index++) { + const [id, expected] = expectedConsumers[index]!; + const actual = consumers.results[index]!; + const bootstrapState = expected.initial === "current" ? "pending" : "ready"; + if ( + actual.consumer_id !== id || + actual.generation_id !== state.generation_id || + actual.configured_collections_json !== canonicalCollections(expected.collections) || + actual.configured_phases_json !== canonicalPhases(changeConsumerPhases(expected)) || + actual.initial_mode !== expected.initial || + Number(actual.required_for_activation) !== + (expected.requiredForActivation === true ? 1 : 0) || + actual.bootstrap_state !== bootstrapState || + Number(actual.acknowledged_position) !== 0 || + (expected.initial === "current" + ? Number(actual.bootstrap_anchor_position) !== 0 + : actual.bootstrap_anchor_position !== null) + ) { + throw new Error(`Durable change consumer ${id} is incompatible`); + } + } + + const coverage = await db + .prepare( + `SELECT generation_id, collection, phase, from_position, through_position + FROM change_log_coverage + ORDER BY collection, phase, from_position`, + ) + .all(); + const expectedCoverage = changeLogCoverage(config); + if (coverage.results.length !== expectedCoverage.length) { + throw new Error("Durable change-log coverage is incomplete"); + } + for (let index = 0; index < expectedCoverage.length; index++) { + const actual = coverage.results[index]!; + const expected = expectedCoverage[index]!; + if ( + actual.generation_id !== state.generation_id || + actual.collection !== expected.collection || + actual.phase !== expected.phase || + Number(actual.from_position) !== 0 || + actual.through_position !== null + ) { + throw new Error("Durable change-log coverage is incompatible"); + } + } +} + +export async function getChangeLogState( + db: Database, +): Promise { + const probe = await probeChangeLogSchema(db); + if (!probe.state) return null; + return { + generation: probe.state.generation_id, + head: String(probe.state.head_position), + retainedFloor: String(probe.state.retained_floor_position), + createdAt: Number(probe.state.created_at), + }; +} + +function bounded(value: string | null, label: string, maximum: number): void { + if (value !== null && value.length > maximum) { + throw new Error(`${label} exceeds ${maximum} characters`); + } +} + +function logicalChanges( + events: IngestEvent[], + existing: ReadonlyMap, + config: ContrailConfig, + phase: ProjectionPhase, +): StoredRecordChange[] { + const covered = new Set( + changeLogCoverage(config) + .filter((item) => item.phase === phase) + .map((item) => item.collection), + ); + if (covered.size === 0) return []; + + // projectEvents already selects one source winner per URI. Keep this final + // reduction defensive for direct internal callers. + const final = new Map(); + for (const event of events) final.set(event.uri, event); + + const changes: StoredRecordChange[] = []; + for (const event of final.values()) { + if (!covered.has(event.collection)) continue; + const prior = existing.get(event.uri); + const deleted = event.operation === "delete"; + const visibleChange = deleted + ? prior !== undefined + : prior === undefined || + prior.cid !== event.cid || + (event.cid === null && prior.record !== event.record); + if (!visibleChange) continue; + + const source = event.source; + const change: StoredRecordChange = { + kind: "record", + operation: deleted ? "delete" : "put", + uri: event.uri, + did: event.did, + collection: event.collection, + rkey: event.rkey, + cid: deleted ? null : event.cid, + version: { + sourceId: source?.id ?? "legacy-caller", + sourceEpoch: source?.epoch ?? null, + sourceRevision: source?.revision ?? null, + sourceTimeUs: source?.time_us ?? event.time_us, + sourceCursor: source?.cursor ?? null, + }, + }; + bounded(change.uri, "change URI", 2_048); + bounded(change.did, "change DID", 2_048); + bounded(change.collection, "change collection", 512); + bounded(change.rkey, "change rkey", 512); + bounded(change.cid, "change CID", 512); + bounded(change.version.sourceId, "change source ID", 128); + bounded(change.version.sourceEpoch, "change source epoch", 256); + bounded(change.version.sourceRevision, "change source revision", 2_048); + bounded(change.version.sourceCursor, "change source cursor", 2_048); + if ( + !Number.isSafeInteger(change.version.sourceTimeUs) || + change.version.sourceTimeUs < 0 + ) { + throw new Error("change source time must be a non-negative safe integer"); + } + changes.push(change); + } + return changes; +} + +function commonValue( + values: Array, +): string | null { + if (values.length === 0) return null; + const first = values[0]!; + return values.every((value) => value === first) ? first : null; +} + +/** Statements appended after canonical/derived projection and before source + * checkpoints in the same database transaction. */ +export function appendChangeLogStatements( + db: Database, + events: IngestEvent[], + existing: ReadonlyMap, + config: ContrailConfig, + phase: ProjectionPhase, +): Statement[] { + if (!changesEnabled(config)) return []; + const changes = logicalChanges(events, existing, config, phase); + if (changes.length === 0) return []; + if (changes.length > MAX_CHANGE_BATCH_CHANGES) { + throw new Error( + `Projection change batch contains ${changes.length} changes; maximum is ${MAX_CHANGE_BATCH_CHANGES}`, + ); + } + const serialized = JSON.stringify(changes); + const bytes = new TextEncoder().encode(serialized).byteLength; + if (bytes > MAX_CHANGE_BATCH_BYTES) { + throw new Error( + `Projection change batch contains ${bytes} encoded bytes; maximum is ${MAX_CHANGE_BATCH_BYTES}`, + ); + } + + const sourceIds = changes.map((change) => change.version.sourceId); + const sourceId = commonValue(sourceIds) ?? "mixed"; + const sourceEpoch = commonValue( + changes.map((change) => change.version.sourceEpoch), + ); + const sourceCursor = commonValue( + changes.map((change) => change.version.sourceCursor), + ); + const transactionId = crypto.randomUUID(); + const now = Date.now(); + + return [ + db + .prepare( + `UPDATE change_log_state + SET head_position = head_position + 1 + WHERE id = 1`, + ), + db + .prepare( + `INSERT INTO change_batches + (generation_id, position, projection_transaction_id, source_id, + source_epoch, source_cursor, phase, changes_json, change_count, + created_at) + VALUES ( + (SELECT generation_id FROM change_log_state WHERE id = 1), + (SELECT head_position FROM change_log_state WHERE id = 1), + ?, ?, ?, ?, ?, ?, ?, ? + )`, + ) + .bind( + transactionId, + sourceId, + sourceEpoch, + sourceCursor, + phase, + serialized, + changes.length, + now, + ), + ]; +} diff --git a/packages/contrail/src/core/constellation.ts b/packages/contrail/src/core/constellation.ts index 221bd88..1c4495d 100644 --- a/packages/contrail/src/core/constellation.ts +++ b/packages/contrail/src/core/constellation.ts @@ -184,6 +184,7 @@ export async function backfillFollowersFromConstellation( if (events.length > 0) { known.add(subjectDid); const ingest = await ingestRecords(db, events, config, { + phase: "historical", knownDids: known, }); inserted += ingest.accepted.length; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index e06cac0..e74fb60 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -8,6 +8,7 @@ import type { RecordRow, RecordSource, OrderedSourceConfig, + ProjectionPhase, } from "../types"; import { getNestedValue, @@ -22,6 +23,7 @@ import { normalizeFeedTarget, feedTargetMaxItems, DEFAULT_FOLLOW_SHORT, + changesEnabled, } from "../types"; import { getSearchableFields, @@ -35,6 +37,7 @@ import { sqliteFtsContentExpression, } from "../dialect"; import type { SourcePosition } from "../sources"; +import { appendChangeLogStatements } from "../change-log"; // --- Counts --- @@ -700,9 +703,13 @@ export interface RecordVersionInfo { source_time_us: number; source_cursor: string | null; indexed_at: number; + /** Opaque optimistic-concurrency token; not part of source ordering. */ + projection_token: string; } -function versionForEvent(event: IngestEvent): RecordVersionInfo { +type ComparableRecordVersion = Omit; + +function versionForEvent(event: IngestEvent): ComparableRecordVersion { const source = event.source; return { uri: event.uri, @@ -742,8 +749,8 @@ function operationRank(operation: RecordVersionInfo["operation"]): number { * tie-breakers; notably a delete wins an exact tie so replay cannot resurrect it. */ export function compareRecordVersions( - left: RecordVersionInfo, - right: RecordVersionInfo, + left: ComparableRecordVersion, + right: ComparableRecordVersion, ): number { if ( left.source_revision !== null && @@ -793,7 +800,7 @@ export async function lookupRecordVersions( const placeholders = chunk.map(() => "?").join(","); const rows = await db .prepare( - `SELECT uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at FROM record_versions WHERE uri IN (${placeholders})`, + `SELECT uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at, projection_token FROM record_versions WHERE uri IN (${placeholders})`, ) .bind(...chunk) .all(); @@ -807,13 +814,13 @@ export interface MutationSelection { superseded: number; } -function selectMutationWinners( +export function selectMutationWinners( events: IngestEvent[], durable: ReadonlyMap, ): MutationSelection { const winners = new Map< string, - { event: IngestEvent; version: RecordVersionInfo; index: number } + { event: IngestEvent; version: ComparableRecordVersion; index: number } >(); let superseded = 0; @@ -918,10 +925,11 @@ const RECORD_UPSERT_BINDINGS = 7; const RECORD_UPSERT_ROWS = Math.floor( MAX_STATEMENT_BINDINGS / RECORD_UPSERT_BINDINGS ); -const RECORD_VERSION_BINDINGS = 12; +const RECORD_VERSION_BINDINGS = 13; const RECORD_VERSION_ROWS = Math.floor( MAX_STATEMENT_BINDINGS / RECORD_VERSION_BINDINGS, ); +const PROJECTION_GUARD_URIS = 40; interface StorageMutation { event: IngestEvent; @@ -932,17 +940,18 @@ function buildRecordVersionStatements( db: Database, events: IngestEvent[], existing: Map, + projectionTokens: ReadonlyMap, ): Statement[] { const statements: Statement[] = []; for (let index = 0; index < events.length; index += RECORD_VERSION_ROWS) { const chunk = events.slice(index, index + RECORD_VERSION_ROWS); const values = chunk - .map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + .map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") .join(", "); statements.push( db .prepare( - `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at) VALUES ${values} ON CONFLICT(uri) DO UPDATE SET did = excluded.did, collection = excluded.collection, rkey = excluded.rkey, operation = excluded.operation, cid = excluded.cid, source_id = excluded.source_id, source_epoch = excluded.source_epoch, source_revision = excluded.source_revision, source_time_us = excluded.source_time_us, source_cursor = excluded.source_cursor, indexed_at = excluded.indexed_at`, + `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at, projection_token) VALUES ${values} ON CONFLICT(uri) DO UPDATE SET did = excluded.did, collection = excluded.collection, rkey = excluded.rkey, operation = excluded.operation, cid = excluded.cid, source_id = excluded.source_id, source_epoch = excluded.source_epoch, source_revision = excluded.source_revision, source_time_us = excluded.source_time_us, source_cursor = excluded.source_cursor, indexed_at = excluded.indexed_at, projection_token = excluded.projection_token`, ) .bind( ...chunk.flatMap((event) => { @@ -964,6 +973,7 @@ function buildRecordVersionStatements( version.source_time_us, version.source_cursor, version.indexed_at, + projectionTokens.get(event.uri)!, ]; }), ), @@ -1024,6 +1034,71 @@ function buildRecordMutationStatements( return statements; } +function buildProjectionGuardStatements( + db: Database, + events: IngestEvent[], + predecessors: ReadonlyMap, +): Statement[] { + const uris = [...new Set(events.map((event) => event.uri))]; + const statements: Statement[] = [ + db.prepare( + `INSERT INTO _contrail_projection_state (id, revision, guard) + VALUES (1, 0, 1) ON CONFLICT(id) DO NOTHING`, + ), + // PostgreSQL takes a row lock here. The following statement then receives a + // fresh READ COMMITTED snapshot after any earlier projector commits. SQLite + // and D1 already serialize the containing write batch. + db.prepare( + `UPDATE _contrail_projection_state + SET revision = revision + 1 + WHERE id = 1`, + ), + ]; + for (let index = 0; index < uris.length; index += PROJECTION_GUARD_URIS) { + const chunk = uris.slice(index, index + PROJECTION_GUARD_URIS); + const conditions: string[] = []; + const bindings: string[] = []; + for (const uri of chunk) { + const predecessor = predecessors.get(uri); + if (!predecessor) { + conditions.push( + "NOT EXISTS (SELECT 1 FROM record_versions WHERE uri = ?)", + ); + bindings.push(uri); + continue; + } + if (!predecessor.projection_token) { + throw new Error(`Record version ${uri} has no projection token`); + } + conditions.push( + "EXISTS (SELECT 1 FROM record_versions WHERE uri = ? AND projection_token = ?)", + ); + bindings.push(uri, predecessor.projection_token); + } + statements.push( + db + .prepare( + `UPDATE _contrail_projection_state + SET guard = CASE WHEN ${conditions.join(" AND ")} THEN 1 ELSE 0 END + WHERE id = 1`, + ) + .bind(...bindings), + ); + } + return statements; +} + +/** Adapter-neutral classification for the named optimistic guard constraint. */ +export function isProjectionConflictError(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const candidate = error as { code?: unknown; constraint?: unknown; message?: unknown }; + return ( + (candidate.code === "23514" && + candidate.constraint === "projection_guard_valid") || + /projection_guard_valid/i.test(String(candidate.message ?? "")) + ); +} + export async function projectEvents( db: Database, events: IngestEvent[], @@ -1033,19 +1108,26 @@ export async function projectEvents( skipFeedFanout?: boolean; /** Skip FTS and relation-count maintenance during canonical bulk loading. */ skipDerivedProjections?: boolean; - /** Pre-fetched existing records — skips the internal lookup when provided */ + /** @deprecated Existing rows are re-read for transaction conflict safety. */ existing?: Map; /** Statements committed after projection in the same database batch. */ trailingStatements?: Statement[]; /** Internal: ingestRecords already checked durable source order. */ sourceOrderingChecked?: boolean; + /** Durable versions observed while selecting source winners. */ + predecessors?: ReadonlyMap; + /** Acquisition phase persisted on an optional change batch. */ + phase?: ProjectionPhase; }, ): Promise { if (events.length === 0) return { applied: [], superseded: 0 }; + const predecessors = + options?.predecessors ?? + (await lookupRecordVersions(db, events.map((event) => event.uri))); const selection = options?.sourceOrderingChecked ? { applied: events, superseded: 0 } - : await selectCurrentMutations(db, events); + : selectMutationWinners(events, predecessors); events = selection.applied; if (events.length === 0) { if (options?.trailingStatements?.length) { @@ -1062,17 +1144,19 @@ export async function projectEvents( (relation) => relation.count !== false ) ); - const needRecordContent = followCollections.length > 0 || hasCountingRelations; - - // Use pre-fetched data or look up existing records - let existingMap: Map; - if (options?.existing) { - existingMap = options.existing; - } else if (!options?.skipReplayDetection) { - existingMap = await lookupExistingRecords(db, events, needRecordContent, config); - } else { - existingMap = new Map(); - } + const needRecordContent = + followCollections.length > 0 || + hasCountingRelations || + changesEnabled(config); + + // Existing state must be read after predecessor selection. A caller-provided + // map can predate that selection and would make derived changes incorrect + // even when the optimistic token guard itself succeeds. + const needExistingState = + !options?.skipReplayDetection || needRecordContent || changesEnabled(config); + const existingMap = needExistingState + ? await lookupExistingRecords(db, events, needRecordContent, config) + : new Map(); const batch: Statement[] = []; // Keep only the final storage mutation for a URI within this atomic batch. @@ -1122,15 +1206,36 @@ export async function projectEvents( } } - // Storage and durable version/tombstone metadata run first so FTS, feeds, - // and count statements in the same atomic batch observe the final records. + const projectionTokens = new Map( + events.map((event) => [event.uri, crypto.randomUUID()] as const), + ); + + // Lock, verify the exact durable predecessors selected by the caller, then + // write storage and version metadata. Any changed token violates the named + // guard constraint and rolls the complete database batch back. batch.unshift( + ...buildProjectionGuardStatements(db, events, predecessors), ...buildRecordMutationStatements(db, storageMutations.values()), - ...buildRecordVersionStatements(db, events, existingMap), + ...buildRecordVersionStatements( + db, + events, + existingMap, + projectionTokens, + ), ); - // Build deduplicated count statements — one UPDATE per unique target. + // Build deduplicated count statements, then append the compact change batch. + // Caller-provided source checkpoints deliberately remain last. batch.push(...buildBatchCountStatements(db, config, countTargets)); + batch.push( + ...appendChangeLogStatements( + db, + events, + existingMap, + config, + options?.phase ?? "live", + ), + ); if (options?.trailingStatements?.length) { batch.push(...options.trailingStatements); } diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 95fa42d..51dfe2f 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -25,9 +25,19 @@ import { getSearchableFields, } from "../search"; import { buildLabelsSchema } from "../labels/schema"; +import { + assertFreshChangeLogGeneration, + buildChangeLogSchema, + initializeChangeLog, + probeChangeLogSchema, +} from "../change-log"; +import { + canonicalChangeDefinitions, + changesEnabled, +} from "../types"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 11; +export const CONTRAIL_SCHEMA_VERSION = 12; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -130,7 +140,8 @@ CREATE TABLE IF NOT EXISTS record_versions ( source_revision TEXT, source_time_us ${dialect.bigintType} NOT NULL, source_cursor TEXT, - indexed_at ${dialect.bigintType} NOT NULL + indexed_at ${dialect.bigintType} NOT NULL, + projection_token TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_record_versions_collection ON record_versions(collection); CREATE INDEX IF NOT EXISTS idx_record_versions_did ON record_versions(did); @@ -140,6 +151,12 @@ CREATE TABLE IF NOT EXISTS ingest_diagnostics ( total ${dialect.bigintType} NOT NULL, last_seen_at ${dialect.bigintType} NOT NULL ); +CREATE TABLE IF NOT EXISTS _contrail_projection_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision ${dialect.bigintType} NOT NULL DEFAULT 0, + guard INTEGER NOT NULL DEFAULT 1 + CONSTRAINT projection_guard_valid CHECK (guard = 1) +); `; } @@ -576,6 +593,11 @@ const MIGRATIONS: MigrationOp[] = [ column: "source_epoch", columnDef: "TEXT", }, + { + table: "record_versions", + column: "projection_token", + columnDef: "TEXT", + }, { table: "discovery", column: "retries", @@ -639,8 +661,8 @@ async function seedLegacyRecordVersions( const table = recordsTableName(shortName); await db .prepare( - `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at) - SELECT uri, did, ?, rkey, 'update', cid, 'legacy', NULL, NULL, indexed_at, NULL, indexed_at FROM ${table} + `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at, projection_token) + SELECT uri, did, ?, rkey, 'update', cid, 'legacy', NULL, NULL, indexed_at, NULL, indexed_at, uri FROM ${table} WHERE 1 = 1 ON CONFLICT(uri) DO NOTHING`, ) .bind(collection.collection) @@ -671,6 +693,7 @@ function schemaFingerprint( indexes: string[]; feeds: string[]; fts: string[]; + changes: string[]; }, ): string { return hashStrings([ @@ -682,6 +705,8 @@ function schemaFingerprint( ...ddl.indexes, ...ddl.feeds, ...ddl.fts, + ...ddl.changes, + canonicalChangeDefinitions(config), ...buildCountColumns(config), ...(config.labels ? buildLabelsSchema(dialect) : []), JSON.stringify(MIGRATIONS), @@ -702,12 +727,14 @@ export async function initSchema( const indexes = buildDynamicIndexes(config, dialect); const feeds = buildFeedTables(config, dialect); const fts = buildFtsTables(config, dialect); + const changes = buildChangeLogSchema(config, dialect); const fingerprint = schemaFingerprint(config, dialect, { base, collections, indexes, feeds, fts, + changes, }); if ((await getMeta(db, SCHEMA_FINGERPRINT_KEY)) === fingerprint) { @@ -715,6 +742,13 @@ export async function initSchema( return; } + const priorChangeLog = await probeChangeLogSchema(db); + if (!changesEnabled(config) && priorChangeLog.exists) { + throw new Error( + "Durable change logging cannot be disabled or removed during ordinary initialization", + ); + } + for (const statement of [...base, ...collections, ...indexes, ...feeds]) { await runIdempotentDdl(db, statement); } @@ -729,9 +763,29 @@ export async function initSchema( const hasFeeds = !!(config.feeds && Object.keys(config.feeds).length > 0); await runMigrations(db, hasFeeds); + await db + .prepare( + "UPDATE record_versions SET projection_token = uri WHERE projection_token IS NULL", + ) + .run(); + await db + .prepare( + `INSERT INTO _contrail_projection_state (id, revision, guard) + VALUES (1, 0, 1) ON CONFLICT(id) DO NOTHING`, + ) + .run(); await applyCountColumns(db, config); await seedLegacyRecordVersions(db, config); + if (changesEnabled(config)) { + const concurrentChangeLog = await probeChangeLogSchema(db); + if (!priorChangeLog.state && !concurrentChangeLog.state) { + await assertFreshChangeLogGeneration(db); + } + for (const statement of changes) await runIdempotentDdl(db, statement); + await initializeChangeLog(db, config); + } + for (const apply of options.extraSchemas ?? []) await apply(db); await setMeta(db, SCHEMA_FINGERPRINT_KEY, fingerprint); } diff --git a/packages/contrail/src/core/ingest.ts b/packages/contrail/src/core/ingest.ts index 48527cf..fe1392c 100644 --- a/packages/contrail/src/core/ingest.ts +++ b/packages/contrail/src/core/ingest.ts @@ -4,6 +4,7 @@ import type { Database, IngestEvent, MutationSource, + ProjectionPhase, Statement, } from "./types"; import { @@ -11,9 +12,11 @@ import { resolveCollectionKey, } from "./types"; import { + isProjectionConflictError, + lookupRecordVersions, projectEvents, selectAuthoritativeMutations, - selectCurrentMutations, + selectMutationWinners, type ExistingRecordInfo, } from "./db/records"; import { @@ -114,6 +117,9 @@ export interface IngestRecordsOptions { /** The source response is a current authoritative snapshot, so it supersedes * durable observations without a redundant version lookup. */ authoritativeSourceObservation?: boolean; + /** Acquisition phase for optional durable consumers. Defaults to live for + * backwards-compatible direct ingestRecords() calls. */ + phase?: ProjectionPhase; /** @internal Aggregate private diagnostics for one bulk run. The caller * flushes this bounded object once after concurrent page processing. */ aggregateDiagnostics?: IngestDiagnosticCounts; @@ -277,97 +283,131 @@ export async function ingestRecords( ); } - // Reject duplicate/stale source observations before they can admit dependent - // actors in this batch. The winning versions are persisted with projection. - const ordered = options.authoritativeSourceObservation - ? selectAuthoritativeMutations(accepted) - : await selectCurrentMutations(db, accepted); - dropped.superseded += ordered.superseded; + // Winner reads occur before db.batch on D1, so each projection carries the + // exact predecessor tokens it observed. A concurrent projector changes a + // token, the named transaction guard rolls everything back, and this loop + // repeats selection plus derived-state reads from fresh durable state. + const retryBase = { + unknownActor: dropped.unknownActor, + unknownSubject: dropped.unknownSubject, + superseded: dropped.superseded, + }; + const maximumAttempts = 5; + for (let attempt = 1; attempt <= maximumAttempts; attempt++) { + const attemptDropped: IngestDropCounts = { + ...dropped, + unknownActor: retryBase.unknownActor, + unknownSubject: retryBase.unknownSubject, + superseded: retryBase.superseded, + }; + const predecessors = await lookupRecordVersions( + db, + accepted.map((event) => event.uri), + ); + const ordered = options.authoritativeSourceObservation + ? selectAuthoritativeMutations(accepted) + : selectMutationWinners(accepted, predecessors); + attemptDropped.superseded += ordered.superseded; - const projectionExclusions = ordered.applied.filter((event) => - policyExcluded.has(event), - ); - const admitted = ordered.applied.filter((event) => !policyExcluded.has(event)); + const projectionExclusions = ordered.applied.filter((event) => + policyExcluded.has(event), + ); + const admitted = ordered.applied.filter( + (event) => !policyExcluded.has(event), + ); - const effectiveKnownDids = options.knownDids - ? new Set(options.knownDids) - : undefined; - const discoveredDids: string[] = []; - if (effectiveKnownDids) { + const effectiveKnownDids = options.knownDids + ? new Set(options.knownDids) + : undefined; + const discoveredDids: string[] = []; + if (effectiveKnownDids) { + for (const event of admitted) { + if (event.operation === "delete") continue; + const shortName = resolveCollectionKey(config, event.collection); + const collection = shortName ? config.collections[shortName] : undefined; + if (collection?.discover === false || effectiveKnownDids.has(event.did)) { + continue; + } + effectiveKnownDids.add(event.did); + discoveredDids.push(event.did); + } + } + + const actorFiltered: IngestEvent[] = []; for (const event of admitted) { - if (event.operation === "delete") continue; + if (event.operation === "delete" || !effectiveKnownDids) { + actorFiltered.push(event); + continue; + } const shortName = resolveCollectionKey(config, event.collection); const collection = shortName ? config.collections[shortName] : undefined; - if (collection?.discover === false || effectiveKnownDids.has(event.did)) { + if (collection?.discover !== false || effectiveKnownDids.has(event.did)) { + actorFiltered.push(event); continue; } - effectiveKnownDids.add(event.did); - discoveredDids.push(event.did); + attemptDropped.unknownActor++; } - } - const actorFiltered: IngestEvent[] = []; - for (const event of admitted) { - if (event.operation === "delete" || !effectiveKnownDids) { - actorFiltered.push(event); - continue; - } - const shortName = resolveCollectionKey(config, event.collection); - const collection = shortName ? config.collections[shortName] : undefined; - if (collection?.discover !== false || effectiveKnownDids.has(event.did)) { - actorFiltered.push(event); - continue; - } - dropped.unknownActor++; - } + const subjectFiltered = await filterUnknownSubjects( + db, + config, + actorFiltered, + effectiveKnownDids, + attemptDropped, + ); + const projectionEvents = [...subjectFiltered, ...projectionExclusions]; + const diagnosticCounts: IngestDiagnosticCounts = { + unknown_collection: attemptDropped.unknownCollection, + invalid_json: attemptDropped.invalidRecord, + lexicon_validation: attemptDropped.lexiconValidation, + cid_mismatch: attemptDropped.cidMismatch, + cid_encoding: attemptDropped.cidEncoding, + missing_cid: attemptDropped.missingCid, + record_filter: attemptDropped.recordFilter, + unknown_actor: attemptDropped.unknownActor, + unknown_subject: attemptDropped.unknownSubject, + superseded: attemptDropped.superseded, + }; + const diagnostics = options.aggregateDiagnostics + ? null + : ingestDiagnosticsStatement(db, diagnosticCounts); + const trailingStatements = [ + ...(diagnostics ? [diagnostics] : []), + ...(options.trailingStatements ?? []), + ]; - const subjectFiltered = await filterUnknownSubjects( - db, - config, - actorFiltered, - effectiveKnownDids, - dropped, - ); + try { + if (projectionEvents.length > 0) { + await projectEvents(db, projectionEvents, config, { + ...options, + phase: options.phase ?? "live", + // A pre-fetched visible-row map is not safe after a conflict; the + // projector deliberately reloads it under this predecessor attempt. + existing: undefined, + trailingStatements, + sourceOrderingChecked: true, + predecessors, + }); + } else if (trailingStatements.length > 0) { + await db.batch(trailingStatements); + } + } catch (error) { + if (isProjectionConflictError(error) && attempt < maximumAttempts) { + continue; + } + throw error; + } - const projectionEvents = [ - ...subjectFiltered, - ...projectionExclusions, - ]; - const diagnosticCounts: IngestDiagnosticCounts = { - unknown_collection: dropped.unknownCollection, - invalid_json: dropped.invalidRecord, - lexicon_validation: dropped.lexiconValidation, - cid_mismatch: dropped.cidMismatch, - cid_encoding: dropped.cidEncoding, - missing_cid: dropped.missingCid, - record_filter: dropped.recordFilter, - unknown_actor: dropped.unknownActor, - unknown_subject: dropped.unknownSubject, - superseded: dropped.superseded, - }; - const diagnostics = options.aggregateDiagnostics - ? null - : ingestDiagnosticsStatement(db, diagnosticCounts); - const trailingStatements = [ - ...(diagnostics ? [diagnostics] : []), - ...(options.trailingStatements ?? []), - ]; - if (projectionEvents.length > 0) { - await projectEvents(db, projectionEvents, config, { - ...options, - trailingStatements, - sourceOrderingChecked: true, - }); - } else if (trailingStatements.length > 0) { - await db.batch(trailingStatements); - } - // Aggregate only after the canonical projection/checkpoint transaction - // succeeds, so a rolled-back page cannot inflate private diagnostics. - if (options.aggregateDiagnostics) { - addIngestDiagnosticCounts(options.aggregateDiagnostics, diagnosticCounts); + Object.assign(dropped, attemptDropped); + // Aggregate only after the canonical projection/checkpoint transaction + // succeeds, so a rolled-back attempt cannot inflate private diagnostics. + if (options.aggregateDiagnostics) { + addIngestDiagnosticCounts(options.aggregateDiagnostics, diagnosticCounts); + } + return { accepted: subjectFiltered, dropped, discoveredDids }; } - return { accepted: subjectFiltered, dropped, discoveredDids }; + throw new Error("Projection conflict retry limit exhausted"); } function incrementValidationDrop( diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index b29c8d1..e1285e9 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -478,6 +478,7 @@ export async function runIngestCycle( const batch = events.slice(i, i + BATCH_SIZE); const isFinalBatch = i + BATCH_SIZE >= events.length; const result = await ingestRecords(db, batch, config, { + phase: "live", knownDids, // Earlier batches may commit without moving the cursor. A crash replays // them safely; the final batch atomically commits the exact source cursor. diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index 3b96cc6..877526b 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -161,6 +161,7 @@ async function streamAndFlush( let ingestResult: Awaited>; try { ingestResult = await ingestRecords(db, batch, config, { + phase: "live", knownDids, trailingStatements: [ saveCursorStatement(db, lastTimeUs), diff --git a/packages/contrail/src/core/router/notify.ts b/packages/contrail/src/core/router/notify.ts index 1f11282..3688a27 100644 --- a/packages/contrail/src/core/router/notify.ts +++ b/packages/contrail/src/core/router/notify.ts @@ -255,7 +255,7 @@ export async function processNotifyUris( } const appliedEvents = events.length > 0 - ? (await ingestRecords(db, events, config, { existing })).accepted + ? (await ingestRecords(db, events, config, { existing, phase: "live" })).accepted : []; // The shared ingest path fans these records into feed_items exactly like the cron and diff --git a/packages/contrail/src/core/router/profiles.ts b/packages/contrail/src/core/router/profiles.ts index abaa351..5bdf518 100644 --- a/packages/contrail/src/core/router/profiles.ts +++ b/packages/contrail/src/core/router/profiles.ts @@ -185,7 +185,9 @@ async function fetchMissingProfiles( const events = fetched.filter((event) => event !== null); if (events.length === 0) return {}; - const { accepted } = await ingestRecords(db, events, config); + const { accepted } = await ingestRecords(db, events, config, { + phase: "historical", + }); const result: Record = {}; for (const event of accepted) { if (event.operation === "delete") continue; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index c472054..bb5860e 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -249,6 +249,28 @@ export interface OrderedSourceConfig { epoch: string; } +/** How an accepted mutation entered the logical projection. */ +export type ProjectionPhase = "historical" | "live"; + +export type ChangeConsumerInitialMode = "current" | "future" | "history"; + +/** Static, secret-free definition for one durable change-log consumer. */ +export interface ChangeConsumerConfig { + /** Exact configured collection NSIDs. Short aliases are deliberately rejected. */ + collections: string[]; + /** Projection phases to observe. Defaults to both historical and live. */ + phases?: ProjectionPhase[]; + /** How the consumer establishes its first durable position. */ + initial: ChangeConsumerInitialMode; + /** Whether deployment generation activation may require this consumer. */ + requiredForActivation?: boolean; +} + +export interface ChangeLogConfig { + /** Stable consumer IDs mapped to their static delivery policy. */ + consumers: Record; +} + export type AtprotoServiceAuthMethod = "getFeed" | "notifyOfUpdate"; export interface AtprotoServiceAuthConfig { @@ -284,6 +306,9 @@ export interface ContrailConfig { * cursor is persisted atomically with projected mutations and may be exposed * to clients as a cache invalidation coordinate. */ orderedSource?: OrderedSourceConfig; + /** Optional transactional projection change log. Runtime handlers and + * destination credentials are bound separately and never belong here. */ + changes?: ChangeLogConfig; feeds?: Record; logger?: Logger; /** Expose the notifyOfUpdate HTTP endpoint. Off by default. @@ -655,6 +680,58 @@ function validateShortName(short: string): void { } } +const CHANGE_CONSUMER_ID = /^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/; +const MAX_CHANGE_CONSUMERS = 32; +const MAX_CHANGE_CONSUMER_COLLECTIONS = 64; +const MAX_CHANGE_COVERAGE_PAIRS = 256; +const MAX_CHANGE_DEFINITIONS_BYTES = 64 * 1_024; + +/** Whether this configuration requires the optional transactional change log. */ +export function changesEnabled(config: ContrailConfig): boolean { + return Object.keys(config.changes?.consumers ?? {}).length > 0; +} + +/** Canonical phases for a consumer definition. */ +export function changeConsumerPhases( + consumer: ChangeConsumerConfig, +): ProjectionPhase[] { + return consumer.phases ?? ["historical", "live"]; +} + +/** Canonical collection/phase pairs whose changes must be retained. */ +export function changeLogCoverage( + config: ContrailConfig, +): Array<{ collection: string; phase: ProjectionPhase }> { + const pairs = new Map(); + for (const consumer of Object.values(config.changes?.consumers ?? {})) { + for (const collection of consumer.collections) { + for (const phase of changeConsumerPhases(consumer)) { + pairs.set(`${collection}\0${phase}`, { collection, phase }); + } + } + } + return [...pairs.values()].sort( + (left, right) => + left.collection.localeCompare(right.collection) || + left.phase.localeCompare(right.phase), + ); +} + +/** Stable secret-free representation used by schema/config compatibility checks. */ +export function canonicalChangeDefinitions(config: ContrailConfig): string { + return JSON.stringify( + Object.entries(config.changes?.consumers ?? {}) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([id, consumer]) => ({ + id, + collections: [...consumer.collections].sort(), + phases: [...changeConsumerPhases(consumer)].sort(), + initial: consumer.initial, + requiredForActivation: consumer.requiredForActivation === true, + })), + ); +} + export function validateConfig(config: ContrailConfig): void { const shortNames = new Set(); for (const [short, colConfig] of Object.entries(config.collections)) { @@ -715,6 +792,64 @@ export function validateConfig(config: ContrailConfig): void { } } + const consumers = Object.entries(config.changes?.consumers ?? {}); + if (consumers.length > MAX_CHANGE_CONSUMERS) { + throw new Error(`changes supports at most ${MAX_CHANGE_CONSUMERS} consumers`); + } + const configuredNsids = new Set(getCollectionNsids(config)); + for (const [id, consumer] of consumers) { + if (!CHANGE_CONSUMER_ID.test(id)) { + throw new Error( + `Invalid change consumer ID "${id}"; use 1-64 letters, digits, underscores, or hyphens`, + ); + } + if ( + !Array.isArray(consumer.collections) || + consumer.collections.length === 0 || + consumer.collections.length > MAX_CHANGE_CONSUMER_COLLECTIONS || + new Set(consumer.collections).size !== consumer.collections.length + ) { + throw new Error( + `Change consumer "${id}" requires 1-${MAX_CHANGE_CONSUMER_COLLECTIONS} unique collection NSIDs`, + ); + } + for (const collection of consumer.collections) { + if (!configuredNsids.has(collection)) { + throw new Error( + `Change consumer "${id}" references unconfigured collection NSID "${collection}"`, + ); + } + } + const phases = changeConsumerPhases(consumer); + if ( + phases.length === 0 || + phases.length > 2 || + new Set(phases).size !== phases.length || + phases.some((phase) => phase !== "historical" && phase !== "live") + ) { + throw new Error( + `Change consumer "${id}" requires unique historical/live phases`, + ); + } + if (!(["current", "future", "history"] as string[]).includes(consumer.initial)) { + throw new Error(`Change consumer "${id}" has an invalid initial mode`); + } + } + const coveragePairs = changeLogCoverage(config).length; + if (coveragePairs > MAX_CHANGE_COVERAGE_PAIRS) { + throw new Error( + `changes requires ${coveragePairs} collection/phase coverage pairs; maximum is ${MAX_CHANGE_COVERAGE_PAIRS}`, + ); + } + const definitionBytes = new TextEncoder().encode( + canonicalChangeDefinitions(config), + ).byteLength; + if (definitionBytes > MAX_CHANGE_DEFINITIONS_BYTES) { + throw new Error( + `changes definitions contain ${definitionBytes} encoded bytes; maximum is ${MAX_CHANGE_DEFINITIONS_BYTES}`, + ); + } + } // Helpers diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index bd254c2..0e68f6c 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -29,6 +29,12 @@ export * from "./core/persistent"; export * from "./core/backfill"; export * from "./core/status"; export * from "./core/diagnostics"; +export { + getChangeLogState, + MAX_CHANGE_BATCH_BYTES, + MAX_CHANGE_BATCH_CHANGES, +} from "./core/change-log"; +export type { ChangeLogState, RecordChange } from "./core/change-log"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts new file mode 100644 index 0000000..ead13a1 --- /dev/null +++ b/packages/contrail/tests/change-log.test.ts @@ -0,0 +1,493 @@ +import { describe, expect, it } from "vitest"; +import { + Contrail, + createIngestEvent, + getChangeLogState, + ingestRecords, + initSchema, + queryRecords, + resolveConfig, + saveCursorStatement, + type ContrailConfig, + type Database, + type Statement, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const EVENT = "com.example.event"; +const NOTE = "com.example.note"; +const URI = `at://did:plc:alice/${EVENT}/one`; +const logger = { log() {}, warn() {}, error() {} }; + +function config(options: { + changes?: ContrailConfig["changes"]; + includeNote?: boolean; +} = {}) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { + event: { collection: EVENT }, + ...(options.includeNote ? { note: { collection: NOTE } } : {}), + }, + changes: options.changes, + }); +} + +function loggedConfig() { + return config({ + changes: { + consumers: { + search: { + collections: [EVENT], + initial: "current", + requiredForActivation: true, + }, + webhooks: { + collections: [EVENT], + phases: ["live"], + initial: "future", + }, + }, + }, + }); +} + +function mutation(options: { + sourceTime: number; + cid?: string | null; + value?: Record; + operation?: "create" | "update" | "delete"; + revision?: string | null; +}) { + const operation = options.operation ?? "update"; + return createIngestEvent({ + uri: URI, + did: "did:plc:alice", + collection: EVENT, + rkey: "one", + operation, + cid: operation === "delete" ? null : (options.cid ?? `cid-${options.sourceTime}`), + value: + operation === "delete" + ? undefined + : (options.value ?? { name: `event-${options.sourceTime}` }), + timeUs: options.sourceTime, + indexedAt: options.sourceTime + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: options.sourceTime, + revision: options.revision ?? String(options.sourceTime), + cursor: String(options.sourceTime), + }, + }); +} + +async function batches(db: Database) { + return ( + await db + .prepare( + `SELECT generation_id, position, projection_transaction_id, source_id, + source_epoch, source_cursor, phase, changes_json, change_count + FROM change_batches ORDER BY position`, + ) + .all() + ).results; +} + +describe("transactional projection change log", () => { + it("validates bounded static consumer definitions", () => { + const base = { + namespace: "com.example", + profiles: [] as string[], + collections: { event: { collection: EVENT } }, + }; + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + "bad id": { collections: [EVENT], initial: "future" }, + }, + }, + }), + ).toThrow("Invalid change consumer ID"); + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + search: { collections: ["event"], initial: "future" }, + }, + }, + }), + ).toThrow("unconfigured collection NSID"); + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + search: { + collections: [EVENT], + phases: ["live", "live"], + initial: "future", + }, + }, + }, + }), + ).toThrow("unique historical/live phases"); + }); + + it("has no change-log schema or writes when disabled", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config(); + await initSchema(db, resolved); + + const tables = await db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'change_%' ORDER BY name", + ) + .all<{ name: string }>(); + expect(tables.results).toEqual([]); + expect( + await db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '_contrail_projection_state'", + ) + .first(), + ).not.toBeNull(); + + await ingestRecords(db, [mutation({ sourceTime: 1 })], resolved); + expect(await getChangeLogState(db)).toBeNull(); + }); + + it("initializes a fresh generation, registrations, and coverage ledger", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + + const state = await getChangeLogState(db); + expect(state).toMatchObject({ head: "0", retainedFloor: "0" }); + expect(state?.generation).toMatch(/^[0-9a-f-]{36}$/); + + const consumers = await db + .prepare( + `SELECT consumer_id, configured_collections_json, + configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + acknowledged_position, bootstrap_anchor_position + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + expect(consumers.results).toEqual([ + { + consumer_id: "search", + configured_collections_json: JSON.stringify([EVENT]), + configured_phases_json: JSON.stringify(["historical", "live"]), + initial_mode: "current", + required_for_activation: 1, + bootstrap_state: "pending", + acknowledged_position: 0, + bootstrap_anchor_position: 0, + }, + { + consumer_id: "webhooks", + configured_collections_json: JSON.stringify([EVENT]), + configured_phases_json: JSON.stringify(["live"]), + initial_mode: "future", + required_for_activation: 0, + bootstrap_state: "ready", + acknowledged_position: 0, + bootstrap_anchor_position: null, + }, + ]); + + const coverage = await db + .prepare( + `SELECT collection, phase, from_position, through_position + FROM change_log_coverage ORDER BY collection, phase`, + ) + .all(); + expect(coverage.results).toEqual([ + { + collection: EVENT, + phase: "historical", + from_position: 0, + through_position: null, + }, + { + collection: EVENT, + phase: "live", + from_position: 0, + through_position: null, + }, + ]); + + // Initialization is idempotent and retains one random database generation. + await initSchema(db, resolved); + expect((await getChangeLogState(db))?.generation).toBe(state?.generation); + }); + + it("appends only committed logical current-state changes", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + + const original = mutation({ + sourceTime: 100, + cid: "cid-one", + value: { name: "one" }, + }); + await ingestRecords(db, [original], resolved, { phase: "historical" }); + expect((await getChangeLogState(db))?.head).toBe("1"); + + // Exact replay and a newer source observation of the same immutable state + // may update source metadata but do not wake current-state consumers. + await ingestRecords(db, [original], resolved, { phase: "historical" }); + await ingestRecords( + db, + [ + mutation({ + sourceTime: 110, + cid: "cid-one", + value: { name: "one" }, + }), + ], + resolved, + { phase: "live" }, + ); + expect((await getChangeLogState(db))?.head).toBe("1"); + + await ingestRecords( + db, + [ + mutation({ + sourceTime: 200, + cid: "cid-two", + value: { name: "two" }, + }), + ], + resolved, + { phase: "live" }, + ); + await ingestRecords( + db, + [mutation({ sourceTime: 300, operation: "delete" })], + resolved, + { phase: "live" }, + ); + await ingestRecords( + db, + [mutation({ sourceTime: 400, operation: "delete" })], + resolved, + { phase: "live" }, + ); + + const rows = await batches(db); + expect(rows).toHaveLength(3); + expect(rows.map((row) => [row.position, row.phase, row.change_count])).toEqual([ + [1, "historical", 1], + [2, "live", 1], + [3, "live", 1], + ]); + const changes = rows.map((row) => JSON.parse(row.changes_json)[0]); + expect(changes[0]).toMatchObject({ + kind: "record", + operation: "put", + uri: URI, + cid: "cid-one", + version: { sourceId: "source", sourceTimeUs: 100 }, + }); + expect(changes[0]).not.toHaveProperty("id"); + expect(changes[0]).not.toHaveProperty("record"); + expect(changes[2]).toMatchObject({ + operation: "delete", + uri: URI, + cid: null, + }); + expect((await getChangeLogState(db))?.head).toBe("3"); + }); + + it("reduces multiple mutations for one URI to the final state", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + + await ingestRecords( + db, + [ + mutation({ sourceTime: 1, cid: "cid-one" }), + mutation({ sourceTime: 2, cid: "cid-two" }), + mutation({ sourceTime: 3, cid: "cid-three" }), + ], + resolved, + ); + + const rows = await batches(db); + expect(rows).toHaveLength(1); + expect(rows[0].change_count).toBe(1); + expect(JSON.parse(rows[0].changes_json)[0].cid).toBe("cid-three"); + }); + + it("does not log a phase outside every consumer's coverage", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + changes: { + consumers: { + webhook: { + collections: [EVENT], + phases: ["live"], + initial: "future", + }, + }, + }, + }); + await initSchema(db, resolved); + + await ingestRecords(db, [mutation({ sourceTime: 1 })], resolved, { + phase: "historical", + }); + expect((await getChangeLogState(db))?.head).toBe("0"); + expect(await batches(db)).toEqual([]); + }); + + it("rolls projection, log head, batch, and checkpoint back together", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + await db.prepare("CREATE TABLE change_failure (value TEXT UNIQUE)").run(); + await db.prepare("INSERT INTO change_failure VALUES ('duplicate')").run(); + + await expect( + ingestRecords(db, [mutation({ sourceTime: 1 })], resolved, { + trailingStatements: [ + saveCursorStatement(db, 1), + db.prepare("INSERT INTO change_failure VALUES ('duplicate')"), + ], + }), + ).rejects.toThrow(); + + expect((await getChangeLogState(db))?.head).toBe("0"); + expect(await batches(db)).toEqual([]); + expect( + await db.prepare("SELECT uri FROM records_event").first(), + ).toBeNull(); + expect(await db.prepare("SELECT time_us FROM cursor").first()).toBeNull(); + }); + + it("gives each fresh database a distinct generation", async () => { + const resolved = loggedConfig(); + const first = createSqliteDatabase(":memory:"); + const second = createSqliteDatabase(":memory:"); + await initSchema(first, resolved); + await initSchema(second, resolved); + expect((await getChangeLogState(first))?.generation).not.toBe( + (await getChangeLogState(second))?.generation, + ); + }); + + it("fails closed for unsafe enable, disable, and definition changes", async () => { + const populated = createSqliteDatabase(":memory:"); + const disabled = config(); + await initSchema(populated, disabled); + await ingestRecords(populated, [mutation({ sourceTime: 1 })], disabled); + await expect(initSchema(populated, loggedConfig())).rejects.toThrow( + "fresh empty generation", + ); + expect( + await populated + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='change_log_state'", + ) + .first(), + ).toBeNull(); + + const initialized = createSqliteDatabase(":memory:"); + await initSchema(initialized, loggedConfig()); + await expect(initSchema(initialized, config())).rejects.toThrow( + "cannot be disabled", + ); + const changed = config({ + changes: { + consumers: { + replacement: { collections: [EVENT], initial: "future" }, + }, + }, + }); + await expect(initSchema(initialized, changed)).rejects.toThrow( + "definitions differ", + ); + }); + + it("retries a losing overlapping writer from fresh durable state", async () => { + const real = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(real, resolved); + + let arrivals = 0; + let releaseBoth!: () => void; + const both = new Promise((resolve) => { + releaseBoth = resolve; + }); + let releaseNewer!: () => void; + const newerDone = new Promise((resolve) => { + releaseNewer = resolve; + }); + + function overlapping(role: "older" | "newer"): Database { + let projectionBatches = 0; + return { + prepare(sql: string): Statement { + return real.prepare(sql); + }, + async batch(statements: Statement[]) { + projectionBatches++; + if (projectionBatches > 1) return real.batch(statements); + arrivals++; + if (arrivals === 2) releaseBoth(); + await both; + if (role === "older") await newerDone; + try { + return await real.batch(statements); + } finally { + if (role === "newer") releaseNewer(); + } + }, + dialect: real.dialect, + }; + } + + const olderDb = overlapping("older"); + const newerDb = overlapping("newer"); + const [older] = await Promise.all([ + ingestRecords( + olderDb, + [mutation({ sourceTime: 100, cid: "cid-old", value: { name: "old" } })], + resolved, + { trailingStatements: [saveCursorStatement(real, 100)] }, + ), + ingestRecords( + newerDb, + [mutation({ sourceTime: 200, cid: "cid-new", value: { name: "new" } })], + resolved, + { trailingStatements: [saveCursorStatement(real, 200)] }, + ), + ]); + + expect(older.dropped.superseded).toBe(1); + const visible = await queryRecords(real, resolved, { collection: "event" }); + expect(JSON.parse(visible.records[0]!.record!).name).toBe("new"); + expect((await getChangeLogState(real))?.head).toBe("1"); + expect(JSON.parse((await batches(real))[0].changes_json)[0].cid).toBe( + "cid-new", + ); + expect( + await real.prepare("SELECT time_us FROM cursor WHERE id = 1").first(), + ).toEqual({ time_us: 200 }); + }); +}); diff --git a/packages/contrail/tests/ingest.test.ts b/packages/contrail/tests/ingest.test.ts index eefa59b..c70cd74 100644 --- a/packages/contrail/tests/ingest.test.ts +++ b/packages/contrail/tests/ingest.test.ts @@ -106,7 +106,9 @@ describe("ingestRecords", () => { const versionInserts = sql.filter((statement) => statement.startsWith("INSERT INTO record_versions"), ); - expect(versionInserts).toHaveLength(4); + // Thirteen bindings include the optimistic projection token, so seven + // versions fit under D1's 100-binding statement ceiling. + expect(versionInserts).toHaveLength(5); expect( versionInserts.every( (statement) => (statement.match(/\?/g) ?? []).length <= 100, diff --git a/packages/contrail/tests/postgres-concurrent-init.test.ts b/packages/contrail/tests/postgres-concurrent-init.test.ts index 954bbb3..dd6fdae 100644 --- a/packages/contrail/tests/postgres-concurrent-init.test.ts +++ b/packages/contrail/tests/postgres-concurrent-init.test.ts @@ -71,7 +71,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', 'backfills', 'backfill_state', 'discovery', 'cursor', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); diff --git a/packages/contrail/tests/postgres-e2e.test.ts b/packages/contrail/tests/postgres-e2e.test.ts index e5032ea..0e5d499 100644 --- a/packages/contrail/tests/postgres-e2e.test.ts +++ b/packages/contrail/tests/postgres-e2e.test.ts @@ -18,7 +18,7 @@ import { saveCursor, } from "../src/index"; import { resolveConfig } from "../src/index"; -import type { Database } from "../src/index"; +import type { Database, Statement } from "../src/index"; import { resolveHydrates, resolveReferences } from "../src/index"; import { makeEvent } from "./helpers"; @@ -78,7 +78,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', 'backfills', 'backfill_state', 'discovery', 'cursor', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); @@ -171,6 +171,81 @@ if (!PG_URL) { expect(result.records).toHaveLength(0); }); + it("retries an overlapping stale projector after the transaction lock", async () => { + const uri = "at://did:plc:test/community.lexicon.calendar.event/concurrent"; + let arrivals = 0; + let releaseBoth!: () => void; + const both = new Promise((resolve) => { + releaseBoth = resolve; + }); + let releaseNewer!: () => void; + const newerDone = new Promise((resolve) => { + releaseNewer = resolve; + }); + + const overlap = (role: "older" | "newer"): Database => { + let writes = 0; + return { + prepare(sql: string): Statement { + return db.prepare(sql); + }, + async batch(statements: Statement[]) { + writes++; + if (writes > 1) return db.batch(statements); + arrivals++; + if (arrivals === 2) releaseBoth(); + await both; + if (role === "older") await newerDone; + try { + return await db.batch(statements); + } finally { + if (role === "newer") releaseNewer(); + } + }, + dialect: db.dialect, + }; + }; + + const [older] = await Promise.all([ + ingestRecords( + overlap("older"), + [ + makeEvent({ + uri, + rkey: "concurrent", + cid: "cid-old", + record: { name: "old", mode: "online" }, + time_us: 100, + indexed_at: 100, + }), + ], + TEST_CONFIG, + ), + ingestRecords( + overlap("newer"), + [ + makeEvent({ + uri, + rkey: "concurrent", + cid: "cid-new", + record: { name: "new", mode: "online" }, + time_us: 200, + indexed_at: 200, + }), + ], + TEST_CONFIG, + ), + ]); + + expect(older.dropped.superseded).toBe(1); + const row = await pool.query( + "SELECT record, cid FROM records_community_lexicon_calendar_event WHERE uri = $1", + [uri], + ); + expect(row.rows[0]).toMatchObject({ cid: "cid-new" }); + expect(row.rows[0].record.name).toBe("new"); + }); + it("does nothing for empty events", async () => { await ingestRecords(db, []); const cursor = await getLastCursor(db); diff --git a/packages/contrail/tests/postgres.test.ts b/packages/contrail/tests/postgres.test.ts index 2738dc6..0b7b530 100644 --- a/packages/contrail/tests/postgres.test.ts +++ b/packages/contrail/tests/postgres.test.ts @@ -57,7 +57,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', 'backfills', 'backfill_state', 'discovery', 'cursor', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); diff --git a/packages/contrail/tests/schema.test.ts b/packages/contrail/tests/schema.test.ts index 15643dc..3a95149 100644 --- a/packages/contrail/tests/schema.test.ts +++ b/packages/contrail/tests/schema.test.ts @@ -21,6 +21,8 @@ describe("initSchema", () => { expect(names).toContain("identities"); expect(names).toContain("record_versions"); expect(names).toContain("ingest_diagnostics"); + expect(names).toContain("_contrail_projection_state"); + expect(names).not.toContain("change_log_state"); const backfillColumns = await db .prepare("PRAGMA table_info(backfills)") @@ -115,6 +117,52 @@ describe("initSchema", () => { ); }); + it("migrates optimistic tokens onto existing version rows", async () => { + const db = createTestDb(); + await db + .prepare( + `CREATE TABLE record_versions ( + uri TEXT PRIMARY KEY, + did TEXT NOT NULL, + collection TEXT NOT NULL, + rkey TEXT NOT NULL, + operation TEXT NOT NULL, + cid TEXT, + source_id TEXT NOT NULL, + source_revision TEXT, + source_time_us BIGINT NOT NULL, + source_cursor TEXT, + indexed_at BIGINT NOT NULL + )`, + ) + .run(); + await db + .prepare( + `INSERT INTO record_versions + (uri, did, collection, rkey, operation, cid, source_id, + source_revision, source_time_us, source_cursor, indexed_at) + VALUES (?, ?, ?, ?, 'update', ?, 'legacy', NULL, 1, NULL, 1)`, + ) + .bind( + "at://did:plc:legacy/community.lexicon.calendar.event/one", + "did:plc:legacy", + "community.lexicon.calendar.event", + "one", + "cid-legacy", + ) + .run(); + + await initSchema(db, TEST_CONFIG); + expect( + await db + .prepare("SELECT projection_token FROM record_versions") + .first(), + ).toEqual({ + projection_token: + "at://did:plc:legacy/community.lexicon.calendar.event/one", + }); + }); + it("seeds ordering metadata for visible rows from older schemas", async () => { const db = createTestDb(); await db @@ -149,7 +197,7 @@ describe("initSchema", () => { const version = await db .prepare( - "SELECT operation, source_id, source_time_us, cid FROM record_versions WHERE uri = ?", + "SELECT operation, source_id, source_time_us, cid, projection_token FROM record_versions WHERE uri = ?", ) .bind("at://did:plc:legacy/community.lexicon.calendar.event/one") .first<{ @@ -157,12 +205,15 @@ describe("initSchema", () => { source_id: string; source_time_us: number; cid: string; + projection_token: string; }>(); expect(version).toEqual({ operation: "update", source_id: "legacy", source_time_us: 200, cid: "cid-legacy", + projection_token: + "at://did:plc:legacy/community.lexicon.calendar.event/one", }); }); diff --git a/turbo.json b/turbo.json index 94b9292..2279a64 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,7 @@ "dependsOn": ["^build"] }, "test": { - "dependsOn": ["^build"], + "dependsOn": ["^build", "build"], "outputs": [] }, "dev": { -- 2.51.2 From cd0771e3f537e23386aac390cdd0bb7e0819cc2e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:04:25 +0200 Subject: [PATCH 03/34] Add durable change consumer protocol --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 27 +- packages/contrail/package.json | 2 +- packages/contrail/src/cli.ts | 5 +- packages/contrail/src/cli/commands/changes.ts | 127 +++ packages/contrail/src/contrail.ts | 7 + packages/contrail/src/core/change-log.ts | 20 +- packages/contrail/src/core/changes.ts | 981 ++++++++++++++++++ packages/contrail/src/core/db/schema.ts | 13 +- packages/contrail/src/index.ts | 1 + packages/contrail/tests/built-changes.mjs | 42 + packages/contrail/tests/built-sqlite.mjs | 2 + .../contrail/tests/change-consumers.test.ts | 351 +++++++ packages/contrail/tests/postgres-e2e.test.ts | 62 ++ 14 files changed, 1625 insertions(+), 17 deletions(-) create mode 100644 packages/contrail/src/cli/commands/changes.ts create mode 100644 packages/contrail/src/core/changes.ts create mode 100644 packages/contrail/tests/built-changes.mjs create mode 100644 packages/contrail/tests/change-consumers.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 6a3e1e5..b76f0f1 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs plus `contrail changes status/retry` commands. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 99614f1..33d0b54 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -96,7 +96,32 @@ const config = { Static definitions contain no handlers, URLs, clients, credentials, or secrets. Contrail registers them with a random database-generation ID and collection/phase coverage ledger. A winning logical put/delete appends one compact URI/version reference in the same transaction as canonical and derived state plus the source checkpoint. Duplicate, stale, same-CID, absent-delete, rejected, and rolled-back mutations append nothing. Record bodies are hydrated from current state by the later delivery layer rather than copied into the log. -This milestone intentionally exposes only the atomic log foundation; leased claim/hydrate/ack delivery and current-state bootstrap APIs follow separately. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. +Low-level delivery uses bounded leases and compare-and-swap acknowledgement: + +```ts +const claim = await contrail.changes.claim("webhooks", { + maxBatches: 20, + maxChanges: 500, + maxBytes: 512_000, + leaseMs: 30_000, +}); +if (claim) { + try { + const batch = await contrail.changes.hydrate(claim); + await deliverIdempotently(batch); + await contrail.changes.ack(claim); + } catch { + await contrail.changes.fail(claim, { + code: "destination_unavailable", + nextAttemptAt: Date.now() + 30_000, + }); + } +} +``` + +Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and resolve delete/recreate races from newest canonical state. Consumers lease and progress independently; irrelevant position ranges advance without invoking a handler. Delivery is intentionally at least once—a destination success followed by an acknowledgement crash causes duplicate delivery. Handlers must be idempotent by stable record/document key. + +`initial: "future"` and `initial: "history"` consumers can use this API now. `initial: "current"` remains pending until the snapshot-plus-tail bootstrap coordinator lands in the next milestone. `contrail changes status` and `contrail changes retry ` expose private status and manual retry for SQLite or Wrangler D1 deployments. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. ## Local development diff --git a/packages/contrail/package.json b/packages/contrail/package.json index d695f92..86378ce 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,7 +73,7 @@ "clean": "rm -rf dist", "typecheck": "tsc --noEmit", "test": "vitest run", - "test:built": "node tests/built-sqlite.mjs && node tests/built-alluvium.mjs && node tests/built-lexicons.mjs && node tests/built-client.mjs", + "test:built": "node tests/built-sqlite.mjs && node tests/built-alluvium.mjs && node tests/built-lexicons.mjs && node tests/built-client.mjs && node tests/built-changes.mjs", "test:watch": "vitest" }, "dependencies": { diff --git a/packages/contrail/src/cli.ts b/packages/contrail/src/cli.ts index 7299c6b..2aa6d9a 100644 --- a/packages/contrail/src/cli.ts +++ b/packages/contrail/src/cli.ts @@ -11,6 +11,7 @@ import { registerDev } from "./cli/commands/dev.js"; import { registerAppendScheduled } from "./cli/commands/append-scheduled.js"; import { registerConnect } from "./cli/commands/connect.js"; import { registerLexicons } from "./cli/commands/lexicons.js"; +import { registerChanges } from "./cli/commands/changes.js"; const cli = cac("contrail"); @@ -19,11 +20,13 @@ registerDev(cli); registerAppendScheduled(cli); registerConnect(cli); registerLexicons(cli); +registerChanges(cli); cli.help(); try { - cli.parse(); + cli.parse(process.argv, { run: false }); + await cli.runMatchedCommand(); } catch (err) { console.error(err); process.exit(1); diff --git a/packages/contrail/src/cli/commands/changes.ts b/packages/contrail/src/cli/commands/changes.ts new file mode 100644 index 0000000..d035b55 --- /dev/null +++ b/packages/contrail/src/cli/commands/changes.ts @@ -0,0 +1,127 @@ +import type { CAC } from "cac"; +import { Contrail } from "../../contrail.js"; +import type { Database } from "../../core/types.js"; +import { + resolveAndLoadConfig, + resolveValidationLexicons, +} from "../shared.js"; + +interface ChangeCommandOptions { + config?: string; + root?: string; + remote?: boolean; + binding: string; + sqlite?: string; + json?: boolean; +} + +async function withChangesDatabase( + options: ChangeCommandOptions, + callback: (contrail: Contrail, db: Database) => Promise, +): Promise { + if (options.sqlite && options.remote) { + throw new Error("--sqlite cannot be combined with --remote"); + } + const config = await resolveAndLoadConfig(options); + const lexicons = await resolveValidationLexicons(options, config); + const contrail = new Contrail({ ...config, lexicons }); + if (options.sqlite) { + const { createSqliteDatabase } = await import("../../adapters/sqlite.js"); + const db = createSqliteDatabase(options.sqlite); + await contrail.init(db); + return callback(contrail, db); + } + + const { getPlatformProxy } = await import("wrangler"); + const { env, dispose } = await getPlatformProxy({ + environment: options.remote ? "production" : undefined, + }); + const binding = options.binding ?? "DB"; + const db = (env as Record)[binding] as Database | undefined; + if (!db) { + await dispose(); + throw new Error(`No binding named "${binding}" in wrangler env`); + } + try { + await contrail.init(db); + return await callback(contrail, db); + } finally { + await dispose(); + } +} + +function options(command: ReturnType) { + return command + .option("--config ", "Path to Contrail config file (TS or JS)") + .option("--root ", "Project root for auto-detection (default: CWD)") + .option("--remote", "Use production D1 bindings") + .option("--binding ", "D1 binding name in wrangler.jsonc", { + default: "DB", + }) + .option( + "--sqlite ", + "Use a local SQLite database instead of a Wrangler D1 binding", + ); +} + +export function registerChanges(cli: CAC): void { + options( + cli.command( + "changes [consumer]", + "Private change-log operations: status, retry ", + ), + ) + .option("--json", "Print machine-readable status JSON") + .action( + async ( + action: string, + consumer: string | undefined, + commandOptions: ChangeCommandOptions, + ) => { + if (action !== "status" && action !== "retry") { + throw new Error("changes action must be 'status' or 'retry'"); + } + if (action === "retry" && !consumer) { + throw new Error("changes retry requires a consumer ID"); + } + if (action === "status" && consumer) { + throw new Error("changes status does not accept a consumer ID"); + } + + await withChangesDatabase(commandOptions, async (contrail, db) => { + if (action === "retry") { + await contrail.changes.retry(consumer!, undefined, db); + console.log(`change consumer ${consumer}: retry is now due`); + return; + } + + const status = await contrail.changes.status(db); + if (commandOptions.json) { + console.log(JSON.stringify(status, null, 2)); + return; + } + if (!status.enabled || !status.state) { + console.log("change log: disabled"); + return; + } + console.log( + `change log: generation=${status.state.generation} head=${status.state.head} ` + + `floor=${status.state.retainedFloor} rows=${status.rows} ` + + `changes=${status.changes} bytes=${status.bytes}`, + ); + for (const item of status.consumers) { + const retry = + item.nextAttemptAt === null + ? "due" + : new Date(item.nextAttemptAt).toISOString(); + console.log( + ` ${item.id}: state=${item.bootstrapState} ` + + `position=${item.position} backlog=${item.backlogBatches}/` + + `${item.backlogChanges} attempts=${item.attempts} ` + + `retry=${retry} leased=${item.leased ? "yes" : "no"}`, + ); + } + }); + }, + ); +} diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index bba7572..859d054 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -14,6 +14,7 @@ import { prepareRecordValidation, } from "./core/validation"; import { getIngestDiagnostics } from "./core/diagnostics"; +import { ChangeConsumers } from "./core/changes"; import { optimizeDatabase } from "./core/db/optimize"; import { assertServingSourceCompatibility, @@ -60,6 +61,8 @@ export interface ContrailOptions extends ContrailConfig { export class Contrail { readonly config: ResolvedContrailConfig; + /** Durable low-level claim/hydrate/ack consumer API. */ + readonly changes: ChangeConsumers; private _db?: Database; private _ingestState: IngestState = createIngestState(); @@ -72,6 +75,10 @@ export class Contrail { // Otherwise init/app binds the runtime's generated bundle first. if (lexicons) prepareRecordValidation(this.config); this._db = db; + this.changes = new ChangeConsumers( + this.config, + (database) => this.getDb(database), + ); } private getDb(db?: Database): Database { diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 6505ce9..5973925 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -102,6 +102,7 @@ export function buildChangeLogSchema( phase TEXT NOT NULL CHECK (phase IN ('historical', 'live')), changes_json TEXT NOT NULL, change_count INTEGER NOT NULL, + encoded_bytes INTEGER NOT NULL, created_at ${bigint} NOT NULL, PRIMARY KEY (generation_id, position) )`, @@ -235,8 +236,8 @@ export async function initializeChangeLog( configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, bootstrap_anchor_position, attempts, updated_at) - SELECT ?, generation_id, head_position, ?, ?, ?, ?, ?, - CASE WHEN ? = 'current' THEN head_position ELSE NULL END, + SELECT ?, generation_id, 0, ?, ?, ?, ?, ?, + CASE WHEN ? = 'current' THEN 0 ELSE NULL END, 0, ? FROM change_log_state WHERE id = 1 AND definitions_json = ? @@ -262,7 +263,7 @@ export async function initializeChangeLog( .prepare( `INSERT INTO change_log_coverage (generation_id, collection, phase, from_position, through_position) - SELECT generation_id, ?, ?, head_position, NULL + SELECT generation_id, ?, ?, 0, NULL FROM change_log_state WHERE id = 1 AND definitions_json = ? ON CONFLICT(generation_id, collection, phase, from_position) @@ -312,7 +313,6 @@ async function assertChangeLogDefinition( for (let index = 0; index < expectedConsumers.length; index++) { const [id, expected] = expectedConsumers[index]!; const actual = consumers.results[index]!; - const bootstrapState = expected.initial === "current" ? "pending" : "ready"; if ( actual.consumer_id !== id || actual.generation_id !== state.generation_id || @@ -320,12 +320,7 @@ async function assertChangeLogDefinition( actual.configured_phases_json !== canonicalPhases(changeConsumerPhases(expected)) || actual.initial_mode !== expected.initial || Number(actual.required_for_activation) !== - (expected.requiredForActivation === true ? 1 : 0) || - actual.bootstrap_state !== bootstrapState || - Number(actual.acknowledged_position) !== 0 || - (expected.initial === "current" - ? Number(actual.bootstrap_anchor_position) !== 0 - : actual.bootstrap_anchor_position !== null) + (expected.requiredForActivation === true ? 1 : 0) ) { throw new Error(`Durable change consumer ${id} is incompatible`); } @@ -499,11 +494,11 @@ export function appendChangeLogStatements( `INSERT INTO change_batches (generation_id, position, projection_transaction_id, source_id, source_epoch, source_cursor, phase, changes_json, change_count, - created_at) + encoded_bytes, created_at) VALUES ( (SELECT generation_id FROM change_log_state WHERE id = 1), (SELECT head_position FROM change_log_state WHERE id = 1), - ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ? )`, ) .bind( @@ -514,6 +509,7 @@ export function appendChangeLogStatements( phase, serialized, changes.length, + bytes, now, ), ]; diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts new file mode 100644 index 0000000..8482453 --- /dev/null +++ b/packages/contrail/src/core/changes.ts @@ -0,0 +1,981 @@ +import type { + ContrailConfig, + Database, + ProjectionPhase, +} from "./types"; +import { + changeConsumerPhases, + recordsTableName, + resolveCollectionKey, +} from "./types"; +import { + getChangeLogState, + type ChangeLogState, + type RecordChange, +} from "./change-log"; + +const DEFAULT_MAX_BATCHES = 20; +const DEFAULT_MAX_CHANGES = 500; +const DEFAULT_MAX_BYTES = 512_000; +const DEFAULT_LEASE_MS = 30_000; +const DEFAULT_AUTO_ADVANCE_RANGES = 8; +const MAX_CLAIM_BATCHES = 100; +const MAX_CLAIM_CHANGES = 5_000; +const MAX_CLAIM_BYTES = 4 * 1_024 * 1_024; +const MAX_LEASE_MS = 10 * 60_000; +const FAILURE_CODE = /^[a-zA-Z0-9_.:-]{1,64}$/; + +export class ChangeConsumerNotFoundError extends Error {} +export class ChangeClaimTooLargeError extends Error {} +export class ChangeLeaseLostError extends Error {} +export class ChangeHistoryGapError extends Error {} +export class ChangeGenerationMismatchError extends Error {} + +export interface ChangeClaimOptions { + maxBatches?: number; + maxChanges?: number; + maxBytes?: number; + leaseMs?: number; + /** Maximum irrelevant position ranges core may acknowledge without invoking + * a handler in one claim call. */ + maxAutoAdvanceRanges?: number; + /** @internal Deterministic clock seam for conformance tests. */ + now?: number; +} + +export interface ChangeClaim { + consumerId: string; + generation: string; + from: string; + through: string; + changes: RecordChange[]; + attempt: number; + leaseExpiresAt: number; + /** Opaque CAS capability. Do not pass this field to delivery handlers. */ + readonly leaseOwner: string; +} + +export interface CurrentRecord { + uri: string; + did: string; + collection: string; + rkey: string; + cid: string | null; + value: unknown; + timeUs: number; + indexedAt: number; +} + +export interface DeliveryBatch { + consumerId: string; + cursor: { + generation: string; + from: string; + through: string; + }; + changes: RecordChange[]; + currentRecords: CurrentRecord[]; + absentUris: string[]; +} + +export interface ChangeFailure { + /** Stable sanitized category. Raw destination errors are never persisted. */ + code: string; + /** Runtime-computed retry eligibility. Null makes the claim immediately due. */ + nextAttemptAt: number | null; +} + +export interface ChangeConsumerStatus { + id: string; + generation: string; + position: string; + bootstrapState: string; + initialMode: string; + requiredForActivation: boolean; + attempts: number; + nextAttemptAt: number | null; + lastSuccessAt: number | null; + lastErrorCode: string | null; + lastErrorAt: number | null; + leased: boolean; + leaseExpiresAt: number | null; + backlogBatches: number; + backlogChanges: number; + backlogBytes: number; + oldestPendingAt: number | null; + generationMatches: boolean; +} + +export interface ChangeLogStatus { + enabled: boolean; + state: ChangeLogState | null; + rows: number; + changes: number; + bytes: number; + oldestRetainedAt: number | null; + consumers: ChangeConsumerStatus[]; +} + +interface DurableConsumerRow { + consumer_id: string; + generation_id: string; + acknowledged_position: number | string; + configured_collections_json: string; + configured_phases_json: string; + initial_mode: string; + required_for_activation: number | string; + bootstrap_state: string; + lease_owner: string | null; + lease_expires_at: number | string | null; + attempts: number | string; + next_attempt_at: number | string | null; + last_success_at: number | string | null; + last_error_code: string | null; + last_error_at: number | string | null; +} + +interface BatchMetadataRow { + position: number | string; + phase: ProjectionPhase; + change_count: number | string; + encoded_bytes: number | string; +} + +interface StoredBatchRow extends BatchMetadataRow { + changes_json: string; +} + +interface BacklogRow { + backlog_batches: number | string; + backlog_changes: number | string | null; + backlog_bytes: number | string | null; + oldest_pending_at: number | string | null; +} + +function integer( + value: number | undefined, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new TypeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return result; +} + +function timestamp(value: number | undefined = Date.now()): number { + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError("now must be a non-negative safe integer"); + } + return value; +} + +function nullableNumber(value: number | string | null): number | null { + return value === null ? null : Number(value); +} + +function parseStringArray(value: string, label: string): string[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error(`${label} is not valid JSON`); + } + if ( + !Array.isArray(parsed) || + !parsed.every((item) => typeof item === "string") || + new Set(parsed).size !== parsed.length + ) { + throw new Error(`${label} is malformed`); + } + return parsed; +} + +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function decodeChange( + value: unknown, + generation: string, + position: string, + ordinal: number, +): RecordChange { + if ( + !object(value) || + value.kind !== "record" || + (value.operation !== "put" && value.operation !== "delete") || + typeof value.uri !== "string" || + typeof value.did !== "string" || + typeof value.collection !== "string" || + typeof value.rkey !== "string" || + (value.cid !== null && typeof value.cid !== "string") || + !object(value.version) || + typeof value.version.sourceId !== "string" || + (value.version.sourceEpoch !== null && + typeof value.version.sourceEpoch !== "string") || + (value.version.sourceRevision !== null && + typeof value.version.sourceRevision !== "string") || + !Number.isSafeInteger(value.version.sourceTimeUs) || + Number(value.version.sourceTimeUs) < 0 || + (value.version.sourceCursor !== null && + typeof value.version.sourceCursor !== "string") + ) { + throw new Error(`Change batch ${generation}/${position} is malformed`); + } + return { + ...(value as unknown as Omit), + id: `${generation}:${position}:${ordinal}`, + }; +} + +function decodeBatchChanges( + row: StoredBatchRow, + generation: string, +): RecordChange[] { + const position = String(row.position); + const encoded = new TextEncoder().encode(row.changes_json).byteLength; + if (encoded !== Number(row.encoded_bytes)) { + throw new Error(`Change batch ${generation}/${position} byte count is invalid`); + } + let parsed: unknown; + try { + parsed = JSON.parse(row.changes_json); + } catch { + throw new Error(`Change batch ${generation}/${position} is not valid JSON`); + } + if (!Array.isArray(parsed) || parsed.length !== Number(row.change_count)) { + throw new Error(`Change batch ${generation}/${position} count is invalid`); + } + return parsed.map((change, ordinal) => + decodeChange(change, generation, position, ordinal), + ); +} + +async function durableConsumer( + db: Database, + consumerId: string, +): Promise { + return db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, + initial_mode, required_for_activation, bootstrap_state, + lease_owner, lease_expires_at, attempts, next_attempt_at, + last_success_at, last_error_code, last_error_at + FROM change_consumers WHERE consumer_id = ?`, + ) + .bind(consumerId) + .first(); +} + +async function releaseLease( + db: Database, + consumerId: string, + generation: string, + from: string, + owner: string, + now: number, +): Promise { + await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ?`, + ) + .bind(now, consumerId, generation, from, owner) + .run(); +} + +function normalizedClaimOptions(options: ChangeClaimOptions = {}) { + return { + maxBatches: integer( + options.maxBatches, + DEFAULT_MAX_BATCHES, + 1, + MAX_CLAIM_BATCHES, + "maxBatches", + ), + maxChanges: integer( + options.maxChanges, + DEFAULT_MAX_CHANGES, + 1, + MAX_CLAIM_CHANGES, + "maxChanges", + ), + maxBytes: integer( + options.maxBytes, + DEFAULT_MAX_BYTES, + 1, + MAX_CLAIM_BYTES, + "maxBytes", + ), + leaseMs: integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ), + maxAutoAdvanceRanges: integer( + options.maxAutoAdvanceRanges, + DEFAULT_AUTO_ADVANCE_RANGES, + 0, + 32, + "maxAutoAdvanceRanges", + ), + now: timestamp(options.now), + }; +} + +/** Verify that schema initialization registered one configured consumer. */ +export async function registerChangeConsumer( + db: Database, + config: ContrailConfig, + consumerId: string, +): Promise { + const configured = config.changes?.consumers[consumerId]; + if (!configured) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not configured`, + ); + } + const state = await getChangeLogState(db); + if (!state) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const durable = await durableConsumer(db, consumerId); + if (!durable) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const collections = parseStringArray( + durable.configured_collections_json, + `Change consumer ${consumerId} collections`, + ); + const phases = parseStringArray( + durable.configured_phases_json, + `Change consumer ${consumerId} phases`, + ); + if ( + durable.generation_id !== state.generation || + JSON.stringify(collections) !== + JSON.stringify([...configured.collections].sort()) || + JSON.stringify(phases) !== + JSON.stringify([...changeConsumerPhases(configured)].sort()) || + durable.initial_mode !== configured.initial || + Number(durable.required_for_activation) !== + (configured.requiredForActivation === true ? 1 : 0) + ) { + throw new Error(`Change consumer ${consumerId} is incompatible`); + } +} + +/** Claim one bounded contiguous position range. Irrelevant ranges advance by + * CAS without invoking application code. */ +export async function claimChanges( + db: Database, + consumerId: string, + options: ChangeClaimOptions = {}, +): Promise { + const limits = normalizedClaimOptions(options); + if (!(await getChangeLogState(db))) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + + for ( + let autoAdvanced = 0; + autoAdvanced <= limits.maxAutoAdvanceRanges; + autoAdvanced++ + ) { + const owner = crypto.randomUUID(); + const leaseExpiresAt = limits.now + limits.leaseMs; + const leased = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? + AND bootstrap_state = 'ready' + AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) + AND acknowledged_position >= (SELECT retained_floor_position FROM change_log_state WHERE id = 1) + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, + initial_mode, required_for_activation, bootstrap_state, + lease_owner, lease_expires_at, attempts, next_attempt_at, + last_success_at, last_error_code, last_error_at`, + ) + .bind( + owner, + leaseExpiresAt, + limits.now, + consumerId, + limits.now, + limits.now, + ) + .first(); + if (!leased) { + const durable = await durableConsumer(db, consumerId); + if (!durable) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const state = await getChangeLogState(db); + if (state && durable.generation_id !== state.generation) { + throw new ChangeGenerationMismatchError( + `Change consumer ${consumerId} belongs to generation ${durable.generation_id}, not ${state.generation}`, + ); + } + if ( + state && + BigInt(String(durable.acknowledged_position)) < + BigInt(state.retainedFloor) + ) { + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} is behind retained floor ${state.retainedFloor}`, + ); + } + return null; + } + + const generation = leased.generation_id; + const from = String(leased.acknowledged_position); + const metadata = await db + .prepare( + `SELECT position, phase, change_count, encoded_bytes + FROM change_batches + WHERE generation_id = ? AND position > ? + ORDER BY position + LIMIT ?`, + ) + .bind(generation, from, limits.maxBatches) + .all(); + + if (metadata.results.length === 0) { + const state = await getChangeLogState(db); + await releaseLease(db, consumerId, generation, from, owner, limits.now); + if (state && state.generation === generation && state.head !== from) { + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} cannot resume from ${from}; log head is ${state.head}`, + ); + } + return null; + } + + const expectedFirst = BigInt(from) + 1n; + if (BigInt(String(metadata.results[0]!.position)) !== expectedFirst) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} has a gap after ${from}`, + ); + } + + for (let index = 0; index < metadata.results.length; index++) { + const expected = BigInt(from) + BigInt(index) + 1n; + if (BigInt(String(metadata.results[index]!.position)) !== expected) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} has a gap after ${from}`, + ); + } + } + + const selected: BatchMetadataRow[] = []; + let totalChanges = 0; + let totalBytes = 0; + for (const row of metadata.results) { + const count = Number(row.change_count); + const bytes = Number(row.encoded_bytes); + if ( + !Number.isSafeInteger(count) || + count < 1 || + !Number.isSafeInteger(bytes) || + bytes < 1 + ) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new Error(`Change batch ${generation}/${row.position} has invalid bounds`); + } + if ( + selected.length > 0 && + (totalChanges + count > limits.maxChanges || + totalBytes + bytes > limits.maxBytes) + ) { + break; + } + if (count > limits.maxChanges || bytes > limits.maxBytes) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeClaimTooLargeError( + `Change batch ${generation}/${row.position} exceeds this consumer's claim limits`, + ); + } + selected.push(row); + totalChanges += count; + totalBytes += bytes; + } + + const through = String(selected.at(-1)!.position); + const rows = await db + .prepare( + `SELECT position, phase, change_count, encoded_bytes, changes_json + FROM change_batches + WHERE generation_id = ? AND position > ? AND position <= ? + ORDER BY position`, + ) + .bind(generation, from, through) + .all(); + if (rows.results.length !== selected.length) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} lost claimed history`, + ); + } + + const collections = new Set( + parseStringArray( + leased.configured_collections_json, + `Change consumer ${consumerId} collections`, + ), + ); + const phases = new Set( + parseStringArray( + leased.configured_phases_json, + `Change consumer ${consumerId} phases`, + ), + ); + const coalesced = new Map(); + for (const row of rows.results) { + if (!phases.has(row.phase)) continue; + for (const change of decodeBatchChanges(row, generation)) { + if (!collections.has(change.collection)) continue; + coalesced.delete(change.uri); + coalesced.set(change.uri, change); + } + } + + const claim: ChangeClaim = { + consumerId, + generation, + from, + through, + changes: [...coalesced.values()], + attempt: Number(leased.attempts) + 1, + leaseExpiresAt, + leaseOwner: owner, + }; + if (claim.changes.length > 0) return claim; + + await acknowledgeChanges(db, claim, { now: limits.now }); + if (autoAdvanced === limits.maxAutoAdvanceRanges) return null; + } + return null; +} + +/** Hydrate the newest canonical state for each coalesced claimed URI. */ +export async function hydrateChanges( + db: Database, + config: ContrailConfig, + claim: ChangeClaim, +): Promise { + const byCollection = new Map(); + for (const change of claim.changes) { + const uris = byCollection.get(change.collection) ?? []; + uris.push(change.uri); + byCollection.set(change.collection, uris); + } + + const found = new Map(); + for (const [collection, uris] of byCollection) { + const short = resolveCollectionKey(config, collection); + if (!short) { + throw new Error(`Claim references unconfigured collection ${collection}`); + } + const table = recordsTableName(short); + for (let index = 0; index < uris.length; index += 50) { + const chunk = uris.slice(index, index + 50); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare( + `SELECT uri, did, rkey, cid, record, time_us, indexed_at + FROM ${table} WHERE uri IN (${placeholders})`, + ) + .bind(...chunk) + .all<{ + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }>(); + for (const row of rows.results) { + let value: unknown; + try { + value = JSON.parse(row.record); + } catch { + throw new Error(`Current record ${row.uri} is not valid JSON`); + } + found.set(row.uri, { + uri: row.uri, + did: row.did, + collection, + rkey: row.rkey, + cid: row.cid, + value, + timeUs: Number(row.time_us), + indexedAt: Number(row.indexed_at), + }); + } + } + } + + const currentRecords: CurrentRecord[] = []; + const absentUris: string[] = []; + for (const change of claim.changes) { + const current = found.get(change.uri); + if (current) currentRecords.push(current); + else absentUris.push(change.uri); + } + return { + consumerId: claim.consumerId, + cursor: { + generation: claim.generation, + from: claim.from, + through: claim.through, + }, + changes: claim.changes, + currentRecords, + absentUris, + }; +} + +export async function acknowledgeChanges( + db: Database, + claim: ChangeClaim, + options: { now?: number } = {}, +): Promise { + const now = timestamp(options.now); + const acknowledged = await db + .prepare( + `UPDATE change_consumers + SET acknowledged_position = ?, lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_success_at = ?, last_error_code = NULL, last_error_at = NULL, + updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + AND ? > acknowledged_position + AND ? <= ( + SELECT head_position FROM change_log_state + WHERE id = 1 AND generation_id = ? + ) + RETURNING consumer_id`, + ) + .bind( + claim.through, + now, + now, + claim.consumerId, + claim.generation, + claim.from, + claim.leaseOwner, + now, + claim.through, + claim.through, + claim.generation, + ) + .first<{ consumer_id: string }>(); + if (!acknowledged) { + throw new ChangeLeaseLostError( + `Change claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +export async function renewChangeClaim( + db: Database, + claim: ChangeClaim, + options: { leaseMs?: number; now?: number } = {}, +): Promise { + const now = timestamp(options.now); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const expires = now + leaseMs; + const renewed = await db + .prepare( + `UPDATE change_consumers + SET lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + expires, + now, + claim.consumerId, + claim.generation, + claim.from, + claim.leaseOwner, + now, + ) + .first<{ consumer_id: string }>(); + if (!renewed) { + throw new ChangeLeaseLostError( + `Change claim for ${claim.consumerId} cannot be renewed`, + ); + } + return { ...claim, leaseExpiresAt: expires }; +} + +export async function failChanges( + db: Database, + claim: ChangeClaim, + failure: ChangeFailure, + options: { now?: number } = {}, +): Promise<{ attempts: number; nextAttemptAt: number | null }> { + const now = timestamp(options.now); + if (!FAILURE_CODE.test(failure.code)) { + throw new TypeError( + "Change failure code must contain 1-64 safe identifier characters", + ); + } + if ( + failure.nextAttemptAt !== null && + (!Number.isSafeInteger(failure.nextAttemptAt) || + failure.nextAttemptAt < now) + ) { + throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); + } + const failed = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, + attempts = attempts + 1, next_attempt_at = ?, + last_error_code = ?, last_error_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING attempts, next_attempt_at`, + ) + .bind( + failure.nextAttemptAt, + failure.code, + now, + now, + claim.consumerId, + claim.generation, + claim.from, + claim.leaseOwner, + now, + ) + .first<{ attempts: number | string; next_attempt_at: number | string | null }>(); + if (!failed) { + throw new ChangeLeaseLostError( + `Change claim for ${claim.consumerId} is stale or expired`, + ); + } + return { + attempts: Number(failed.attempts), + nextAttemptAt: nullableNumber(failed.next_attempt_at), + }; +} + +export async function retryChangeConsumer( + db: Database, + consumerId: string, + options: { now?: number } = {}, +): Promise { + const now = timestamp(options.now); + if (!(await getChangeLogState(db))) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const retried = await db + .prepare( + `UPDATE change_consumers + SET next_attempt_at = NULL, last_error_code = NULL, + last_error_at = NULL, updated_at = ? + WHERE consumer_id = ? + AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id`, + ) + .bind(now, consumerId, now) + .first<{ consumer_id: string }>(); + if (!retried) { + const existing = await durableConsumer(db, consumerId); + if (!existing) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + throw new ChangeLeaseLostError( + `Change consumer ${consumerId} currently has an active lease`, + ); + } +} + +export async function getChangesStatus(db: Database): Promise { + const state = await getChangeLogState(db); + if (!state) { + return { + enabled: false, + state: null, + rows: 0, + changes: 0, + bytes: 0, + oldestRetainedAt: null, + consumers: [], + }; + } + const aggregate = await db + .prepare( + `SELECT COUNT(*) AS rows, COALESCE(SUM(change_count), 0) AS changes, + COALESCE(SUM(encoded_bytes), 0) AS bytes, + MIN(created_at) AS oldest_retained_at + FROM change_batches WHERE generation_id = ?`, + ) + .bind(state.generation) + .first<{ + rows: number | string; + changes: number | string; + bytes: number | string; + oldest_retained_at: number | string | null; + }>(); + const rows = await db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, + initial_mode, required_for_activation, bootstrap_state, + lease_owner, lease_expires_at, attempts, next_attempt_at, + last_success_at, last_error_code, last_error_at + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + const consumers: ChangeConsumerStatus[] = []; + for (const row of rows.results) { + const backlog = await db + .prepare( + `SELECT COUNT(*) AS backlog_batches, + COALESCE(SUM(change_count), 0) AS backlog_changes, + COALESCE(SUM(encoded_bytes), 0) AS backlog_bytes, + MIN(created_at) AS oldest_pending_at + FROM change_batches + WHERE generation_id = ? AND position > ?`, + ) + .bind(row.generation_id, row.acknowledged_position) + .first(); + const generationMatches = row.generation_id === state.generation; + consumers.push({ + id: row.consumer_id, + generation: row.generation_id, + position: String(row.acknowledged_position), + bootstrapState: generationMatches ? row.bootstrap_state : "reset-required", + initialMode: row.initial_mode, + requiredForActivation: Number(row.required_for_activation) === 1, + attempts: Number(row.attempts), + nextAttemptAt: nullableNumber(row.next_attempt_at), + lastSuccessAt: nullableNumber(row.last_success_at), + lastErrorCode: row.last_error_code, + lastErrorAt: nullableNumber(row.last_error_at), + leased: row.lease_owner !== null, + leaseExpiresAt: nullableNumber(row.lease_expires_at), + backlogBatches: Number(backlog?.backlog_batches ?? 0), + backlogChanges: Number(backlog?.backlog_changes ?? 0), + backlogBytes: Number(backlog?.backlog_bytes ?? 0), + oldestPendingAt: nullableNumber(backlog?.oldest_pending_at ?? null), + generationMatches, + }); + } + return { + enabled: true, + state, + rows: Number(aggregate?.rows ?? 0), + changes: Number(aggregate?.changes ?? 0), + bytes: Number(aggregate?.bytes ?? 0), + oldestRetainedAt: nullableNumber(aggregate?.oldest_retained_at ?? null), + consumers, + }; +} + +type DatabaseResolver = (db?: Database) => Database; + +/** Bound low-level API exposed as `contrail.changes`. */ +export class ChangeConsumers { + constructor( + private readonly config: ContrailConfig, + private readonly database: DatabaseResolver, + ) {} + + register(consumerId: string, db?: Database): Promise { + return registerChangeConsumer(this.database(db), this.config, consumerId); + } + + claim( + consumerId: string, + options?: ChangeClaimOptions, + db?: Database, + ): Promise { + return claimChanges(this.database(db), consumerId, options); + } + + hydrate(claim: ChangeClaim, db?: Database): Promise { + return hydrateChanges(this.database(db), this.config, claim); + } + + ack( + claim: ChangeClaim, + options?: { now?: number }, + db?: Database, + ): Promise { + return acknowledgeChanges(this.database(db), claim, options); + } + + renew( + claim: ChangeClaim, + options?: { leaseMs?: number; now?: number }, + db?: Database, + ): Promise { + return renewChangeClaim(this.database(db), claim, options); + } + + fail( + claim: ChangeClaim, + failure: ChangeFailure, + options?: { now?: number }, + db?: Database, + ): Promise<{ attempts: number; nextAttemptAt: number | null }> { + return failChanges(this.database(db), claim, failure, options); + } + + retry( + consumerId: string, + options?: { now?: number }, + db?: Database, + ): Promise { + return retryChangeConsumer(this.database(db), consumerId, options); + } + + status(db?: Database): Promise { + return getChangesStatus(this.database(db)); + } +} diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 51dfe2f..18b05b5 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -37,7 +37,7 @@ import { } from "../types"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 12; +export const CONTRAIL_SCHEMA_VERSION = 13; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -783,6 +783,17 @@ export async function initSchema( await assertFreshChangeLogGeneration(db); } for (const statement of changes) await runIdempotentDdl(db, statement); + await addColumnIfNotExists( + db, + "change_batches", + "encoded_bytes", + "INTEGER", + ); + await db + .prepare( + "UPDATE change_batches SET encoded_bytes = LENGTH(changes_json) WHERE encoded_bytes IS NULL", + ) + .run(); await initializeChangeLog(db, config); } diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 0e68f6c..47fbcc6 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -35,6 +35,7 @@ export { MAX_CHANGE_BATCH_CHANGES, } from "./core/change-log"; export type { ChangeLogState, RecordChange } from "./core/change-log"; +export * from "./core/changes"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/tests/built-changes.mjs b/packages/contrail/tests/built-changes.mjs new file mode 100644 index 0000000..1c3cf76 --- /dev/null +++ b/packages/contrail/tests/built-changes.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; + +const root = mkdtempSync(join(tmpdir(), "contrail-changes-")); +try { + writeFileSync( + join(root, "contrail.config.js"), + `export const config = { + namespace: "com.example", + profiles: [], + collections: { event: { collection: "com.example.event" } }, + changes: { consumers: { + webhook: { collections: ["com.example.event"], initial: "future" } + } } + };\n`, + ); + const cli = resolve("dist/cli.js"); + const database = join(root, "contrail.sqlite"); + const status = spawnSync( + process.execPath, + [cli, "changes", "status", "--root", root, "--sqlite", database, "--json"], + { cwd: resolve("."), encoding: "utf8" }, + ); + assert.equal(status.status, 0, status.stderr); + const parsed = JSON.parse(status.stdout); + assert.equal(parsed.enabled, true); + assert.equal(parsed.consumers[0].id, "webhook"); + + const retry = spawnSync( + process.execPath, + [cli, "changes", "retry", "webhook", "--root", root, "--sqlite", database], + { cwd: resolve("."), encoding: "utf8" }, + ); + assert.equal(retry.status, 0, retry.stderr); + assert.match(retry.stdout, /retry is now due/); + console.log("built change consumer CLI passed"); +} finally { + rmSync(root, { recursive: true, force: true }); +} diff --git a/packages/contrail/tests/built-sqlite.mjs b/packages/contrail/tests/built-sqlite.mjs index 4c11bb2..be2dbc9 100644 --- a/packages/contrail/tests/built-sqlite.mjs +++ b/packages/contrail/tests/built-sqlite.mjs @@ -12,6 +12,8 @@ assert.equal(row?.ok, 1); const contrail = new Contrail({ namespace: "smoke", collections: {}, db }); await contrail.init(); +assert.equal(typeof contrail.changes.claim, "function"); +assert.equal((await contrail.changes.status()).enabled, false); const statusResponse = await contrail .app() .fetch(new Request("http://localhost/status")); diff --git a/packages/contrail/tests/change-consumers.test.ts b/packages/contrail/tests/change-consumers.test.ts new file mode 100644 index 0000000..71b6184 --- /dev/null +++ b/packages/contrail/tests/change-consumers.test.ts @@ -0,0 +1,351 @@ +import { describe, expect, it } from "vitest"; +import { + ChangeClaimTooLargeError, + ChangeLeaseLostError, + acknowledgeChanges, + claimChanges, + createIngestEvent, + failChanges, + getChangesStatus, + hydrateChanges, + ingestRecords, + initSchema, + registerChangeConsumer, + renewChangeClaim, + resolveConfig, + retryChangeConsumer, + type ChangeClaim, + type ContrailConfig, + type Database, + type ProjectionPhase, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const EVENT = "com.example.event"; +const NOTE = "com.example.note"; +const logger = { log() {}, warn() {}, error() {} }; + +function consumerConfig( + consumers: NonNullable["consumers"], +) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { + event: { collection: EVENT }, + note: { collection: NOTE }, + }, + changes: { consumers }, + }); +} + +function mutation(options: { + collection?: string; + rkey: string; + sourceTime: number; + cid?: string; + name?: string; + operation?: "create" | "update" | "delete"; +}) { + const collection = options.collection ?? EVENT; + const operation = options.operation ?? "update"; + const did = "did:plc:alice"; + return createIngestEvent({ + uri: `at://${did}/${collection}/${options.rkey}`, + did, + collection, + rkey: options.rkey, + operation, + cid: operation === "delete" ? null : (options.cid ?? `cid-${options.sourceTime}`), + value: + operation === "delete" + ? undefined + : { name: options.name ?? `${options.rkey}-${options.sourceTime}` }, + timeUs: options.sourceTime, + indexedAt: options.sourceTime + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: options.sourceTime, + revision: String(options.sourceTime), + cursor: String(options.sourceTime), + }, + }); +} + +async function apply( + db: Database, + config: ReturnType, + events: ReturnType[], + phase: ProjectionPhase = "live", +) { + await ingestRecords(db, events, config, { phase }); +} + +function readyEventConsumer(id = "search") { + return consumerConfig({ + [id]: { + collections: [EVENT], + initial: "history", + }, + }); +} + +describe("durable change consumers", () => { + it("verifies static registration and keeps current consumers pending", async () => { + const db = createSqliteDatabase(":memory:"); + const config = consumerConfig({ + search: { collections: [EVENT], initial: "current" }, + webhook: { collections: [EVENT], phases: ["live"], initial: "future" }, + }); + await initSchema(db, config); + + await expect(registerChangeConsumer(db, config, "search")).resolves.toBeUndefined(); + await expect(registerChangeConsumer(db, config, "missing")).rejects.toThrow( + "not configured", + ); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + expect(await claimChanges(db, "search", { now: 100 })).toBeNull(); + expect(await claimChanges(db, "webhook", { now: 100 })).not.toBeNull(); + }); + + it("claims a bounded range, coalesces URIs, hydrates current state, and acks with CAS", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply( + db, + config, + [mutation({ rkey: "one", sourceTime: 1, cid: "cid-one", name: "one" })], + "historical", + ); + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 2, cid: "cid-two", name: "two" }), + ]); + + const claim = await claimChanges(db, "search", { now: 1_000 }); + expect(claim).toMatchObject({ + consumerId: "search", + from: "0", + through: "2", + attempt: 1, + }); + expect(claim!.changes).toHaveLength(1); + expect(claim!.changes[0]).toMatchObject({ + id: `${claim!.generation}:2:0`, + uri: `at://did:plc:alice/${EVENT}/one`, + cid: "cid-two", + }); + + const delivery = await hydrateChanges(db, config, claim!); + expect(delivery.cursor).toEqual({ + generation: claim!.generation, + from: "0", + through: "2", + }); + expect(delivery.currentRecords).toHaveLength(1); + expect(delivery.currentRecords[0]).toMatchObject({ + cid: "cid-two", + value: { name: "two" }, + }); + expect(delivery.absentUris).toEqual([]); + + await acknowledgeChanges(db, claim!, { now: 1_001 }); + await expect( + acknowledgeChanges(db, claim!, { now: 1_002 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + expect(await claimChanges(db, "search", { now: 1_003 })).toBeNull(); + + const status = await getChangesStatus(db); + expect(status).toMatchObject({ enabled: true, rows: 2, changes: 2 }); + expect(status.consumers[0]).toMatchObject({ + id: "search", + position: "2", + backlogBatches: 0, + attempts: 0, + leased: false, + }); + + // Unrelated schema migrations must preserve mutable consumer progress. + await db + .prepare( + "UPDATE _contrail_meta SET value = 'stale' WHERE key = 'schema_fingerprint'", + ) + .run(); + await initSchema(db, config); + expect((await getChangesStatus(db)).consumers[0].position).toBe("2"); + }); + + it("hydrates newest state when delete/recreate races a claimed change", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + const claim = await claimChanges(db, "search", { now: 100 }); + + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 2, operation: "delete" }), + ]); + let delivery = await hydrateChanges(db, config, claim!); + expect(delivery.currentRecords).toEqual([]); + expect(delivery.absentUris).toEqual([ + `at://did:plc:alice/${EVENT}/one`, + ]); + + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 3, cid: "cid-three", name: "three" }), + ]); + delivery = await hydrateChanges(db, config, claim!); + expect(delivery.absentUris).toEqual([]); + expect(delivery.currentRecords[0]).toMatchObject({ + cid: "cid-three", + value: { name: "three" }, + }); + }); + + it("leases independently, persists failure backoff, and manually retries", async () => { + const db = createSqliteDatabase(":memory:"); + const config = consumerConfig({ + search: { collections: [EVENT], initial: "history" }, + analytics: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + + const search = await claimChanges(db, "search", { now: 1_000 }); + const analytics = await claimChanges(db, "analytics", { now: 1_000 }); + expect(search?.leaseOwner).not.toBe(analytics?.leaseOwner); + + expect( + await failChanges( + db, + search!, + { code: "destination_unavailable", nextAttemptAt: 2_000 }, + { now: 1_001 }, + ), + ).toEqual({ attempts: 1, nextAttemptAt: 2_000 }); + await acknowledgeChanges(db, analytics!, { now: 1_001 }); + + expect(await claimChanges(db, "search", { now: 1_500 })).toBeNull(); + await retryChangeConsumer(db, "search", { now: 1_500 }); + const retry = await claimChanges(db, "search", { now: 1_500 }); + expect(retry?.attempt).toBe(2); + await acknowledgeChanges(db, retry!, { now: 1_501 }); + + const status = await getChangesStatus(db); + expect(status.consumers.find((item) => item.id === "analytics")).toMatchObject({ + position: "1", + attempts: 0, + }); + expect(status.consumers.find((item) => item.id === "search")).toMatchObject({ + position: "1", + attempts: 0, + nextAttemptAt: null, + lastErrorCode: null, + }); + }); + + it("allows only one concurrent lease and rejects an expired owner", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + + const claims = await Promise.all([ + claimChanges(db, "search", { now: 100, leaseMs: 50 }), + claimChanges(db, "search", { now: 100, leaseMs: 50 }), + ]); + expect(claims.filter(Boolean)).toHaveLength(1); + const stale = claims.find(Boolean)!; + const replacement = await claimChanges(db, "search", { + now: 151, + leaseMs: 50, + }); + expect(replacement?.leaseOwner).not.toBe(stale.leaseOwner); + expect(replacement?.changes.map((change) => change.id)).toEqual( + stale.changes.map((change) => change.id), + ); + await expect( + acknowledgeChanges(db, stale, { now: 152 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await acknowledgeChanges(db, replacement!, { now: 152 }); + }); + + it("renews a live lease without changing its CAS owner", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + const claim = await claimChanges(db, "search", { + now: 100, + leaseMs: 50, + }); + const renewed = await renewChangeClaim(db, claim!, { + now: 120, + leaseMs: 100, + }); + expect(renewed.leaseOwner).toBe(claim!.leaseOwner); + expect(renewed.leaseExpiresAt).toBe(220); + await acknowledgeChanges(db, renewed, { now: 200 }); + }); + + it("auto-advances irrelevant ranges while preserving independent progress", async () => { + const db = createSqliteDatabase(":memory:"); + const config = consumerConfig({ + events: { collections: [EVENT], phases: ["live"], initial: "future" }, + notes: { collections: [NOTE], phases: ["live"], initial: "future" }, + }); + await initSchema(db, config); + await apply(db, config, [ + mutation({ collection: NOTE, rkey: "note", sourceTime: 1 }), + ]); + await apply(db, config, [mutation({ rkey: "event", sourceTime: 2 })]); + + const eventClaim = await claimChanges(db, "events", { + now: 100, + maxBatches: 1, + }); + expect(eventClaim).toMatchObject({ from: "1", through: "2" }); + expect(eventClaim!.changes[0].collection).toBe(EVENT); + await acknowledgeChanges(db, eventClaim!, { now: 101 }); + + const noteClaim = await claimChanges(db, "notes", { now: 100 }); + expect(noteClaim).toMatchObject({ from: "0", through: "2" }); + expect(noteClaim!.changes[0].collection).toBe(NOTE); + await acknowledgeChanges(db, noteClaim!, { now: 101 }); + }); + + it("releases a lease when the first durable batch exceeds claim limits", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 1 }), + mutation({ rkey: "two", sourceTime: 2 }), + ]); + + await expect( + claimChanges(db, "search", { now: 100, maxChanges: 1 }), + ).rejects.toBeInstanceOf(ChangeClaimTooLargeError); + const claim = await claimChanges(db, "search", { + now: 101, + maxChanges: 2, + }); + expect(claim?.changes).toHaveLength(2); + }); + + it("rejects forged generation cursors and never regresses a checkpoint", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + const claim = await claimChanges(db, "search", { now: 100 }); + const forged: ChangeClaim = { ...claim!, generation: crypto.randomUUID() }; + await expect( + acknowledgeChanges(db, forged, { now: 101 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await acknowledgeChanges(db, claim!, { now: 101 }); + expect((await getChangesStatus(db)).consumers[0].position).toBe("1"); + }); +}); diff --git a/packages/contrail/tests/postgres-e2e.test.ts b/packages/contrail/tests/postgres-e2e.test.ts index 0e5d499..2e199b8 100644 --- a/packages/contrail/tests/postgres-e2e.test.ts +++ b/packages/contrail/tests/postgres-e2e.test.ts @@ -12,6 +12,10 @@ import pg from "pg"; import { createPostgresDatabase } from "../src/adapters/postgres"; import { initSchema } from "../src/index"; import { + acknowledgeChanges, + claimChanges, + getChangesStatus, + hydrateChanges, ingestRecords, queryRecords, getLastCursor, @@ -55,6 +59,26 @@ const TEST_CONFIG = resolveConfig({ }, }); +const CHANGE_CONFIG = resolveConfig({ + namespace: "com.example", + profiles: [], + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + analytics: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, +}); + const PG_URL = process.env.TEST_DATABASE_URL; if (!PG_URL) { describe.skip("PostgreSQL e2e (TEST_DATABASE_URL not set)", () => { @@ -559,6 +583,44 @@ if (!PG_URL) { }); }); + describe("durable change consumers", () => { + it("claims, hydrates, and acknowledges independently", async () => { + await initSchema(db, CHANGE_CONFIG); + await ingestRecords( + db, + [ + makeEvent({ + uri: "at://did:plc:test/community.lexicon.calendar.event/change", + rkey: "change", + cid: "cid-change", + record: { name: "Change", mode: "online" }, + time_us: 100, + indexed_at: 100, + }), + ], + CHANGE_CONFIG, + ); + + const search = await claimChanges(db, "search", { now: 1_000 }); + const analytics = await claimChanges(db, "analytics", { now: 1_000 }); + expect(search?.through).toBe("1"); + expect(analytics?.through).toBe("1"); + const delivery = await hydrateChanges(db, CHANGE_CONFIG, search!); + expect(delivery.currentRecords[0]).toMatchObject({ + cid: "cid-change", + value: { name: "Change", mode: "online" }, + }); + await acknowledgeChanges(db, search!, { now: 1_001 }); + expect((await getChangesStatus(db)).consumers).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "search", position: "1" }), + expect.objectContaining({ id: "analytics", position: "0" }), + ]), + ); + await acknowledgeChanges(db, analytics!, { now: 1_001 }); + }); + }); + // --- Cursor --- describe("cursor persistence", () => { -- 2.51.2 From 1d27c86f3eb0bd5e6a39ce3ece471bd33ee5ba9e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:21:03 +0200 Subject: [PATCH 04/34] Add current-state consumer bootstrap and retention --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 4 +- packages/contrail/src/cli/commands/changes.ts | 61 +- .../contrail/src/core/change-bootstrap.ts | 663 ++++++++++++++++++ packages/contrail/src/core/change-log.ts | 145 +++- packages/contrail/src/core/changes.ts | 488 ++++++++++++- packages/contrail/src/core/db/schema.ts | 2 +- packages/contrail/src/index.ts | 1 + packages/contrail/tests/built-changes.mjs | 8 + .../contrail/tests/change-bootstrap.test.ts | 354 ++++++++++ packages/contrail/tests/change-log.test.ts | 2 +- .../tests/postgres-concurrent-init.test.ts | 2 +- packages/contrail/tests/postgres-e2e.test.ts | 39 +- packages/contrail/tests/postgres.test.ts | 2 +- 14 files changed, 1730 insertions(+), 43 deletions(-) create mode 100644 packages/contrail/src/core/change-bootstrap.ts create mode 100644 packages/contrail/tests/change-bootstrap.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index b76f0f1..27dd926 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs plus `contrail changes status/retry` commands. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 33d0b54..09f1052 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -121,7 +121,9 @@ if (claim) { Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and resolve delete/recreate races from newest canonical state. Consumers lease and progress independently; irrelevant position ranges advance without invoking a handler. Delivery is intentionally at least once—a destination success followed by an acknowledgement crash causes duplicate delivery. Handlers must be idempotent by stable record/document key. -`initial: "future"` and `initial: "history"` consumers can use this API now. `initial: "current"` remains pending until the snapshot-plus-tail bootstrap coordinator lands in the next milestone. `contrail changes status` and `contrail changes retry ` expose private status and manual retry for SQLite or Wrangler D1 deployments. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. +`initial: "current"` uses a durable snapshot-plus-tail coordinator. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. + +A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. ## Local development diff --git a/packages/contrail/src/cli/commands/changes.ts b/packages/contrail/src/cli/commands/changes.ts index d035b55..1dfd205 100644 --- a/packages/contrail/src/cli/commands/changes.ts +++ b/packages/contrail/src/cli/commands/changes.ts @@ -13,6 +13,11 @@ interface ChangeCommandOptions { binding: string; sqlite?: string; json?: boolean; + through?: string; + reason?: string; + yes?: boolean; + maxBatches?: number; + olderThan?: number; } async function withChangesDatabase( @@ -68,24 +73,32 @@ export function registerChanges(cli: CAC): void { options( cli.command( "changes [consumer]", - "Private change-log operations: status, retry ", + "Private change-log operations: status, retry, prune, skip", ), ) .option("--json", "Print machine-readable status JSON") + .option("--through ", "Required position for changes skip") + .option("--reason ", "Required operator reason for changes skip") + .option("--yes", "Confirm the data loss caused by changes skip") + .option("--max-batches ", "Maximum rows for one prune slice", { + default: 500, + }) + .option("--older-than ", "Prune only rows older than milliseconds") .action( async ( action: string, consumer: string | undefined, commandOptions: ChangeCommandOptions, ) => { - if (action !== "status" && action !== "retry") { - throw new Error("changes action must be 'status' or 'retry'"); + const actions = ["status", "retry", "prune", "skip"]; + if (!actions.includes(action)) { + throw new Error(`changes action must be one of: ${actions.join(", ")}`); } - if (action === "retry" && !consumer) { - throw new Error("changes retry requires a consumer ID"); + if ((action === "retry" || action === "skip") && !consumer) { + throw new Error(`changes ${action} requires a consumer ID`); } - if (action === "status" && consumer) { - throw new Error("changes status does not accept a consumer ID"); + if ((action === "status" || action === "prune") && consumer) { + throw new Error(`changes ${action} does not accept a consumer ID`); } await withChangesDatabase(commandOptions, async (contrail, db) => { @@ -94,6 +107,40 @@ export function registerChanges(cli: CAC): void { console.log(`change consumer ${consumer}: retry is now due`); return; } + if (action === "skip") { + if (!commandOptions.through || !commandOptions.reason) { + throw new Error("changes skip requires --through and --reason"); + } + await contrail.changes.skip( + consumer!, + { + through: commandOptions.through, + reason: commandOptions.reason, + confirm: commandOptions.yes === true, + }, + db, + ); + console.log( + `change consumer ${consumer}: explicitly skipped through ${commandOptions.through}`, + ); + return; + } + if (action === "prune") { + const result = await contrail.changes.prune( + { + maxBatches: Number(commandOptions.maxBatches), + ...(commandOptions.olderThan === undefined + ? {} + : { olderThan: Number(commandOptions.olderThan) }), + }, + db, + ); + console.log( + `change log: pruned=${result.pruned} floor=${result.retainedFloor} ` + + `safeThrough=${result.safeThrough} done=${result.done}`, + ); + return; + } const status = await contrail.changes.status(db); if (commandOptions.json) { diff --git a/packages/contrail/src/core/change-bootstrap.ts b/packages/contrail/src/core/change-bootstrap.ts new file mode 100644 index 0000000..0cc6a97 --- /dev/null +++ b/packages/contrail/src/core/change-bootstrap.ts @@ -0,0 +1,663 @@ +import type { ContrailConfig, Database } from "./types"; +import { recordsTableName, resolveCollectionKey } from "./types"; +import type { ChangeFailure, CurrentRecord } from "./changes"; +import { + ChangeConsumerNotFoundError, + ChangeGenerationMismatchError, + ChangeLeaseLostError, +} from "./changes"; +import { getChangeLogState } from "./change-log"; + +const DEFAULT_PAGE_SIZE = 100; +const MAX_PAGE_SIZE = 500; +const DEFAULT_LEASE_MS = 30_000; +const MAX_LEASE_MS = 10 * 60_000; +const FAILURE_CODE = /^[a-zA-Z0-9_.:-]{1,64}$/; + +export interface CurrentBootstrapClaimOptions { + pageSize?: number; + leaseMs?: number; + /** @internal Deterministic clock seam for conformance tests. */ + now?: number; +} + +export interface CurrentSnapshotClaim { + kind: "snapshot"; + consumerId: string; + generation: string; + bootstrapToken: string; + collection: string; + fromUri: string | null; + throughUri: string; + pageId: string; + records: CurrentRecord[]; + attempt: number; + leaseExpiresAt: number; + readonly leaseOwner: string; +} + +export interface CurrentActivationClaim { + kind: "activation"; + consumerId: string; + generation: string; + bootstrapToken: string; + target: string; + attempt: number; + leaseExpiresAt: number; + readonly leaseOwner: string; +} + +export interface CurrentBootstrapStatus { + consumerId: string; + generation: string; + state: string; + anchor: string | null; + scanCollection: string | null; + scanCursor: string | null; + target: string | null; + token: string | null; + position: string; +} + +interface BootstrapRow { + consumer_id: string; + generation_id: string; + acknowledged_position: number | string; + configured_collections_json: string; + initial_mode: string; + bootstrap_state: string; + bootstrap_anchor_position: number | string | null; + bootstrap_scan_collection: string | null; + bootstrap_scan_cursor: string | null; + bootstrap_target_position: number | string | null; + bootstrap_token: string | null; + lease_owner: string | null; + lease_expires_at: number | string | null; + attempts: number | string; +} + +function integer( + value: number | undefined, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new TypeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return result; +} + +function now(value: number | undefined): number { + const result = value ?? Date.now(); + if (!Number.isSafeInteger(result) || result < 0) { + throw new TypeError("now must be a non-negative safe integer"); + } + return result; +} + +function parseCollections(value: string, consumerId: string): string[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error(`Current consumer ${consumerId} collections are not valid JSON`); + } + if ( + !Array.isArray(parsed) || + parsed.length === 0 || + !parsed.every((item) => typeof item === "string") + ) { + throw new Error(`Current consumer ${consumerId} collections are malformed`); + } + return [...parsed].sort(); +} + +async function bootstrapRow( + db: Database, + consumerId: string, +): Promise { + return db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, initial_mode, bootstrap_state, + bootstrap_anchor_position, bootstrap_scan_collection, + bootstrap_scan_cursor, bootstrap_target_position, + bootstrap_token, lease_owner, lease_expires_at, attempts + FROM change_consumers WHERE consumer_id = ?`, + ) + .bind(consumerId) + .first(); +} + +async function release( + db: Database, + consumerId: string, + generation: string, + owner: string, + timestamp: number, +): Promise { + await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? AND lease_owner = ?`, + ) + .bind(timestamp, consumerId, generation, owner) + .run(); +} + +function currentRecord( + row: { + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }, + collection: string, +): CurrentRecord { + let value: unknown; + try { + value = JSON.parse(row.record); + } catch { + throw new Error(`Current record ${row.uri} is not valid JSON`); + } + return { + uri: row.uri, + did: row.did, + collection, + rkey: row.rkey, + cid: row.cid, + value, + timeUs: Number(row.time_us), + indexedAt: Number(row.indexed_at), + }; +} + +/** Claim one stable URI-keyset page. Empty collections advance internally; + * exhausting the final collection atomically pins the catch-up target. */ +export async function claimCurrentSnapshotPage( + db: Database, + config: ContrailConfig, + consumerId: string, + options: CurrentBootstrapClaimOptions = {}, +): Promise { + const pageSize = integer( + options.pageSize, + DEFAULT_PAGE_SIZE, + 1, + MAX_PAGE_SIZE, + "pageSize", + ); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const timestamp = now(options.now); + + const configured = config.changes?.consumers[consumerId]; + if (!configured || configured.initial !== "current") { + throw new ChangeConsumerNotFoundError( + `Current-state change consumer ${consumerId} is not configured`, + ); + } + const expectedCollections = [...configured.collections].sort(); + + for (let step = 0; step <= expectedCollections.length; step++) { + const owner = crypto.randomUUID(); + const expires = timestamp + leaseMs; + const leased = await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'scanning', + bootstrap_scan_collection = CASE + WHEN bootstrap_state = 'pending' THEN ? + ELSE bootstrap_scan_collection END, + bootstrap_scan_cursor = CASE + WHEN bootstrap_state = 'pending' THEN NULL + ELSE bootstrap_scan_cursor END, + lease_owner = ?, lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND initial_mode = 'current' + AND bootstrap_state IN ('pending', 'scanning') + AND generation_id = ( + SELECT generation_id FROM change_log_state WHERE id = 1 + ) + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id, generation_id, acknowledged_position, + configured_collections_json, initial_mode, bootstrap_state, + bootstrap_anchor_position, bootstrap_scan_collection, + bootstrap_scan_cursor, bootstrap_target_position, + bootstrap_token, lease_owner, lease_expires_at, attempts`, + ) + .bind( + expectedCollections[0], + owner, + expires, + timestamp, + consumerId, + timestamp, + timestamp, + ) + .first(); + if (!leased) { + const durable = await bootstrapRow(db, consumerId); + if (!durable) { + throw new ChangeConsumerNotFoundError( + `Current-state change consumer ${consumerId} is not initialized`, + ); + } + const state = await getChangeLogState(db); + if (state && durable.generation_id !== state.generation) { + throw new ChangeGenerationMismatchError( + `Current-state consumer ${consumerId} belongs to another generation`, + ); + } + return null; + } + if (!leased.bootstrap_token || !leased.bootstrap_scan_collection) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state change consumer ${consumerId} is malformed`); + } + const durableCollections = parseCollections( + leased.configured_collections_json, + consumerId, + ); + if (JSON.stringify(durableCollections) !== JSON.stringify(expectedCollections)) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state change consumer ${consumerId} changed collections`); + } + const collectionIndex = expectedCollections.indexOf( + leased.bootstrap_scan_collection, + ); + if (collectionIndex < 0) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state change consumer ${consumerId} has an invalid scan collection`); + } + const short = resolveCollectionKey(config, leased.bootstrap_scan_collection); + if (!short) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state scan collection is not configured`); + } + const table = recordsTableName(short); + const result = leased.bootstrap_scan_cursor === null + ? await db + .prepare( + `SELECT uri, did, rkey, cid, record, time_us, indexed_at + FROM ${table} ORDER BY uri LIMIT ?`, + ) + .bind(pageSize) + .all<{ + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }>() + : await db + .prepare( + `SELECT uri, did, rkey, cid, record, time_us, indexed_at + FROM ${table} WHERE uri > ? ORDER BY uri LIMIT ?`, + ) + .bind(leased.bootstrap_scan_cursor, pageSize) + .all<{ + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }>(); + + if (result.results.length > 0) { + const records = result.results.map((row) => + currentRecord(row, leased.bootstrap_scan_collection!), + ); + const throughUri = records.at(-1)!.uri; + return { + kind: "snapshot", + consumerId, + generation: leased.generation_id, + bootstrapToken: leased.bootstrap_token, + collection: leased.bootstrap_scan_collection, + fromUri: leased.bootstrap_scan_cursor, + throughUri, + pageId: + `${leased.bootstrap_token}:snapshot:` + + `${leased.bootstrap_scan_collection}:${throughUri}`, + records, + attempt: Number(leased.attempts) + 1, + leaseExpiresAt: expires, + leaseOwner: owner, + }; + } + + const nextCollection = expectedCollections[collectionIndex + 1] ?? null; + if (nextCollection !== null) { + await db + .prepare( + `UPDATE change_consumers + SET bootstrap_scan_collection = ?, bootstrap_scan_cursor = NULL, + lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'scanning' AND lease_owner = ?`, + ) + .bind( + nextCollection, + timestamp, + consumerId, + leased.generation_id, + owner, + ) + .run(); + continue; + } + + await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'catching-up', + bootstrap_target_position = ( + SELECT head_position FROM change_log_state WHERE id = 1 + ), + bootstrap_scan_collection = NULL, bootstrap_scan_cursor = NULL, + lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'scanning' AND lease_owner = ?`, + ) + .bind(timestamp, consumerId, leased.generation_id, owner) + .run(); + return null; + } + return null; +} + +export async function acknowledgeCurrentSnapshotPage( + db: Database, + claim: CurrentSnapshotClaim, + options: { now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const acknowledged = await db + .prepare( + `UPDATE change_consumers + SET bootstrap_scan_cursor = ?, lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_success_at = ?, last_error_code = NULL, last_error_at = NULL, + updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'scanning' AND bootstrap_token = ? + AND bootstrap_scan_collection = ? + AND ((bootstrap_scan_cursor = ?) OR + (bootstrap_scan_cursor IS NULL AND CAST(? AS TEXT) IS NULL)) + AND lease_owner = ? AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + claim.throughUri, + timestamp, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.collection, + claim.fromUri, + claim.fromUri, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!acknowledged) { + throw new ChangeLeaseLostError( + `Current snapshot claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +async function failBootstrapLease( + db: Database, + claim: CurrentSnapshotClaim | CurrentActivationClaim, + failure: ChangeFailure, + timestamp: number, +): Promise { + if (!FAILURE_CODE.test(failure.code)) { + throw new TypeError( + "Change failure code must contain 1-64 safe identifier characters", + ); + } + if ( + failure.nextAttemptAt !== null && + (!Number.isSafeInteger(failure.nextAttemptAt) || + failure.nextAttemptAt < timestamp) + ) { + throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); + } + const failed = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, + attempts = attempts + 1, next_attempt_at = ?, + last_error_code = ?, last_error_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_token = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + failure.nextAttemptAt, + failure.code, + timestamp, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!failed) { + throw new ChangeLeaseLostError( + `Current bootstrap claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +export function failCurrentSnapshotPage( + db: Database, + claim: CurrentSnapshotClaim, + failure: ChangeFailure, + options: { now?: number } = {}, +): Promise { + return failBootstrapLease(db, claim, failure, now(options.now)); +} + +export async function renewCurrentBootstrapClaim< + T extends CurrentSnapshotClaim | CurrentActivationClaim, +>( + db: Database, + claim: T, + options: { leaseMs?: number; now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const expires = timestamp + leaseMs; + const renewed = await db + .prepare( + `UPDATE change_consumers + SET lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_token = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + expires, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!renewed) { + throw new ChangeLeaseLostError( + `Current bootstrap claim for ${claim.consumerId} cannot be renewed`, + ); + } + return { ...claim, leaseExpiresAt: expires }; +} + +/** Claim the idempotent destination-activation step after fixed-target catch-up. */ +export async function claimCurrentActivation( + db: Database, + consumerId: string, + options: { leaseMs?: number; now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const owner = crypto.randomUUID(); + const expires = timestamp + leaseMs; + const row = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND initial_mode = 'current' + AND bootstrap_state = 'activating' + AND generation_id = ( + SELECT generation_id FROM change_log_state WHERE id = 1 + ) + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id, generation_id, acknowledged_position, + configured_collections_json, initial_mode, bootstrap_state, + bootstrap_anchor_position, bootstrap_scan_collection, + bootstrap_scan_cursor, bootstrap_target_position, + bootstrap_token, lease_owner, lease_expires_at, attempts`, + ) + .bind(owner, expires, timestamp, consumerId, timestamp, timestamp) + .first(); + if (!row) { + const durable = await bootstrapRow(db, consumerId); + const state = await getChangeLogState(db); + if (durable && state && durable.generation_id !== state.generation) { + throw new ChangeGenerationMismatchError( + `Current-state consumer ${consumerId} belongs to another generation`, + ); + } + return null; + } + if (!row.bootstrap_token || row.bootstrap_target_position === null) { + await release(db, consumerId, row.generation_id, owner, timestamp); + throw new Error(`Current activation ${consumerId} is malformed`); + } + return { + kind: "activation", + consumerId, + generation: row.generation_id, + bootstrapToken: row.bootstrap_token, + target: String(row.bootstrap_target_position), + attempt: Number(row.attempts) + 1, + leaseExpiresAt: expires, + leaseOwner: owner, + }; +} + +export async function completeCurrentActivation( + db: Database, + claim: CurrentActivationClaim, + options: { now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const completed = await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'ready', lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_success_at = ?, last_error_code = NULL, last_error_at = NULL, + updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'activating' AND bootstrap_token = ? + AND bootstrap_target_position = ? + AND acknowledged_position = bootstrap_target_position + AND lease_owner = ? AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + timestamp, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.target, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!completed) { + throw new ChangeLeaseLostError( + `Current activation claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +export function failCurrentActivation( + db: Database, + claim: CurrentActivationClaim, + failure: ChangeFailure, + options: { now?: number } = {}, +): Promise { + return failBootstrapLease(db, claim, failure, now(options.now)); +} + +export async function getCurrentBootstrapStatus( + db: Database, + consumerId: string, +): Promise { + const row = await bootstrapRow(db, consumerId); + if (!row || row.initial_mode !== "current") { + throw new ChangeConsumerNotFoundError( + `Current-state change consumer ${consumerId} is not initialized`, + ); + } + return { + consumerId, + generation: row.generation_id, + state: row.bootstrap_state, + anchor: + row.bootstrap_anchor_position === null + ? null + : String(row.bootstrap_anchor_position), + scanCollection: row.bootstrap_scan_collection, + scanCursor: row.bootstrap_scan_cursor, + target: + row.bootstrap_target_position === null + ? null + : String(row.bootstrap_target_position), + token: row.bootstrap_token, + position: String(row.acknowledged_position), + }; +} diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 5973925..b50a551 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -139,6 +139,17 @@ export function buildChangeLogSchema( through_position ${bigint}, PRIMARY KEY (generation_id, collection, phase, from_position) )`, + `CREATE TABLE IF NOT EXISTS change_consumer_actions ( + action_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + consumer_id TEXT NOT NULL, + action TEXT NOT NULL CHECK (action IN ('skip', 'remove')), + from_position ${bigint} NOT NULL, + through_position ${bigint} NOT NULL, + reason TEXT NOT NULL, + created_at ${bigint} NOT NULL + )`, + "CREATE INDEX IF NOT EXISTS idx_change_consumer_actions_consumer ON change_consumer_actions(generation_id, consumer_id, created_at)", ]; } @@ -201,8 +212,88 @@ function canonicalPhases(phases: ProjectionPhase[]): string { return JSON.stringify([...phases].sort()); } -/** Initialize one immutable milestone-1 logging definition. Later milestones - * add explicit quiet-boundary operations for changing this durable definition. */ +function sameStaticConsumer( + actual: ChangeConsumerRow, + expected: NonNullable["consumers"][string], +): boolean { + return ( + actual.configured_collections_json === + canonicalCollections(expected.collections) && + actual.configured_phases_json === + canonicalPhases(changeConsumerPhases(expected)) && + actual.initial_mode === expected.initial && + Number(actual.required_for_activation) === + (expected.requiredForActivation === true ? 1 : 0) + ); +} + +/** Reconcile only additive consumers whose collection/phase coverage is already + * durable. Expanding coverage still requires a fresh generation or an explicit + * old-writer quiet boundary. */ +async function reconcileAdditiveDefinitions( + db: Database, + config: ContrailConfig, + state: ChangeLogStateRow, + definitions: string, +): Promise { + const durableConsumers = await db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + bootstrap_anchor_position + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + const configured = config.changes?.consumers ?? {}; + for (const durable of durableConsumers.results) { + const expected = configured[durable.consumer_id]; + if (!expected || !sameStaticConsumer(durable, expected)) { + throw new Error( + "Durable change consumers cannot be removed or modified during ordinary initialization", + ); + } + } + + const coverage = await db + .prepare( + `SELECT generation_id, collection, phase, from_position, through_position + FROM change_log_coverage + WHERE generation_id = ? AND through_position IS NULL + ORDER BY collection, phase, from_position`, + ) + .bind(state.generation_id) + .all(); + const durablePairs = new Set( + coverage.results.map((item) => `${item.collection}\0${item.phase}`), + ); + const expectedPairs = changeLogCoverage(config); + if ( + durablePairs.size !== expectedPairs.length || + expectedPairs.some( + (item) => !durablePairs.has(`${item.collection}\0${item.phase}`), + ) + ) { + throw new Error( + "Adding this change consumer expands collection/phase coverage; use a fresh generation or an explicit quiet-boundary migration", + ); + } + + await db + .prepare( + `UPDATE change_log_state SET definitions_json = ? + WHERE id = 1 AND generation_id = ? AND definitions_json = ?`, + ) + .bind(definitions, state.generation_id, state.definitions_json) + .run(); + const updated = (await probeChangeLogSchema(db)).state; + if (!updated || updated.definitions_json !== definitions) { + throw new Error("Change consumer definitions changed concurrently"); + } +} + +/** Initialize the fresh log or safely add consumers over already-logged + * collection/phase coverage. */ export async function initializeChangeLog( db: Database, config: ContrailConfig, @@ -211,19 +302,26 @@ export async function initializeChangeLog( const definitions = canonicalChangeDefinitions(config); const now = Date.now(); - const candidateGeneration = crypto.randomUUID(); - const statements: Statement[] = [ - db - .prepare( - `INSERT INTO change_log_state - (id, generation_id, head_position, retained_floor_position, - definitions_json, created_at) - VALUES (1, ?, 0, 0, ?, ?) - ON CONFLICT(id) DO NOTHING`, - ) - .bind(candidateGeneration, definitions, now), - ]; + await db + .prepare( + `INSERT INTO change_log_state + (id, generation_id, head_position, retained_floor_position, + definitions_json, created_at) + VALUES (1, ?, 0, 0, ?, ?) + ON CONFLICT(id) DO NOTHING`, + ) + .bind(crypto.randomUUID(), definitions, now) + .run(); + + let state = (await probeChangeLogSchema(db)).state; + if (!state) throw new Error("Could not initialize change-log state"); + if (state.definitions_json !== definitions) { + await reconcileAdditiveDefinitions(db, config, state, definitions); + state = (await probeChangeLogSchema(db)).state; + if (!state) throw new Error("Could not reload change-log state"); + } + const statements: Statement[] = []; for (const [consumerId, consumer] of Object.entries( config.changes?.consumers ?? {}, ).sort(([left], [right]) => left.localeCompare(right))) { @@ -235,9 +333,13 @@ export async function initializeChangeLog( (consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, - bootstrap_anchor_position, attempts, updated_at) - SELECT ?, generation_id, 0, ?, ?, ?, ?, ?, - CASE WHEN ? = 'current' THEN 0 ELSE NULL END, + bootstrap_anchor_position, bootstrap_token, attempts, updated_at) + SELECT ?, generation_id, + CASE WHEN ? = 'history' THEN retained_floor_position + ELSE head_position END, + ?, ?, ?, ?, ?, + CASE WHEN ? = 'current' THEN head_position ELSE NULL END, + CASE WHEN ? = 'current' THEN ? ELSE NULL END, 0, ? FROM change_log_state WHERE id = 1 AND definitions_json = ? @@ -245,12 +347,15 @@ export async function initializeChangeLog( ) .bind( consumerId, + consumer.initial, canonicalCollections(consumer.collections), canonicalPhases(changeConsumerPhases(consumer)), consumer.initial, consumer.requiredForActivation === true ? 1 : 0, initialReady, consumer.initial, + consumer.initial, + crypto.randomUUID(), now, definitions, ), @@ -273,11 +378,7 @@ export async function initializeChangeLog( ); } - // Keep initialization under conservative D1 statement limits. The durable - // definitions_json winner makes these resumable chunks safe under concurrent - // initialization; init does not return until the complete set verifies. - await db.batch(statements.slice(0, 1)); - for (let index = 1; index < statements.length; index += 50) { + for (let index = 0; index < statements.length; index += 50) { await db.batch(statements.slice(index, index + 50)); } await assertChangeLogDefinition(db, config); diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 8482453..4875240 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -13,6 +13,20 @@ import { type ChangeLogState, type RecordChange, } from "./change-log"; +import { + acknowledgeCurrentSnapshotPage, + claimCurrentActivation, + claimCurrentSnapshotPage, + completeCurrentActivation, + failCurrentActivation, + failCurrentSnapshotPage, + getCurrentBootstrapStatus, + renewCurrentBootstrapClaim, + type CurrentActivationClaim, + type CurrentBootstrapClaimOptions, + type CurrentBootstrapStatus, + type CurrentSnapshotClaim, +} from "./change-bootstrap"; const DEFAULT_MAX_BATCHES = 20; const DEFAULT_MAX_CHANGES = 500; @@ -51,6 +65,10 @@ export interface ChangeClaim { changes: RecordChange[]; attempt: number; leaseExpiresAt: number; + /** Generation-scoped destination token during current-state catch-up. */ + readonly bootstrapToken?: string; + /** Fixed catch-up target for a current-state bootstrap claim. */ + readonly bootstrapTarget?: string; /** Opaque CAS capability. Do not pass this field to delivery handlers. */ readonly leaseOwner: string; } @@ -125,6 +143,8 @@ interface DurableConsumerRow { initial_mode: string; required_for_activation: number | string; bootstrap_state: string; + bootstrap_target_position: number | string | null; + bootstrap_token: string | null; lease_owner: string | null; lease_expires_at: number | string | null; attempts: number | string; @@ -264,6 +284,7 @@ async function durableConsumer( `SELECT consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, + bootstrap_target_position, bootstrap_token, lease_owner, lease_expires_at, attempts, next_attempt_at, last_success_at, last_error_code, last_error_at FROM change_consumers WHERE consumer_id = ?`, @@ -380,12 +401,14 @@ export async function registerChangeConsumer( /** Claim one bounded contiguous position range. Irrelevant ranges advance by * CAS without invoking application code. */ -export async function claimChanges( +async function claimChangeRange( db: Database, consumerId: string, options: ChangeClaimOptions = {}, + bootstrap = false, ): Promise { const limits = normalizedClaimOptions(options); + const requiredState = bootstrap ? "catching-up" : "ready"; if (!(await getChangeLogState(db))) { throw new ChangeConsumerNotFoundError( `Change consumer ${consumerId} is not initialized`, @@ -404,7 +427,7 @@ export async function claimChanges( `UPDATE change_consumers SET lease_owner = ?, lease_expires_at = ?, updated_at = ? WHERE consumer_id = ? - AND bootstrap_state = 'ready' + AND bootstrap_state = '${requiredState}' AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) AND acknowledged_position >= (SELECT retained_floor_position FROM change_log_state WHERE id = 1) AND (next_attempt_at IS NULL OR next_attempt_at <= ?) @@ -412,6 +435,7 @@ export async function claimChanges( RETURNING consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, + bootstrap_target_position, bootstrap_token, lease_owner, lease_expires_at, attempts, next_attempt_at, last_success_at, last_error_code, last_error_at`, ) @@ -451,23 +475,57 @@ export async function claimChanges( const generation = leased.generation_id; const from = String(leased.acknowledged_position); + const bootstrapTarget = bootstrap + ? String(leased.bootstrap_target_position) + : null; + if (bootstrap && leased.bootstrap_target_position === null) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new Error(`Change consumer ${consumerId} has no bootstrap target`); + } const metadata = await db .prepare( `SELECT position, phase, change_count, encoded_bytes FROM change_batches WHERE generation_id = ? AND position > ? + AND (? IS NULL OR position <= ?) ORDER BY position LIMIT ?`, ) - .bind(generation, from, limits.maxBatches) + .bind( + generation, + from, + bootstrapTarget, + bootstrapTarget, + limits.maxBatches, + ) .all(); if (metadata.results.length === 0) { + if (bootstrap && from === bootstrapTarget) { + await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'activating', lease_owner = NULL, + lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'catching-up' + AND acknowledged_position = bootstrap_target_position + AND acknowledged_position = ? AND lease_owner = ?`, + ) + .bind(limits.now, consumerId, generation, from, owner) + .run(); + return null; + } const state = await getChangeLogState(db); await releaseLease(db, consumerId, generation, from, owner, limits.now); - if (state && state.generation === generation && state.head !== from) { + const expectedHead = bootstrap ? bootstrapTarget : state?.head; + if ( + expectedHead !== null && + expectedHead !== undefined && + expectedHead !== from + ) { throw new ChangeHistoryGapError( - `Change consumer ${consumerId} cannot resume from ${from}; log head is ${state.head}`, + `Change consumer ${consumerId} cannot resume from ${from}; target is ${expectedHead}`, ); } return null; @@ -571,6 +629,10 @@ export async function claimChanges( changes: [...coalesced.values()], attempt: Number(leased.attempts) + 1, leaseExpiresAt, + ...(leased.bootstrap_token === null + ? {} + : { bootstrapToken: leased.bootstrap_token }), + ...(bootstrapTarget === null ? {} : { bootstrapTarget }), leaseOwner: owner, }; if (claim.changes.length > 0) return claim; @@ -581,6 +643,23 @@ export async function claimChanges( return null; } +export function claimChanges( + db: Database, + consumerId: string, + options: ChangeClaimOptions = {}, +): Promise { + return claimChangeRange(db, consumerId, options, false); +} + +/** Claim changes only through the fixed target of a current-state bootstrap. */ +export function claimCurrentBootstrapChanges( + db: Database, + consumerId: string, + options: ChangeClaimOptions = {}, +): Promise { + return claimChangeRange(db, consumerId, options, true); +} + /** Hydrate the newest canonical state for each coalesced claimed URI. */ export async function hydrateChanges( db: Database, @@ -669,8 +748,13 @@ export async function acknowledgeChanges( const acknowledged = await db .prepare( `UPDATE change_consumers - SET acknowledged_position = ?, lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + SET acknowledged_position = ?, + bootstrap_state = CASE + WHEN bootstrap_state = 'catching-up' + AND ? = bootstrap_target_position THEN 'activating' + ELSE bootstrap_state END, + lease_owner = NULL, lease_expires_at = NULL, + attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? @@ -684,6 +768,7 @@ export async function acknowledgeChanges( RETURNING consumer_id`, ) .bind( + claim.through, claim.through, now, now, @@ -866,6 +951,7 @@ export async function getChangesStatus(db: Database): Promise { `SELECT consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, + bootstrap_target_position, bootstrap_token, lease_owner, lease_expires_at, attempts, next_attempt_at, last_success_at, last_error_code, last_error_at FROM change_consumers ORDER BY consumer_id`, @@ -917,6 +1003,280 @@ export async function getChangesStatus(db: Database): Promise { }; } +export interface RequiredChangeConsumerReadiness { + ready: boolean; + through: string; + pending: Array<{ + id: string; + state: string; + position: string; + }>; +} + +/** Check activation-gating consumers against one fixed log target. */ +export async function getRequiredChangeConsumerReadiness( + db: Database, + through?: string, +): Promise { + const state = await getChangeLogState(db); + if (!state) return { ready: true, through: "0", pending: [] }; + const target = through ?? state.head; + if (!/^\d+$/.test(target) || BigInt(target) > BigInt(state.head)) { + throw new TypeError("Required-consumer target must be a retained log position at or below head"); + } + const rows = await db + .prepare( + `SELECT consumer_id, bootstrap_state, acknowledged_position + FROM change_consumers + WHERE generation_id = ? AND required_for_activation = 1 + ORDER BY consumer_id`, + ) + .bind(state.generation) + .all<{ + consumer_id: string; + bootstrap_state: string; + acknowledged_position: number | string; + }>(); + const pending = rows.results + .filter( + (row) => + row.bootstrap_state !== "ready" || + BigInt(String(row.acknowledged_position)) < BigInt(target), + ) + .map((row) => ({ + id: row.consumer_id, + state: row.bootstrap_state, + position: String(row.acknowledged_position), + })); + return { ready: pending.length === 0, through: target, pending }; +} + +export async function assertRequiredChangeConsumersReady( + db: Database, + through?: string, +): Promise { + const readiness = await getRequiredChangeConsumerReadiness(db, through); + if (!readiness.ready) { + throw new Error( + `Required change consumers are not ready through ${readiness.through}: ` + + readiness.pending.map((item) => item.id).join(", "), + ); + } +} + +export interface SkipChangeConsumerOptions { + through: string; + reason: string; + confirm: boolean; + /** @internal Deterministic clock seam for conformance tests. */ + now?: number; +} + +/** Explicit audited data-loss operation for one ready consumer. */ +export async function skipChangeConsumer( + db: Database, + consumerId: string, + options: SkipChangeConsumerOptions, +): Promise { + if (options.confirm !== true) { + throw new Error("Skipping change delivery requires confirm: true"); + } + const reason = options.reason.trim(); + if (reason.length === 0 || reason.length > 256) { + throw new TypeError("Skip reason must contain 1-256 characters"); + } + if (!/^\d+$/.test(options.through)) { + throw new TypeError("Skip position must be an opaque decimal string"); + } + const timestampValue = timestamp(options.now); + const state = await getChangeLogState(db); + const consumer = state ? await durableConsumer(db, consumerId) : null; + if (!state || !consumer) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const from = String(consumer.acknowledged_position); + if ( + consumer.generation_id !== state.generation || + consumer.bootstrap_state !== "ready" || + BigInt(options.through) <= BigInt(from) || + BigInt(options.through) > BigInt(state.head) + ) { + throw new Error("Skip position is outside this ready consumer's pending range"); + } + const actionId = crypto.randomUUID(); + const actionMarker = `operator_skip:${actionId}`; + const results = await db.batch([ + db + .prepare( + `UPDATE change_consumers + SET acknowledged_position = ?, lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_error_code = ?, last_error_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? + AND (lease_owner IS NULL OR lease_expires_at <= ?)`, + ) + .bind( + options.through, + actionMarker, + timestampValue, + timestampValue, + consumerId, + state.generation, + from, + timestampValue, + ), + db + .prepare( + `INSERT INTO change_consumer_actions + (action_id, generation_id, consumer_id, action, from_position, + through_position, reason, created_at) + SELECT ?, generation_id, consumer_id, 'skip', ?, + acknowledged_position, ?, ? + FROM change_consumers + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND last_error_code = ?`, + ) + .bind( + actionId, + from, + reason, + timestampValue, + consumerId, + state.generation, + options.through, + actionMarker, + ), + ]); + if (affectedRows(results[0]) !== 1 || affectedRows(results[1]) !== 1) { + throw new ChangeLeaseLostError( + `Change consumer ${consumerId} changed during skip`, + ); + } +} + +export interface PruneChangesOptions { + maxBatches?: number; + /** Delete only batches older than this timestamp. */ + olderThan?: number; +} + +export interface PruneChangesResult { + pruned: number; + retainedFloor: string; + safeThrough: string; + done: boolean; +} + +function affectedRows(result: unknown): number { + if (!result || typeof result !== "object") return 0; + const value = result as { + changes?: unknown; + rowsAffected?: unknown; + meta?: { changes?: unknown }; + }; + return Number(value.changes ?? value.rowsAffected ?? value.meta?.changes ?? 0); +} + +/** Consumer-aware bounded pruning. The slowest durable position, including a + * current bootstrap anchor, is the hard safety boundary. */ +export async function pruneChanges( + db: Database, + options: PruneChangesOptions = {}, +): Promise { + const maxBatches = integer( + options.maxBatches, + 500, + 1, + 5_000, + "maxBatches", + ); + if ( + options.olderThan !== undefined && + (!Number.isSafeInteger(options.olderThan) || options.olderThan < 0) + ) { + throw new TypeError("olderThan must be a non-negative safe timestamp"); + } + const state = await getChangeLogState(db); + if (!state) { + return { + pruned: 0, + retainedFloor: "0", + safeThrough: "0", + done: true, + }; + } + const slowest = await db + .prepare( + `SELECT MIN(acknowledged_position) AS safe_through + FROM change_consumers WHERE generation_id = ?`, + ) + .bind(state.generation) + .first<{ safe_through: number | string | null }>(); + const safeThrough = String(slowest?.safe_through ?? state.retainedFloor); + if (BigInt(safeThrough) <= BigInt(state.retainedFloor)) { + return { + pruned: 0, + retainedFloor: state.retainedFloor, + safeThrough, + done: true, + }; + } + + const results = await db.batch([ + db + .prepare( + `DELETE FROM change_batches + WHERE generation_id = ? AND position IN ( + SELECT position FROM change_batches + WHERE generation_id = ? AND position > ? AND position <= ? + AND (? IS NULL OR created_at < ?) + ORDER BY position LIMIT ? + )`, + ) + .bind( + state.generation, + state.generation, + state.retainedFloor, + safeThrough, + options.olderThan ?? null, + options.olderThan ?? null, + maxBatches, + ), + db + .prepare( + `UPDATE change_log_state + SET retained_floor_position = CASE + WHEN EXISTS ( + SELECT 1 FROM change_batches WHERE generation_id = ? + ) THEN ( + SELECT MIN(position) - 1 FROM change_batches + WHERE generation_id = ? + ) + ELSE head_position END + WHERE id = 1 AND generation_id = ?`, + ) + .bind(state.generation, state.generation, state.generation), + ]); + const updated = await getChangeLogState(db); + if (!updated || updated.generation !== state.generation) { + throw new ChangeGenerationMismatchError( + "Change-log generation changed during pruning", + ); + } + const pruned = affectedRows(results[0]); + return { + pruned, + retainedFloor: updated.retainedFloor, + safeThrough, + done: + pruned < maxBatches || + BigInt(updated.retainedFloor) >= BigInt(safeThrough), + }; +} + type DatabaseResolver = (db?: Database) => Database; /** Bound low-level API exposed as `contrail.changes`. */ @@ -938,6 +1298,98 @@ export class ChangeConsumers { return claimChanges(this.database(db), consumerId, options); } + claimBootstrapChanges( + consumerId: string, + options?: ChangeClaimOptions, + db?: Database, + ): Promise { + return claimCurrentBootstrapChanges( + this.database(db), + consumerId, + options, + ); + } + + claimSnapshotPage( + consumerId: string, + options?: CurrentBootstrapClaimOptions, + db?: Database, + ): Promise { + return claimCurrentSnapshotPage( + this.database(db), + this.config, + consumerId, + options, + ); + } + + ackSnapshotPage( + claim: CurrentSnapshotClaim, + options?: { now?: number }, + db?: Database, + ): Promise { + return acknowledgeCurrentSnapshotPage(this.database(db), claim, options); + } + + failSnapshotPage( + claim: CurrentSnapshotClaim, + failure: ChangeFailure, + options?: { now?: number }, + db?: Database, + ): Promise { + return failCurrentSnapshotPage( + this.database(db), + claim, + failure, + options, + ); + } + + claimActivation( + consumerId: string, + options?: { leaseMs?: number; now?: number }, + db?: Database, + ): Promise { + return claimCurrentActivation(this.database(db), consumerId, options); + } + + completeActivation( + claim: CurrentActivationClaim, + options?: { now?: number }, + db?: Database, + ): Promise { + return completeCurrentActivation(this.database(db), claim, options); + } + + renewBootstrap( + claim: T, + options?: { leaseMs?: number; now?: number }, + db?: Database, + ): Promise { + return renewCurrentBootstrapClaim(this.database(db), claim, options); + } + + failActivation( + claim: CurrentActivationClaim, + failure: ChangeFailure, + options?: { now?: number }, + db?: Database, + ): Promise { + return failCurrentActivation( + this.database(db), + claim, + failure, + options, + ); + } + + bootstrapStatus( + consumerId: string, + db?: Database, + ): Promise { + return getCurrentBootstrapStatus(this.database(db), consumerId); + } + hydrate(claim: ChangeClaim, db?: Database): Promise { return hydrateChanges(this.database(db), this.config, claim); } @@ -978,4 +1430,26 @@ export class ChangeConsumers { status(db?: Database): Promise { return getChangesStatus(this.database(db)); } + + readiness( + through?: string, + db?: Database, + ): Promise { + return getRequiredChangeConsumerReadiness(this.database(db), through); + } + + skip( + consumerId: string, + options: SkipChangeConsumerOptions, + db?: Database, + ): Promise { + return skipChangeConsumer(this.database(db), consumerId, options); + } + + prune( + options?: PruneChangesOptions, + db?: Database, + ): Promise { + return pruneChanges(this.database(db), options); + } } diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 18b05b5..d412723 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -37,7 +37,7 @@ import { } from "../types"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 13; +export const CONTRAIL_SCHEMA_VERSION = 14; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 47fbcc6..36b6e5d 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -36,6 +36,7 @@ export { } from "./core/change-log"; export type { ChangeLogState, RecordChange } from "./core/change-log"; export * from "./core/changes"; +export * from "./core/change-bootstrap"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/tests/built-changes.mjs b/packages/contrail/tests/built-changes.mjs index 1c3cf76..e73a1e8 100644 --- a/packages/contrail/tests/built-changes.mjs +++ b/packages/contrail/tests/built-changes.mjs @@ -36,6 +36,14 @@ try { ); assert.equal(retry.status, 0, retry.stderr); assert.match(retry.stdout, /retry is now due/); + + const prune = spawnSync( + process.execPath, + [cli, "changes", "prune", "--root", root, "--sqlite", database], + { cwd: resolve("."), encoding: "utf8" }, + ); + assert.equal(prune.status, 0, prune.stderr); + assert.match(prune.stdout, /pruned=0/); console.log("built change consumer CLI passed"); } finally { rmSync(root, { recursive: true, force: true }); diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts new file mode 100644 index 0000000..9571bfe --- /dev/null +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -0,0 +1,354 @@ +import { describe, expect, it } from "vitest"; +import { + acknowledgeChanges, + acknowledgeCurrentSnapshotPage, + claimChanges, + claimCurrentActivation, + claimCurrentBootstrapChanges, + claimCurrentSnapshotPage, + completeCurrentActivation, + createIngestEvent, + failCurrentSnapshotPage, + getChangesStatus, + getCurrentBootstrapStatus, + getRequiredChangeConsumerReadiness, + hydrateChanges, + ingestRecords, + initSchema, + pruneChanges, + resolveConfig, + retryChangeConsumer, + skipChangeConsumer, + type ChangeConsumerConfig, + type Database, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const EVENT = "com.example.event"; +const NOTE = "com.example.note"; +const logger = { log() {}, warn() {}, error() {} }; + +function config(consumers: Record) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { + event: { collection: EVENT }, + note: { collection: NOTE }, + }, + changes: { consumers }, + }); +} + +function event(options: { + rkey: string; + time: number; + operation?: "create" | "update" | "delete"; + collection?: string; + name?: string; +}) { + const collection = options.collection ?? EVENT; + const operation = options.operation ?? "update"; + const did = "did:plc:alice"; + return createIngestEvent({ + uri: `at://${did}/${collection}/${options.rkey}`, + did, + collection, + rkey: options.rkey, + operation, + cid: operation === "delete" ? null : `cid-${options.rkey}-${options.time}`, + value: + operation === "delete" + ? undefined + : { name: options.name ?? `${options.rkey}-${options.time}` }, + timeUs: options.time, + indexedAt: options.time + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: options.time, + revision: String(options.time), + cursor: String(options.time), + }, + }); +} + +async function apply(db: Database, resolved: ReturnType, ...events: ReturnType[]) { + await ingestRecords(db, events, resolved, { phase: "live" }); +} + +const keeper: ChangeConsumerConfig = { + collections: [EVENT, NOTE], + initial: "history", +}; + +describe("current-state change consumer bootstrap", () => { + it("adds a current consumer over existing coverage and converges snapshot plus racing tail", async () => { + const db = createSqliteDatabase(":memory:"); + const original = config({ keeper }); + await initSchema(db, original); + await apply( + db, + original, + event({ rkey: "a", time: 1, name: "a-one" }), + event({ rkey: "b", time: 2, name: "b-one" }), + ); + + const withSearch = config({ + keeper, + search: { + collections: [EVENT], + initial: "current", + requiredForActivation: true, + }, + }); + await initSchema(db, withSearch); + let status = await getCurrentBootstrapStatus(db, "search"); + expect(status).toMatchObject({ state: "pending", anchor: "1", position: "1" }); + expect(status.token).toMatch(/^[0-9a-f-]{36}$/); + expect(await getRequiredChangeConsumerReadiness(db, "1")).toMatchObject({ + ready: false, + pending: [expect.objectContaining({ id: "search", state: "pending" })], + }); + + const first = await claimCurrentSnapshotPage(db, withSearch, "search", { + pageSize: 1, + leaseMs: 10, + now: 100, + }); + expect(first?.records[0]).toMatchObject({ + rkey: "a", + value: { name: "a-one" }, + }); + + // Destination success followed by an ack crash replays the same stable page. + const replay = await claimCurrentSnapshotPage(db, withSearch, "search", { + pageSize: 1, + leaseMs: 10, + now: 111, + }); + expect(replay?.pageId).toBe(first?.pageId); + expect(replay?.bootstrapToken).toBe(first?.bootstrapToken); + + // Mutations after anchor M race the scan and must be corrected by the tail. + await apply( + db, + withSearch, + event({ rkey: "a", time: 3, name: "a-two" }), + event({ rkey: "b", time: 4, operation: "delete" }), + event({ rkey: "c", time: 5, name: "c-one" }), + ); + await acknowledgeCurrentSnapshotPage(db, replay!, { now: 112 }); + + let clock = 120; + for (;;) { + const page = await claimCurrentSnapshotPage(db, withSearch, "search", { + pageSize: 1, + now: clock++, + }); + if (!page) break; + await acknowledgeCurrentSnapshotPage(db, page, { now: clock++ }); + } + status = await getCurrentBootstrapStatus(db, "search"); + expect(status).toMatchObject({ + state: "catching-up", + anchor: "1", + target: "2", + position: "1", + }); + + const tail = await claimCurrentBootstrapChanges(db, "search", { + now: 200, + }); + expect(tail).toMatchObject({ + from: "1", + through: "2", + bootstrapTarget: "2", + bootstrapToken: status.token, + }); + const delivery = await hydrateChanges(db, withSearch, tail!); + expect(delivery.currentRecords.map((record) => record.rkey).sort()).toEqual([ + "a", + "c", + ]); + expect(delivery.absentUris).toEqual([ + `at://did:plc:alice/${EVENT}/b`, + ]); + await acknowledgeChanges(db, tail!, { now: 201 }); + expect((await getCurrentBootstrapStatus(db, "search")).state).toBe( + "activating", + ); + + const activation = await claimCurrentActivation(db, "search", { + now: 300, + leaseMs: 10, + }); + const repeatedActivation = await claimCurrentActivation(db, "search", { + now: 311, + leaseMs: 10, + }); + expect(repeatedActivation?.bootstrapToken).toBe( + activation?.bootstrapToken, + ); + await completeCurrentActivation(db, repeatedActivation!, { now: 312 }); + expect(await getCurrentBootstrapStatus(db, "search")).toMatchObject({ + state: "ready", + position: "2", + target: "2", + }); + expect(await getRequiredChangeConsumerReadiness(db, "2")).toMatchObject({ + ready: true, + pending: [], + }); + + await apply(db, withSearch, event({ rkey: "d", time: 6 })); + const ordinary = await claimChanges(db, "search", { now: 400 }); + expect(ordinary).toMatchObject({ from: "2", through: "3" }); + }); + + it("persists snapshot failure backoff and resumes the same page", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + keeper, + search: { collections: [EVENT], initial: "current" }, + }); + await initSchema(db, resolved); + await apply(db, resolved, event({ rkey: "a", time: 1 })); + const page = await claimCurrentSnapshotPage(db, resolved, "search", { + now: 100, + }); + await failCurrentSnapshotPage( + db, + page!, + { code: "destination_unavailable", nextAttemptAt: 200 }, + { now: 101 }, + ); + expect( + await claimCurrentSnapshotPage(db, resolved, "search", { now: 150 }), + ).toBeNull(); + await retryChangeConsumer(db, "search", { now: 150 }); + const retry = await claimCurrentSnapshotPage(db, resolved, "search", { + now: 150, + }); + expect(retry?.pageId).toBe(page?.pageId); + }); + + it("rejects additive consumers that require unlogged coverage", async () => { + const db = createSqliteDatabase(":memory:"); + const eventOnly = config({ + keeper: { collections: [EVENT], phases: ["live"], initial: "history" }, + }); + await initSchema(db, eventOnly); + const expanded = config({ + keeper: { collections: [EVENT], phases: ["live"], initial: "history" }, + notes: { collections: [NOTE], phases: ["live"], initial: "current" }, + }); + await expect(initSchema(db, expanded)).rejects.toThrow( + "expands collection/phase coverage", + ); + }); +}); + +describe("consumer-aware change pruning", () => { + it("requires explicit confirmation and audits an operator skip", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + blocked: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, resolved); + for (let position = 1; position <= 3; position++) { + await apply(db, resolved, event({ rkey: String(position), time: position })); + } + + await expect( + skipChangeConsumer(db, "blocked", { + through: "2", + reason: "destination was rebuilt out of band", + confirm: false, + now: 100, + }), + ).rejects.toThrow("confirm: true"); + await skipChangeConsumer(db, "blocked", { + through: "2", + reason: "destination was rebuilt out of band", + confirm: true, + now: 100, + }); + expect((await getChangesStatus(db)).consumers[0]).toMatchObject({ + position: "2", + lastErrorCode: expect.stringMatching(/^operator_skip:/), + }); + expect( + await db + .prepare( + `SELECT action, from_position, through_position, reason + FROM change_consumer_actions`, + ) + .first(), + ).toEqual({ + action: "skip", + from_position: 0, + through_position: 2, + reason: "destination was rebuilt out of band", + }); + }); + + it("prunes only through the slowest consumer in bounded resumable slices", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + fast: { collections: [EVENT], initial: "history" }, + slow: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, resolved); + for (let position = 1; position <= 5; position++) { + await apply(db, resolved, event({ rkey: String(position), time: position })); + } + + const fast = await claimChanges(db, "fast", { now: 100 }); + await acknowledgeChanges(db, fast!, { now: 101 }); + const slowFirst = await claimChanges(db, "slow", { + now: 100, + maxBatches: 2, + }); + await acknowledgeChanges(db, slowFirst!, { now: 101 }); + + let pruned = await pruneChanges(db, { maxBatches: 10 }); + expect(pruned).toEqual({ + pruned: 2, + retainedFloor: "2", + safeThrough: "2", + done: true, + }); + expect((await getChangesStatus(db)).rows).toBe(3); + + const slowRest = await claimChanges(db, "slow", { now: 102 }); + await acknowledgeChanges(db, slowRest!, { now: 103 }); + pruned = await pruneChanges(db, { maxBatches: 2 }); + expect(pruned).toMatchObject({ + pruned: 2, + retainedFloor: "4", + safeThrough: "5", + done: false, + }); + pruned = await pruneChanges(db, { maxBatches: 2 }); + expect(pruned).toMatchObject({ + pruned: 1, + retainedFloor: "5", + safeThrough: "5", + done: true, + }); + expect((await getChangesStatus(db)).rows).toBe(0); + + // A later history consumer starts at the retained floor, never a deleted row. + const withLate = config({ + fast: { collections: [EVENT], initial: "history" }, + slow: { collections: [EVENT], initial: "history" }, + late: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, withLate); + expect( + (await getChangesStatus(db)).consumers.find((item) => item.id === "late"), + ).toMatchObject({ position: "5", backlogBatches: 0 }); + }); +}); diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index ead13a1..00ae632 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -420,7 +420,7 @@ describe("transactional projection change log", () => { }, }); await expect(initSchema(initialized, changed)).rejects.toThrow( - "definitions differ", + "cannot be removed or modified", ); }); diff --git a/packages/contrail/tests/postgres-concurrent-init.test.ts b/packages/contrail/tests/postgres-concurrent-init.test.ts index dd6fdae..f6238de 100644 --- a/packages/contrail/tests/postgres-concurrent-init.test.ts +++ b/packages/contrail/tests/postgres-concurrent-init.test.ts @@ -71,7 +71,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage', 'change_consumer_actions'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); diff --git a/packages/contrail/tests/postgres-e2e.test.ts b/packages/contrail/tests/postgres-e2e.test.ts index 2e199b8..e71c6c2 100644 --- a/packages/contrail/tests/postgres-e2e.test.ts +++ b/packages/contrail/tests/postgres-e2e.test.ts @@ -13,7 +13,12 @@ import { createPostgresDatabase } from "../src/adapters/postgres"; import { initSchema } from "../src/index"; import { acknowledgeChanges, + acknowledgeCurrentSnapshotPage, claimChanges, + claimCurrentActivation, + claimCurrentBootstrapChanges, + claimCurrentSnapshotPage, + completeCurrentActivation, getChangesStatus, hydrateChanges, ingestRecords, @@ -75,6 +80,10 @@ const CHANGE_CONFIG = resolveConfig({ collections: ["community.lexicon.calendar.event"], initial: "history", }, + cache: { + collections: ["community.lexicon.calendar.event"], + initial: "current", + }, }, }, }); @@ -102,7 +111,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage', 'change_consumer_actions'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); @@ -618,6 +627,34 @@ if (!PG_URL) { ]), ); await acknowledgeChanges(db, analytics!, { now: 1_001 }); + + const snapshot = await claimCurrentSnapshotPage( + db, + CHANGE_CONFIG, + "cache", + { now: 2_000 }, + ); + expect(snapshot?.records).toHaveLength(1); + await acknowledgeCurrentSnapshotPage(db, snapshot!, { now: 2_001 }); + expect( + await claimCurrentSnapshotPage(db, CHANGE_CONFIG, "cache", { + now: 2_002, + }), + ).toBeNull(); + const tail = await claimCurrentBootstrapChanges(db, "cache", { + now: 2_003, + }); + expect(tail).toMatchObject({ from: "0", through: "1" }); + await acknowledgeChanges(db, tail!, { now: 2_004 }); + const activation = await claimCurrentActivation(db, "cache", { + now: 2_005, + }); + await completeCurrentActivation(db, activation!, { now: 2_006 }); + expect( + (await getChangesStatus(db)).consumers.find( + (consumer) => consumer.id === "cache", + ), + ).toMatchObject({ bootstrapState: "ready", position: "1" }); }); }); diff --git a/packages/contrail/tests/postgres.test.ts b/packages/contrail/tests/postgres.test.ts index 0b7b530..3e8bc8a 100644 --- a/packages/contrail/tests/postgres.test.ts +++ b/packages/contrail/tests/postgres.test.ts @@ -57,7 +57,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage', 'change_consumer_actions'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); -- 2.51.2 From 21d8ed5302107d796ecc860c32118ad362fabcb1 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:31:06 +0200 Subject: [PATCH 05/34] Add change delivery runtimes --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 28 + packages/contrail/src/contrail.ts | 25 + packages/contrail/src/core/delivery.ts | 503 ++++++++++++++++++ .../contrail/src/core/router/diagnostics.ts | 11 +- packages/contrail/src/core/router/index.ts | 6 + packages/contrail/src/index.ts | 1 + packages/contrail/src/worker/index.ts | 113 +++- packages/contrail/tests/delivery.test.ts | 333 ++++++++++++ packages/contrail/tests/worker.test.ts | 171 +++++- 10 files changed, 1178 insertions(+), 15 deletions(-) create mode 100644 packages/contrail/src/core/delivery.ts create mode 100644 packages/contrail/tests/delivery.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 27dd926..619eb38 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 09f1052..6eac71f 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -96,6 +96,34 @@ const config = { Static definitions contain no handlers, URLs, clients, credentials, or secrets. Contrail registers them with a random database-generation ID and collection/phase coverage ledger. A winning logical put/delete appends one compact URI/version reference in the same transaction as canonical and derived state plus the source checkpoint. Duplicate, stale, same-CID, absent-delete, rejected, and rolled-back mutations append nothing. Record bodies are hydrated from current state by the later delivery layer rather than copied into the log. +Cloudflare Workers bind handlers and runtime-only secrets separately from static policy: + +```ts +export default createWorker(config, { + deliveries: { + search: async (batch, { env, signal }) => { + await updateSearch(env.SEARCH_KEY, batch, signal); + }, + }, + changeBootstraps: { + search: { + snapshot: async (page, { env, signal }) => { + await writeCandidateIndex(env.SEARCH_KEY, page, signal); + }, + activate: async (activation, { env, signal }) => { + await activateCandidateIdempotently( + env.SEARCH_KEY, + activation.bootstrapToken, + signal, + ); + }, + }, + }, +}); +``` + +Scheduled execution runs ingestion, due historical retries, then bounded fair delivery rounds. One consumer failure is persisted with backoff and does not stop ingestion or another consumer. A successful notify request schedules a best-effort one-round wake through `ExecutionContext`; projection success never depends on it. Persistent deployments run `contrail.runPersistentDeliveries()` alongside `runPersistent()`. + Low-level delivery uses bounded leases and compare-and-swap acknowledgement: ```ts diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index 859d054..8cccb88 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -15,6 +15,12 @@ import { } from "./core/validation"; import { getIngestDiagnostics } from "./core/diagnostics"; import { ChangeConsumers } from "./core/changes"; +import { + runPersistentChangeDeliveries, + type CurrentBootstrapRuntimeHandlers, + type DeliveryHandlers, + type DeliveryRuntimeOptions, +} from "./core/delivery"; import { optimizeDatabase } from "./core/db/optimize"; import { assertServingSourceCompatibility, @@ -160,6 +166,25 @@ export class Contrail { await Promise.all(tasks); } + /** Run a persistent fair change-delivery supervisor. Run this alongside + * `runPersistent()`; destination failures never stop source ingestion. */ + async runPersistentDeliveries( + options: { + env: Env; + deliveries: DeliveryHandlers; + bootstraps?: CurrentBootstrapRuntimeHandlers; + runtime?: DeliveryRuntimeOptions & { idleMs?: number }; + }, + db?: Database, + ): Promise { + await runPersistentChangeDeliveries({ + changes: this.changes, + config: this.config, + db: this.getDb(db), + ...options, + }); + } + /** Run *only* the labeler ingestion cycle. Escape hatch for callers who * want to run record and label ingestion in separate processes / workers. * `ingest()` already covers the typical case. */ diff --git a/packages/contrail/src/core/delivery.ts b/packages/contrail/src/core/delivery.ts new file mode 100644 index 0000000..65a2283 --- /dev/null +++ b/packages/contrail/src/core/delivery.ts @@ -0,0 +1,503 @@ +import type { ContrailConfig, Database, Logger } from "./types"; +import type { + ChangeClaimOptions, + ChangeConsumers, + DeliveryBatch, +} from "./changes"; +import type { + CurrentActivationClaim, + CurrentSnapshotClaim, +} from "./change-bootstrap"; + +export interface DeliveryContext { + env: Env; + signal: AbortSignal; + attempt: number; +} + +export type DeliveryHandler = ( + batch: DeliveryBatch, + context: DeliveryContext, +) => Promise; + +export type SnapshotDeliveryPage = Omit; +export type ActivationDelivery = Omit; + +export interface CurrentBootstrapRuntimeHandler { + snapshot: ( + page: SnapshotDeliveryPage, + context: DeliveryContext, + ) => Promise; + activate: ( + activation: ActivationDelivery, + context: DeliveryContext, + ) => Promise; +} + +export type DeliveryHandlers = Record>; +export type CurrentBootstrapRuntimeHandlers = Record< + string, + CurrentBootstrapRuntimeHandler +>; + +export interface DeliveryRuntimeOptions { + maxRounds?: number; + maxDurationMs?: number; + claim?: ChangeClaimOptions; + baseRetryMs?: number; + maxRetryMs?: number; + jitter?: number; + signal?: AbortSignal; + logger?: Logger; + /** @internal Deterministic runtime seams. */ + clock?: () => number; + random?: () => number; +} + +export interface DeliverySliceResult { + steps: number; + delivered: number; + snapshotPages: number; + activations: number; + failures: number; + consumerErrors: Record; + deadlineReached: boolean; +} + +function boundedInteger( + value: number | undefined, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new TypeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return result; +} + +export function validateDeliveryHandlers( + config: ContrailConfig, + deliveries: DeliveryHandlers, + bootstraps: CurrentBootstrapRuntimeHandlers = {}, +): void { + const consumers = config.changes?.consumers ?? {}; + for (const id of Object.keys(consumers)) { + if (typeof deliveries[id] !== "function") { + throw new Error(`Missing runtime delivery handler for change consumer ${id}`); + } + if ( + consumers[id]!.initial === "current" && + (!bootstraps[id] || + typeof bootstraps[id].snapshot !== "function" || + typeof bootstraps[id].activate !== "function") + ) { + throw new Error( + `Missing current-state bootstrap handlers for change consumer ${id}`, + ); + } + } + for (const id of Object.keys(deliveries)) { + if (!consumers[id]) { + throw new Error(`Runtime delivery handler ${id} has no static consumer definition`); + } + } + for (const id of Object.keys(bootstraps)) { + if (!consumers[id] || consumers[id]!.initial !== "current") { + throw new Error(`Runtime bootstrap handler ${id} has no current consumer definition`); + } + } +} + +function publicSnapshot(claim: CurrentSnapshotClaim): SnapshotDeliveryPage { + const { leaseOwner: _leaseOwner, ...page } = claim; + return page; +} + +function publicActivation(claim: CurrentActivationClaim): ActivationDelivery { + const { leaseOwner: _leaseOwner, ...activation } = claim; + return activation; +} + +async function withDeadline( + parent: AbortSignal | undefined, + deadline: number, + clock: () => number, + callback: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + const abort = () => controller.abort(parent?.reason ?? new Error("Delivery cancelled")); + if (parent?.aborted) abort(); + else parent?.addEventListener("abort", abort, { once: true }); + const remaining = Math.max(0, deadline - clock()); + const timer = setTimeout( + () => controller.abort(new Error("Delivery deadline reached")), + remaining, + ); + try { + return await callback(controller.signal); + } finally { + clearTimeout(timer); + parent?.removeEventListener("abort", abort); + } +} + +function retryAt( + attempt: number, + now: number, + base: number, + maximum: number, + jitter: number, + random: () => number, +): number { + const exponential = Math.min(maximum, base * 2 ** Math.min(20, attempt - 1)); + const factor = 1 + (random() * 2 - 1) * jitter; + return now + Math.max(1, Math.round(exponential * factor)); +} + +interface RuntimeState { + changes: ChangeConsumers; + config: ContrailConfig; + db: Database; + env: Env; + deliveries: DeliveryHandlers; + bootstraps: CurrentBootstrapRuntimeHandlers; + claim: ChangeClaimOptions; + deadline: number; + baseRetryMs: number; + maxRetryMs: number; + jitter: number; + signal?: AbortSignal; + logger: Logger; + clock: () => number; + random: () => number; +} + +async function persistFailure( + state: RuntimeState, + consumerId: string, + claim: Parameters[0], + attempt: number, +): Promise { + const now = state.clock(); + try { + await state.changes.fail( + claim, + { + code: "handler_error", + nextAttemptAt: retryAt( + attempt, + now, + state.baseRetryMs, + state.maxRetryMs, + state.jitter, + state.random, + ), + }, + { now }, + state.db, + ); + } catch (error) { + state.logger.warn( + `[changes] consumer=${consumerId} could not persist failure: ${error}`, + ); + } +} + +async function runNormal( + state: RuntimeState, + consumerId: string, + bootstrap: boolean, +): Promise<"empty" | "delivered" | "failed"> { + const now = state.clock(); + const claimOptions = { ...state.claim, now }; + const claim = bootstrap + ? await state.changes.claimBootstrapChanges( + consumerId, + claimOptions, + state.db, + ) + : await state.changes.claim(consumerId, claimOptions, state.db); + if (!claim) return "empty"; + try { + const batch = await state.changes.hydrate(claim, state.db); + await withDeadline(state.signal, state.deadline, state.clock, (signal) => + state.deliveries[consumerId]!(batch, { + env: state.env, + signal, + attempt: claim.attempt, + }), + ); + await state.changes.ack( + claim, + { now: state.clock() }, + state.db, + ); + return "delivered"; + } catch (error) { + state.logger.warn(`[changes] consumer=${consumerId} delivery failed: ${error}`); + await persistFailure( + state as RuntimeState, + consumerId, + claim, + claim.attempt, + ); + return "failed"; + } +} + +async function runCurrent( + state: RuntimeState, + consumerId: string, +): Promise<"empty" | "delivered" | "snapshot" | "activation" | "failed" | "progressed"> { + const before = await state.changes.bootstrapStatus(consumerId, state.db); + if (before.state === "ready") { + return runNormal(state, consumerId, false); + } + if (before.state === "pending" || before.state === "scanning") { + const claim = await state.changes.claimSnapshotPage( + consumerId, + { ...state.claim, now: state.clock() }, + state.db, + ); + if (!claim) { + const after = await state.changes.bootstrapStatus(consumerId, state.db); + return after.state !== before.state ? "progressed" : "empty"; + } + try { + await withDeadline(state.signal, state.deadline, state.clock, (signal) => + state.bootstraps[consumerId]!.snapshot(publicSnapshot(claim), { + env: state.env, + signal, + attempt: claim.attempt, + }), + ); + await state.changes.ackSnapshotPage( + claim, + { now: state.clock() }, + state.db, + ); + return "snapshot"; + } catch (error) { + state.logger.warn(`[changes] consumer=${consumerId} snapshot failed: ${error}`); + try { + await state.changes.failSnapshotPage( + claim, + { + code: "snapshot_handler_error", + nextAttemptAt: retryAt( + claim.attempt, + state.clock(), + state.baseRetryMs, + state.maxRetryMs, + state.jitter, + state.random, + ), + }, + { now: state.clock() }, + state.db, + ); + } catch (failureError) { + state.logger.warn(`[changes] consumer=${consumerId} snapshot failure state lost: ${failureError}`); + } + return "failed"; + } + } + if (before.state === "catching-up") { + const result = await runNormal(state, consumerId, true); + if (result !== "empty") return result; + const after = await state.changes.bootstrapStatus(consumerId, state.db); + return after.state !== before.state ? "progressed" : "empty"; + } + if (before.state === "activating") { + const claim = await state.changes.claimActivation( + consumerId, + { now: state.clock() }, + state.db, + ); + if (!claim) return "empty"; + try { + await withDeadline(state.signal, state.deadline, state.clock, (signal) => + state.bootstraps[consumerId]!.activate(publicActivation(claim), { + env: state.env, + signal, + attempt: claim.attempt, + }), + ); + await state.changes.completeActivation( + claim, + { now: state.clock() }, + state.db, + ); + return "activation"; + } catch (error) { + state.logger.warn(`[changes] consumer=${consumerId} activation failed: ${error}`); + try { + await state.changes.failActivation( + claim, + { + code: "activation_handler_error", + nextAttemptAt: retryAt( + claim.attempt, + state.clock(), + state.baseRetryMs, + state.maxRetryMs, + state.jitter, + state.random, + ), + }, + { now: state.clock() }, + state.db, + ); + } catch (failureError) { + state.logger.warn(`[changes] consumer=${consumerId} activation failure state lost: ${failureError}`); + } + return "failed"; + } + } + return "empty"; +} + +/** Run fair round-robin delivery work without coupling failures to ingestion. */ +export async function runChangeDeliverySlice(options: { + changes: ChangeConsumers; + config: ContrailConfig; + db: Database; + env: Env; + deliveries: DeliveryHandlers; + bootstraps?: CurrentBootstrapRuntimeHandlers; + runtime?: DeliveryRuntimeOptions; +}): Promise { + const bootstraps = options.bootstraps ?? {}; + validateDeliveryHandlers(options.config, options.deliveries, bootstraps); + const runtime = options.runtime ?? {}; + const clock = runtime.clock ?? Date.now; + const maxRounds = boundedInteger(runtime.maxRounds, 4, 1, 100, "maxRounds"); + const maxDurationMs = boundedInteger( + runtime.maxDurationMs, + 15_000, + 1, + 10 * 60_000, + "maxDurationMs", + ); + const baseRetryMs = boundedInteger( + runtime.baseRetryMs, + 1_000, + 1, + 60 * 60_000, + "baseRetryMs", + ); + const maxRetryMs = boundedInteger( + runtime.maxRetryMs, + 60 * 60_000, + baseRetryMs, + 48 * 60 * 60_000, + "maxRetryMs", + ); + const jitter = runtime.jitter ?? 0.2; + if (!Number.isFinite(jitter) || jitter < 0 || jitter > 1) { + throw new TypeError("jitter must be between 0 and 1"); + } + const deadline = clock() + maxDurationMs; + const state: RuntimeState = { + changes: options.changes, + config: options.config, + db: options.db, + env: options.env, + deliveries: options.deliveries, + bootstraps, + claim: runtime.claim ?? {}, + deadline, + baseRetryMs, + maxRetryMs, + jitter, + signal: runtime.signal, + logger: runtime.logger ?? options.config.logger ?? console, + clock, + random: runtime.random ?? Math.random, + }; + const result: DeliverySliceResult = { + steps: 0, + delivered: 0, + snapshotPages: 0, + activations: 0, + failures: 0, + consumerErrors: {}, + deadlineReached: false, + }; + const consumers = Object.entries(options.config.changes?.consumers ?? {}).sort( + ([left], [right]) => left.localeCompare(right), + ); + + for (let round = 0; round < maxRounds; round++) { + let progressed = false; + for (const [consumerId, consumer] of consumers) { + if (runtime.signal?.aborted || clock() >= deadline) { + result.deadlineReached = true; + return result; + } + try { + const outcome = consumer.initial === "current" + ? await runCurrent(state, consumerId) + : await runNormal(state, consumerId, false); + if (outcome !== "empty") { + progressed = true; + result.steps++; + } + if (outcome === "delivered") result.delivered++; + else if (outcome === "snapshot") result.snapshotPages++; + else if (outcome === "activation") result.activations++; + else if (outcome === "failed") result.failures++; + } catch (error) { + result.failures++; + result.consumerErrors[consumerId] = + error instanceof Error ? error.message : String(error); + state.logger.error(`[changes] consumer=${consumerId} runtime failed: ${error}`); + } + } + if (!progressed) break; + } + result.deadlineReached = clock() >= deadline; + return result; +} + +function waitFor(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(done, ms); + function done() { + clearTimeout(timer); + signal?.removeEventListener("abort", done); + resolve(); + } + signal?.addEventListener("abort", done, { once: true }); + }); +} + +/** Persistent fair supervisor. Source ingestion remains a separate task and is + * never cancelled by a destination outage. */ +export async function runPersistentChangeDeliveries(options: { + changes: ChangeConsumers; + config: ContrailConfig; + db: Database; + env: Env; + deliveries: DeliveryHandlers; + bootstraps?: CurrentBootstrapRuntimeHandlers; + runtime?: DeliveryRuntimeOptions & { idleMs?: number }; +}): Promise { + const idleMs = boundedInteger( + options.runtime?.idleMs, + 1_000, + 1, + 60_000, + "idleMs", + ); + while (!options.runtime?.signal?.aborted) { + const result = await runChangeDeliverySlice(options); + if (result.steps === 0) { + await waitFor(idleMs, options.runtime?.signal); + } + } +} diff --git a/packages/contrail/src/core/router/diagnostics.ts b/packages/contrail/src/core/router/diagnostics.ts index 5a7927d..a080011 100644 --- a/packages/contrail/src/core/router/diagnostics.ts +++ b/packages/contrail/src/core/router/diagnostics.ts @@ -3,6 +3,7 @@ import type { ContrailConfig, Database } from "../types"; import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; import { getLastCursor, getServingSourcePosition } from "../db"; import { getBackfillStatus } from "../status"; +import { getRequiredChangeConsumerReadiness } from "../changes"; export interface CursorStatus { cursor: number | null; @@ -48,9 +49,12 @@ export async function getStatusOverview(db: Database, config: ContrailConfig) { } } - const [ingestion, backfill] = await Promise.all([ + const [ingestion, backfill, requiredDelivery] = await Promise.all([ getCursorStatus(db), getBackfillStatus(db, config), + config.changes && Object.keys(config.changes.consumers).length > 0 + ? getRequiredChangeConsumerReadiness(db) + : Promise.resolve({ ready: true, through: "0", pending: [] }), ]); return { @@ -59,6 +63,11 @@ export async function getStatusOverview(db: Database, config: ContrailConfig) { collections, ingestion, backfill, + delivery: { + required: requiredDelivery.ready ? "ready" as const : "catching_up" as const, + pending: requiredDelivery.pending.length, + through: requiredDelivery.through, + }, }; } diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index a4e9720..8f66072 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -53,6 +53,9 @@ export function createApp( const overview = await getStatusOverview(db, config); if (!options.publicService) return c.json(overview); c.header("cache-control", "public, max-age=15, stale-while-revalidate=45"); + const hasRequiredDelivery = Object.values( + config.changes?.consumers ?? {}, + ).some((consumer) => consumer.requiredForActivation === true); return c.json({ status: overview.status, serving: "ready", @@ -63,6 +66,9 @@ export function createApp( seconds_ago: overview.ingestion.seconds_ago, }, backfill: overview.backfill, + ...(hasRequiredDelivery + ? { required_delivery: overview.delivery.required } + : {}), }); }); app.get("/health", (c) => c.json({ status: "ok" })); diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 36b6e5d..dfd17b6 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -37,6 +37,7 @@ export { export type { ChangeLogState, RecordChange } from "./core/change-log"; export * from "./core/changes"; export * from "./core/change-bootstrap"; +export * from "./core/delivery"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/src/worker/index.ts b/packages/contrail/src/worker/index.ts index a75f0a0..d38ba04 100644 --- a/packages/contrail/src/worker/index.ts +++ b/packages/contrail/src/worker/index.ts @@ -18,6 +18,13 @@ import { Contrail } from "../contrail.js"; import { createHandler } from "../server.js"; import type { ContrailConfig, Database } from "../core/types.js"; import type { BackfillRetryOptions } from "../core/backfill.js"; +import { + runChangeDeliverySlice, + validateDeliveryHandlers, + type CurrentBootstrapRuntimeHandlers, + type DeliveryHandlers, + type DeliveryRuntimeOptions, +} from "../core/delivery.js"; import { normalizePublicServiceEndpoint, validatePublicServiceAuthEndpoint, @@ -25,7 +32,9 @@ import { type PublicServiceOptions, } from "../public-service.js"; -export interface CreateWorkerOptions { +type WorkerEnv = Record; + +export interface CreateWorkerOptions { /** D1 binding name in wrangler env. Default: `"DB"`. */ binding?: string; /** Exact generated/pinned bundle exposed for type generation and used by @@ -36,18 +45,33 @@ export interface CreateWorkerOptions { /** Bounded pending-account retry slice after each scheduled ingest. Enabled * by default; pass `false` to disable or options to tune its budget. */ backfillRetries?: BackfillRetryOptions | false; + /** Runtime delivery handlers, kept separate from static consumer policy. */ + deliveries?: DeliveryHandlers; + /** Snapshot and activation handlers for `initial: "current"` consumers. */ + changeBootstraps?: CurrentBootstrapRuntimeHandlers; + /** Bounded scheduled delivery policy. Set false only when another runtime + * owns all configured consumers. */ + delivery?: DeliveryRuntimeOptions | false; /** Runs once per isolate, after schema init, before handling the first * request. Use for app-specific setup that needs a live DB handle. */ - onInit?: (env: Record, db: Database) => void | Promise; + onInit?: (env: Env, db: Database) => void | Promise; } -type WorkerEnv = Record; - -export function createWorker( +export function createWorker( config: ContrailConfig, - options: CreateWorkerOptions = {} + options: CreateWorkerOptions = {} ) { const binding = options.binding ?? "DB"; + const deliveryEnabled = + options.delivery !== false && + Object.keys(config.changes?.consumers ?? {}).length > 0; + if (deliveryEnabled) { + validateDeliveryHandlers( + config, + options.deliveries ?? {}, + options.changeBootstraps ?? {}, + ); + } if (options.publicService) { normalizePublicServiceEndpoint(options.publicService.endpoint); validatePublicServiceLexicons(config, options.lexicons ?? []); @@ -60,7 +84,7 @@ export function createWorker( }); let ready = false; - const ensureReady = async (env: WorkerEnv, db: Database): Promise => { + const ensureReady = async (env: Env, db: Database): Promise => { if (ready) return; await contrail.init(db); await options.onInit?.(env, db); @@ -68,14 +92,52 @@ export function createWorker( }; return { - async fetch(request: Request, env: WorkerEnv): Promise { + async fetch( + request: Request, + env: Env, + ctx?: ExecutionContext, + ): Promise { const db = env[binding] as Database; await ensureReady(env, db); - return (await handle(request, db)) as Response; + const response = (await handle(request, db)) as Response; + const notifyPath = `/xrpc/${contrail.config.namespace}.notifyOfUpdate`; + if ( + deliveryEnabled && + ctx && + response.ok && + request.method === "POST" && + new URL(request.url).pathname === notifyPath + ) { + ctx.waitUntil( + runChangeDeliverySlice({ + changes: contrail.changes, + config: contrail.config, + db, + env, + deliveries: options.deliveries!, + bootstraps: options.changeBootstraps, + runtime: { + ...(options.delivery || {}), + maxRounds: 1, + maxDurationMs: Math.min( + options.delivery && options.delivery.maxDurationMs + ? options.delivery.maxDurationMs + : 5_000, + 5_000, + ), + }, + }).catch((error) => { + contrail.config.logger?.error( + `[changes] immediate delivery wake failed: ${error}`, + ); + }), + ); + } + return response; }, async scheduled( _event: ScheduledEvent, - env: WorkerEnv, + env: Env, ctx: ExecutionContext ): Promise { const db = env[binding] as Database; @@ -84,9 +146,36 @@ export function createWorker( // failures. A database lease prevents overlap with a manual backfill. ctx.waitUntil( (async () => { - await contrail.ingest({}, db); + try { + await contrail.ingest({}, db); + } catch (error) { + contrail.config.logger?.error(`[ingest] scheduled cycle failed: ${error}`); + } if (options.backfillRetries !== false) { - await contrail.retryBackfill(options.backfillRetries, db); + try { + await contrail.retryBackfill(options.backfillRetries, db); + } catch (error) { + contrail.config.logger?.error( + `[backfill] scheduled retry slice failed: ${error}`, + ); + } + } + if (deliveryEnabled) { + try { + await runChangeDeliverySlice({ + changes: contrail.changes, + config: contrail.config, + db, + env, + deliveries: options.deliveries!, + bootstraps: options.changeBootstraps, + runtime: options.delivery || undefined, + }); + } catch (error) { + contrail.config.logger?.error( + `[changes] scheduled delivery slice failed: ${error}`, + ); + } } })() ); diff --git a/packages/contrail/tests/delivery.test.ts b/packages/contrail/tests/delivery.test.ts new file mode 100644 index 0000000..2835fe5 --- /dev/null +++ b/packages/contrail/tests/delivery.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; +import { + createIngestEvent, + getChangesStatus, + ingestRecords, + initSchema, + resolveConfig, + runChangeDeliverySlice, + runPersistentChangeDeliveries, + validateDeliveryHandlers, + type ChangeConsumerConfig, + type Database, + type DeliveryHandlers, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { Contrail } from "../src/contrail"; + +const EVENT = "com.example.event"; +const logger = { log() {}, warn() {}, error() {} }; + +function config(consumers: Record) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { event: { collection: EVENT } }, + changes: { consumers }, + }); +} + +function event(rkey: string, time: number) { + const did = "did:plc:alice"; + return createIngestEvent({ + uri: `at://${did}/${EVENT}/${rkey}`, + did, + collection: EVENT, + rkey, + operation: "update", + cid: `cid-${rkey}-${time}`, + value: { name: rkey }, + timeUs: time, + indexedAt: time + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: time, + revision: String(time), + cursor: String(time), + }, + }); +} + +async function append(db: Database, resolved: ReturnType, rkey: string, time: number) { + await ingestRecords(db, [event(rkey, time)], resolved); +} + +describe("change delivery runtime", () => { + it("fails startup for missing, extra, or incomplete runtime handlers", () => { + const resolved = config({ + search: { collections: [EVENT], initial: "current" }, + }); + expect(() => validateDeliveryHandlers(resolved, {})).toThrow( + "Missing runtime delivery handler", + ); + expect(() => + validateDeliveryHandlers(resolved, { search: async () => {} }), + ).toThrow("Missing current-state bootstrap handlers"); + expect(() => + validateDeliveryHandlers( + resolved, + { search: async () => {}, extra: async () => {} }, + { + search: { + snapshot: async () => {}, + activate: async () => {}, + }, + }, + ), + ).toThrow("extra has no static consumer definition"); + }); + + it("runs one bounded claim per consumer per fair round", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + alpha: { collections: [EVENT], initial: "history" }, + beta: { collections: [EVENT], initial: "history" }, + gamma: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + await append(db, resolved, "two", 2); + + const order: string[] = []; + const handlers: DeliveryHandlers<{}> = Object.fromEntries( + ["alpha", "beta", "gamma"].map((id) => [ + id, + async () => { + order.push(id); + }, + ]), + ); + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: handlers, + runtime: { + maxRounds: 2, + claim: { maxBatches: 1 }, + jitter: 0, + }, + }); + expect(order).toEqual([ + "alpha", + "beta", + "gamma", + "alpha", + "beta", + "gamma", + ]); + expect(result).toMatchObject({ + delivered: 6, + failures: 0, + steps: 6, + }); + }); + + it("isolates one failing consumer and resumes it after persisted backoff", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + broken: { collections: [EVENT], initial: "history" }, + healthy: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + + let fail = true; + const handlers = { + broken: async () => { + if (fail) throw new Error("destination down"); + }, + healthy: async () => {}, + }; + let current = 1_000; + const first = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: handlers, + runtime: { + maxRounds: 1, + baseRetryMs: 100, + maxRetryMs: 100, + jitter: 0, + clock: () => current, + }, + }); + expect(first).toMatchObject({ delivered: 1, failures: 1 }); + let status = await getChangesStatus(db); + expect(status.consumers.find((item) => item.id === "broken")).toMatchObject({ + position: "0", + attempts: 1, + nextAttemptAt: 1_100, + }); + expect(status.consumers.find((item) => item.id === "healthy")).toMatchObject({ + position: "1", + }); + + fail = false; + current = 1_101; + const resumed = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: handlers, + runtime: { + maxRounds: 1, + jitter: 0, + clock: () => current, + }, + }); + expect(resumed.delivered).toBe(1); + status = await getChangesStatus(db); + expect(status.consumers.find((item) => item.id === "broken")).toMatchObject({ + position: "1", + attempts: 0, + }); + }); + + it("drives current snapshot, catch-up, and idempotent activation", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + search: { + collections: [EVENT], + initial: "current", + requiredForActivation: true, + }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + expect( + await ( + await contrail.app().fetch(new Request("http://localhost/status")) + ).json(), + ).toMatchObject({ + delivery: { required: "catching_up", pending: 1 }, + }); + + const calls: string[] = []; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: { secret: "runtime-only" }, + deliveries: { + search: async (batch, context) => { + calls.push(`tail:${batch.cursor.through}:${context.env.secret}`); + }, + }, + bootstraps: { + search: { + snapshot: async (page) => { + expect(page).not.toHaveProperty("leaseOwner"); + calls.push(`snapshot:${page.records.length}`); + }, + activate: async (activation) => { + expect(activation).not.toHaveProperty("leaseOwner"); + calls.push(`activate:${activation.target}`); + }, + }, + }, + runtime: { maxRounds: 6, jitter: 0 }, + }); + expect(calls).toEqual([ + "snapshot:1", + "tail:1:runtime-only", + "activate:1", + ]); + expect(result).toMatchObject({ + snapshotPages: 1, + delivered: 1, + activations: 1, + failures: 0, + }); + expect(await contrail.changes.bootstrapStatus("search")).toMatchObject({ + state: "ready", + position: "1", + }); + expect( + await ( + await contrail.app().fetch(new Request("http://localhost/status")) + ).json(), + ).toMatchObject({ + delivery: { required: "ready", pending: 0 }, + }); + }); + + it("aborts destination work at the runtime deadline", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + webhook: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + let aborted = false; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { + webhook: async (_batch, { signal }) => { + await new Promise((resolve) => { + if (signal.aborted) { + aborted = true; + resolve(); + return; + } + signal.addEventListener( + "abort", + () => { + aborted = true; + resolve(); + }, + { once: true }, + ); + }); + }, + }, + runtime: { maxRounds: 1, maxDurationMs: 5 }, + }); + expect(aborted).toBe(true); + expect(result.deadlineReached).toBe(true); + }); + + it("runs a persistent supervisor until cancellation", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + webhook: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + const controller = new AbortController(); + let deliveries = 0; + await runPersistentChangeDeliveries({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { + webhook: async () => { + deliveries++; + controller.abort(); + }, + }, + runtime: { + signal: controller.signal, + idleMs: 1, + maxRounds: 1, + }, + }); + expect(deliveries).toBe(1); + expect((await getChangesStatus(db)).consumers[0].position).toBe("1"); + }); +}); diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts index bd69244..49cedd6 100644 --- a/packages/contrail/tests/worker.test.ts +++ b/packages/contrail/tests/worker.test.ts @@ -2,7 +2,13 @@ import { describe, it, expect, vi } from "vitest"; import { createWorker } from "../src/worker"; import { Contrail } from "../src/contrail"; import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { saveCursor, type ContrailConfig } from "../src/index"; +import { + createIngestEvent, + ingestRecords, + resolveConfig, + saveCursor, + type ContrailConfig, +} from "../src/index"; const MINIMAL_CONFIG: ContrailConfig = { namespace: "com.example", @@ -71,6 +77,24 @@ describe("createWorker", () => { ).not.toThrow(); }); + it("rejects configured consumers without matching runtime handlers", () => { + const config: ContrailConfig = { + ...MINIMAL_CONFIG, + changes: { + consumers: { + webhook: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, + }; + expect(() => createWorker(config)).toThrow( + "Missing runtime delivery handler", + ); + expect(() => createWorker(config, { delivery: false })).not.toThrow(); + }); + it("returns an object with fetch + scheduled handlers", () => { const worker = createWorker(MINIMAL_CONFIG); expect(typeof worker.fetch).toBe("function"); @@ -497,6 +521,151 @@ describe("createWorker", () => { ).toThrow("public method requires a matching query Lexicon"); }); + it("scheduled handler isolates ingest failure before retry and delivery", async () => { + const order: string[] = []; + const ingest = vi + .spyOn(Contrail.prototype, "ingest") + .mockImplementation(async () => { + order.push("ingest"); + throw new Error("source unavailable"); + }); + const retry = vi + .spyOn(Contrail.prototype, "retryBackfill") + .mockImplementation(async () => { + order.push("retry"); + return { + attempted: 0, + completed: 0, + failed: 0, + records: 0, + skipped: false, + }; + }); + const config: ContrailConfig = { + namespace: "com.example", + profiles: [], + logger: { log() {}, warn() {}, error() {} }, + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + webhook: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, + }; + + try { + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(config, { + deliveries: { + webhook: async () => { + order.push("delivery"); + }, + }, + }); + const env = { DB: db }; + await worker.fetch(new Request("http://localhost/health"), env); + const resolved = resolveConfig(config); + await ingestRecords( + db, + [ + createIngestEvent({ + did: "did:plc:alice", + collection: "community.lexicon.calendar.event", + rkey: "one", + operation: "create", + cid: "cid-one", + value: { name: "one" }, + timeUs: 1, + }), + ], + resolved, + ); + const waitUntil = vi.fn(); + const ctx = { + waitUntil, + passThroughOnException: vi.fn(), + } as unknown as ExecutionContext; + + await worker.scheduled({} as ScheduledEvent, env, ctx); + await waitUntil.mock.calls[0][0]; + expect(order).toEqual(["ingest", "retry", "delivery"]); + } finally { + ingest.mockRestore(); + retry.mockRestore(); + } + }); + + it("schedules a best-effort delivery wake after successful notify", async () => { + const config: ContrailConfig = { + namespace: "com.example", + profiles: [], + notify: true, + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + webhook: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, + }; + const delivered = vi.fn(async () => {}); + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(config, { + deliveries: { webhook: delivered }, + }); + const env = { DB: db }; + await worker.fetch(new Request("http://localhost/health"), env); + await ingestRecords( + db, + [ + createIngestEvent({ + did: "did:plc:alice", + collection: "community.lexicon.calendar.event", + rkey: "one", + operation: "create", + cid: "cid-one", + value: { name: "one" }, + timeUs: 1, + }), + ], + resolveConfig(config), + ); + + const handler = vi + .spyOn(Contrail.prototype, "handler") + .mockReturnValue(async () => Response.json({ ok: true })); + try { + const waitUntil = vi.fn(); + const ctx = { + waitUntil, + passThroughOnException: vi.fn(), + } as unknown as ExecutionContext; + const response = await worker.fetch( + new Request("http://localhost/xrpc/com.example.notifyOfUpdate", { + method: "POST", + body: JSON.stringify({ uri: "at://did:plc:alice/community.lexicon.calendar.event/one" }), + }), + env, + ctx, + ); + expect(response.status).toBe(200); + expect(waitUntil).toHaveBeenCalledTimes(1); + await waitUntil.mock.calls[0][0]; + expect(delivered).toHaveBeenCalledTimes(1); + } finally { + handler.mockRestore(); + } + }); + it("scheduled handler runs live ingest then a bounded backfill retry slice", async () => { const order: string[] = []; const ingest = vi -- 2.51.2 From 9820a04fc2272e5deb455007a8124145eb49200e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:37:51 +0200 Subject: [PATCH 06/34] Add atmo Meilisearch reference consumer --- .changeset/durable-projection-log.md | 2 +- apps/atmo-rsvp/README.md | 16 + apps/atmo-rsvp/src/contrail.config.ts | 17 + apps/atmo-rsvp/src/meilisearch.ts | 446 ++++++++++++++++++ apps/atmo-rsvp/src/search-worker.ts | 18 + apps/atmo-rsvp/tests/meilisearch.test.ts | 274 +++++++++++ packages/contrail/src/core/changes.ts | 5 + .../contrail/tests/change-bootstrap.test.ts | 1 + 8 files changed, 778 insertions(+), 1 deletion(-) create mode 100644 apps/atmo-rsvp/src/meilisearch.ts create mode 100644 apps/atmo-rsvp/src/search-worker.ts create mode 100644 apps/atmo-rsvp/tests/meilisearch.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 619eb38..08d7e18 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/apps/atmo-rsvp/README.md b/apps/atmo-rsvp/README.md index fbc10e8..bdb8cf5 100644 --- a/apps/atmo-rsvp/README.md +++ b/apps/atmo-rsvp/README.md @@ -45,6 +45,22 @@ Event, RSVP, and feed reads can hydrate indexed actor profiles. Follows are an i The service has no user sessions and never signs or publishes records. Applications authenticate users and write through their PDSes. +## Meilisearch candidate generation + +`src/meilisearch.ts` is the application-owned reference consumer for the transactional Contrail change log. It normalizes event documents, excludes cancelled events, restores direct geo/FSQ coordinates (with an injection seam for geocode-cache/H3 lookup), performs idempotent upsert/delete batches, and acknowledges only after every Meilisearch task reports `succeeded`. + +`searchGenerationConfig` and `src/search-worker.ts` are intentionally separate from the active Worker configuration. Use them only with a fresh D1 generation and these runtime secrets: + +```text +MEILI_URL +MEILI_KEY +MEILI_EVENTS_INDEX # optional; defaults to atmo_events +``` + +Current-state bootstrap writes a token-scoped candidate index, catches up a fixed retained tail, then swaps it into the stable UID. A reserved generation marker makes activation idempotent even if the Worker dies after Meilisearch succeeds but before Contrail acknowledges activation. Search clients must filter `kind = event`, which excludes that control marker. The previous stable contents remain under the candidate UID for rollback handling. + +Do not point the candidate config at the active populated D1 database: first-time logging fails closed there by design. Build/import a fresh candidate database, drain and verify the required `search` consumer, then switch the Worker/D1/index tuple together. + ## Development Build the workspace package before running the app directly: diff --git a/apps/atmo-rsvp/src/contrail.config.ts b/apps/atmo-rsvp/src/contrail.config.ts index b907d21..37d894e 100644 --- a/apps/atmo-rsvp/src/contrail.config.ts +++ b/apps/atmo-rsvp/src/contrail.config.ts @@ -72,3 +72,20 @@ export const config: ContrailConfig = { }, }, }; + +/** Candidate-generation configuration for the Meilisearch reference consumer. + * The active Worker keeps `config` until a fresh D1 + candidate index are built + * and activated together. */ +export const searchGenerationConfig: ContrailConfig = { + ...config, + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "current", + requiredForActivation: true, + }, + }, + }, +}; diff --git a/apps/atmo-rsvp/src/meilisearch.ts b/apps/atmo-rsvp/src/meilisearch.ts new file mode 100644 index 0000000..d483926 --- /dev/null +++ b/apps/atmo-rsvp/src/meilisearch.ts @@ -0,0 +1,446 @@ +import type { + ActivationDelivery, + CurrentRecord, + DeliveryBatch, + DeliveryHandlers, + CurrentBootstrapRuntimeHandlers, + SnapshotDeliveryPage, +} from "@atmo-dev/contrail"; + +export const EVENT_COLLECTION = "community.lexicon.calendar.event"; +export const SEARCH_CONSUMER_ID = "search"; +const CONTROL_ID = "__contrail_generation__"; +const EVENT_KIND = "event"; +const CONTROL_KIND = "contrail-control"; + +export interface AtmoMeilisearchEnv { + MEILI_URL: string; + MEILI_KEY: string; + /** Stable serving index UID. Default: `atmo_events`. */ + MEILI_EVENTS_INDEX?: string; +} + +export interface GeoPoint { + lat: number; + lng: number; +} + +export interface AtmoEventDocument extends Record { + id: string; + kind: typeof EVENT_KIND; + uri: string; + did: string; + rkey: string; + cid: string | null; + name: string; + description?: string; + mode?: string; + status?: string; + startsAt?: string; + endsAt?: string; + createdAt?: string; + locations?: unknown[]; + _geo?: GeoPoint; +} + +export interface AtmoMeilisearchOptions { + fetch?: typeof fetch; + taskPollMs?: number; + maxTaskPolls?: number; + /** Application-owned geocode cache/H3 seam. Direct geo and FSQ coordinates + * are extracted before this fallback is called. */ + resolveGeo?: ( + record: CurrentRecord, + env: Env, + signal: AbortSignal, + ) => Promise; + /** Default excludes explicitly cancelled events. */ + isDiscoverable?: (record: CurrentRecord) => boolean; +} + +interface MeiliTaskResponse { + taskUid?: number; + uid?: number; + status?: string; + error?: { code?: string; message?: string }; +} + +function base64Url(value: string): string { + const bytes = new TextEncoder().encode(value); + let binary = ""; + for (let index = 0; index < bytes.length; index += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000)); + } + return btoa(binary) + .replaceAll("+", "-") + .replaceAll("/", "_") + .replace(/=+$/, ""); +} + +export function eventDocumentId(uri: string): string { + return base64Url(uri); +} + +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function string(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function coordinate(value: unknown, minimum: number, maximum: number): number | null { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : null; +} + +function directGeo(value: unknown): GeoPoint | null { + if (!object(value)) return null; + const type = string(value.$type); + if ( + type !== "community.lexicon.location.geo" && + type !== "community.lexicon.location.fsq" + ) { + return null; + } + const lat = coordinate(value.latitude, -90, 90); + const lng = coordinate(value.longitude, -180, 180); + return lat === null || lng === null ? null : { lat, lng }; +} + +export function eventGeo(value: unknown): GeoPoint | null { + if (!object(value) || !Array.isArray(value.locations)) return null; + for (const location of value.locations) { + const geo = directGeo(location); + if (geo) return geo; + } + return null; +} + +export function defaultEventDiscoverability(record: CurrentRecord): boolean { + if (!object(record.value) || typeof record.value.name !== "string") return false; + return record.value.status !== "community.lexicon.calendar.event#cancelled"; +} + +export function eventDocument( + record: CurrentRecord, + geo: GeoPoint | null = eventGeo(record.value), +): AtmoEventDocument { + if (!object(record.value) || typeof record.value.name !== "string") { + throw new Error(`Event ${record.uri} has no valid name`); + } + const value = record.value; + const name = value.name as string; + return { + id: eventDocumentId(record.uri), + kind: EVENT_KIND, + uri: record.uri, + did: record.did, + rkey: record.rkey, + cid: record.cid, + name, + ...(string(value.description) ? { description: string(value.description) } : {}), + ...(string(value.mode) ? { mode: string(value.mode) } : {}), + ...(string(value.status) ? { status: string(value.status) } : {}), + ...(string(value.startsAt) ? { startsAt: string(value.startsAt) } : {}), + ...(string(value.endsAt) ? { endsAt: string(value.endsAt) } : {}), + ...(string(value.createdAt) ? { createdAt: string(value.createdAt) } : {}), + ...(Array.isArray(value.locations) ? { locations: value.locations } : {}), + ...(geo ? { _geo: geo } : {}), + }; +} + +function endpoint(env: AtmoMeilisearchEnv): URL { + if (typeof env.MEILI_KEY !== "string" || env.MEILI_KEY.length === 0) { + throw new Error("MEILI_KEY is required"); + } + if (typeof env.MEILI_URL !== "string" || env.MEILI_URL.length === 0) { + throw new Error("MEILI_URL is required"); + } + const url = new URL(env.MEILI_URL); + if (url.username || url.password || url.search || url.hash) { + throw new Error("MEILI_URL cannot contain credentials, query, or fragment"); + } + return url; +} + +function stableIndex(env: AtmoMeilisearchEnv): string { + return env.MEILI_EVENTS_INDEX?.trim() || "atmo_events"; +} + +export function candidateIndex(stable: string, token: string): string { + const suffix = token.replace(/[^a-zA-Z0-9_-]/g, "_"); + if (!suffix || suffix.length > 128) throw new Error("Invalid bootstrap token"); + return `${stable}__candidate__${suffix}`; +} + +class MeiliClient { + private readonly opened = new Set(); + + constructor( + private readonly env: AtmoMeilisearchEnv, + private readonly request: typeof fetch, + private readonly pollMs: number, + private readonly maxPolls: number, + ) {} + + private async fetch( + path: string, + init: RequestInit & { signal: AbortSignal }, + ): Promise { + const url = new URL(path, endpoint(this.env)); + return this.request(url, { + ...init, + headers: { + authorization: `Bearer ${this.env.MEILI_KEY}`, + "content-type": "application/json", + ...init.headers, + }, + }); + } + + private async json(response: Response, label: string): Promise { + if (!response.ok) { + throw new Error(`${label} failed with HTTP ${response.status}`); + } + const text = await response.text(); + if (text.length > 64_000) throw new Error(`${label} response is too large`); + try { + return text ? JSON.parse(text) : {}; + } catch { + throw new Error(`${label} returned malformed JSON`); + } + } + + private async task(response: Response, signal: AbortSignal): Promise { + const accepted = (await this.json(response, "Meilisearch mutation")) as MeiliTaskResponse; + const uid = accepted.taskUid ?? accepted.uid; + if (!Number.isSafeInteger(uid)) { + throw new Error("Meilisearch mutation returned no task UID"); + } + for (let poll = 0; poll < this.maxPolls; poll++) { + if (signal.aborted) throw signal.reason ?? new Error("Meilisearch task cancelled"); + const task = (await this.json( + await this.fetch(`/tasks/${uid}`, { method: "GET", signal }), + "Meilisearch task", + )) as MeiliTaskResponse; + if (task.status === "succeeded") return; + if (task.status === "failed" || task.status === "canceled") { + throw new Error( + `Meilisearch task ${uid} ${task.status}: ${task.error?.code ?? "unknown"}`, + ); + } + await new Promise((resolve, reject) => { + const timer = setTimeout(done, this.pollMs); + const abort = () => { + clearTimeout(timer); + reject(signal.reason ?? new Error("Meilisearch task cancelled")); + }; + function done() { + signal.removeEventListener("abort", abort); + resolve(); + } + signal.addEventListener("abort", abort, { once: true }); + }); + } + throw new Error("Meilisearch task did not finish within its polling budget"); + } + + async ensureIndex(index: string, token: string | null, signal: AbortSignal): Promise { + const cacheKey = `${index}:${token ?? "stable"}`; + if (this.opened.has(cacheKey)) return; + const indexResponse = await this.fetch(`/indexes/${encodeURIComponent(index)}`, { + method: "GET", + signal, + }); + if (indexResponse.status === 404) { + await this.task( + await this.fetch("/indexes", { + method: "POST", + body: JSON.stringify({ uid: index, primaryKey: "id" }), + signal, + }), + signal, + ); + } else if (!indexResponse.ok) { + throw new Error(`Meilisearch index lookup failed with HTTP ${indexResponse.status}`); + } + await this.task( + await this.fetch(`/indexes/${encodeURIComponent(index)}/settings`, { + method: "PATCH", + body: JSON.stringify({ + filterableAttributes: ["kind", "did", "mode", "status", "startsAt", "endsAt"], + sortableAttributes: ["startsAt", "endsAt", "createdAt"], + searchableAttributes: ["name", "description", "locations"], + }), + signal, + }), + signal, + ); + if (token) { + await this.upsert( + index, + [{ id: CONTROL_ID, kind: CONTROL_KIND, generation: token }], + signal, + ); + } + this.opened.add(cacheKey); + } + + async upsert(index: string, documents: unknown[], signal: AbortSignal): Promise { + if (documents.length === 0) return; + await this.task( + await this.fetch( + `/indexes/${encodeURIComponent(index)}/documents?primaryKey=id`, + { method: "POST", body: JSON.stringify(documents), signal }, + ), + signal, + ); + } + + async delete(index: string, ids: string[], signal: AbortSignal): Promise { + if (ids.length === 0) return; + await this.task( + await this.fetch(`/indexes/${encodeURIComponent(index)}/documents/delete-batch`, { + method: "POST", + body: JSON.stringify(ids), + signal, + }), + signal, + ); + } + + async marker(index: string, signal: AbortSignal): Promise { + const response = await this.fetch( + `/indexes/${encodeURIComponent(index)}/documents/${encodeURIComponent(CONTROL_ID)}`, + { method: "GET", signal }, + ); + if (response.status === 404) return null; + const value = await this.json(response, "Meilisearch generation marker"); + return typeof value.generation === "string" ? value.generation : null; + } + + async activate(stable: string, candidate: string, token: string, signal: AbortSignal): Promise { + await this.ensureIndex(candidate, token, signal); + if ((await this.marker(stable, signal)) === token) return; + await this.ensureIndex(stable, "contrail-empty", signal); + if ((await this.marker(stable, signal)) === token) return; + if ((await this.marker(candidate, signal)) !== token) { + throw new Error("Meilisearch candidate has the wrong generation marker"); + } + await this.task( + await this.fetch("/swap-indexes", { + method: "POST", + body: JSON.stringify([{ indexes: [stable, candidate] }]), + signal, + }), + signal, + ); + if ((await this.marker(stable, signal)) !== token) { + throw new Error("Meilisearch activation did not expose the candidate generation"); + } + } +} + +async function documentsFor( + records: CurrentRecord[], + env: Env, + signal: AbortSignal, + options: AtmoMeilisearchOptions, +): Promise<{ documents: AtmoEventDocument[]; hidden: string[] }> { + const discoverable = options.isDiscoverable ?? defaultEventDiscoverability; + const documents: AtmoEventDocument[] = []; + const hidden: string[] = []; + for (const record of records) { + if (!discoverable(record)) { + hidden.push(eventDocumentId(record.uri)); + continue; + } + const geo = eventGeo(record.value) ?? + (options.resolveGeo ? await options.resolveGeo(record, env, signal) : null); + documents.push(eventDocument(record, geo)); + } + return { documents, hidden }; +} + +export function createAtmoMeilisearchRuntime( + options: AtmoMeilisearchOptions = {}, +): { + deliveries: DeliveryHandlers; + changeBootstraps: CurrentBootstrapRuntimeHandlers; +} { + const clients = new WeakMap(); + const client = (env: Env) => { + const key = env as object; + let value = clients.get(key); + if (!value) { + value = new MeiliClient( + env, + options.fetch ?? fetch, + options.taskPollMs ?? 100, + options.maxTaskPolls ?? 300, + ); + clients.set(key, value); + } + return value; + }; + + const deliver = async ( + batch: DeliveryBatch, + env: Env, + signal: AbortSignal, + ) => { + const meili = client(env); + const stable = stableIndex(env); + const index = batch.destinationToken + ? candidateIndex(stable, batch.destinationToken) + : stable; + await meili.ensureIndex(index, batch.destinationToken ?? null, signal); + const { documents, hidden } = await documentsFor( + batch.currentRecords, + env, + signal, + options, + ); + const absent = batch.absentUris.map(eventDocumentId); + await meili.upsert(index, documents, signal); + await meili.delete(index, [...new Set([...hidden, ...absent])], signal); + }; + + return { + deliveries: { + [SEARCH_CONSUMER_ID]: async (batch, { env, signal }) => { + await deliver(batch, env, signal); + }, + }, + changeBootstraps: { + [SEARCH_CONSUMER_ID]: { + snapshot: async (page: SnapshotDeliveryPage, { env, signal }) => { + const meili = client(env); + const stable = stableIndex(env); + const index = candidateIndex(stable, page.bootstrapToken); + await meili.ensureIndex(index, page.bootstrapToken, signal); + const { documents, hidden } = await documentsFor( + page.records, + env, + signal, + options, + ); + await meili.upsert(index, documents, signal); + await meili.delete(index, hidden, signal); + }, + activate: async (activation: ActivationDelivery, { env, signal }) => { + const stable = stableIndex(env); + await client(env).activate( + stable, + candidateIndex(stable, activation.bootstrapToken), + activation.bootstrapToken, + signal, + ); + }, + }, + }, + }; +} diff --git a/apps/atmo-rsvp/src/search-worker.ts b/apps/atmo-rsvp/src/search-worker.ts new file mode 100644 index 0000000..65bb714 --- /dev/null +++ b/apps/atmo-rsvp/src/search-worker.ts @@ -0,0 +1,18 @@ +/** Candidate-generation Worker entrypoint. Switch wrangler `main` to this file + * only while activating a fresh D1 + Meilisearch index generation. */ +import { createWorker } from "@atmo-dev/contrail/worker"; +import { lexicons } from "../lexicons/generated"; +import { searchGenerationConfig } from "./contrail.config"; +import { + createAtmoMeilisearchRuntime, + type AtmoMeilisearchEnv, +} from "./meilisearch"; + +type Env = AtmoMeilisearchEnv & Record; +const search = createAtmoMeilisearchRuntime(); + +export default createWorker(searchGenerationConfig, { + lexicons, + publicService: { endpoint: "https://api.atmo.rsvp" }, + ...search, +}); diff --git a/apps/atmo-rsvp/tests/meilisearch.test.ts b/apps/atmo-rsvp/tests/meilisearch.test.ts new file mode 100644 index 0000000..e4f8f0f --- /dev/null +++ b/apps/atmo-rsvp/tests/meilisearch.test.ts @@ -0,0 +1,274 @@ +import { describe, expect, it } from "vitest"; +import type { + ActivationDelivery, + CurrentRecord, + DeliveryBatch, + SnapshotDeliveryPage, +} from "@atmo-dev/contrail"; +import { + candidateIndex, + createAtmoMeilisearchRuntime, + defaultEventDiscoverability, + eventDocument, + eventDocumentId, + eventGeo, + type AtmoMeilisearchEnv, +} from "../src/meilisearch"; + +class FakeMeili { + indexes = new Map>(); + tasks = new Map(); + nextTask = 1; + failNext = false; + swaps = 0; + + task(apply?: () => void): Response { + const uid = this.nextTask++; + const status = this.failNext ? "failed" : "succeeded"; + this.failNext = false; + this.tasks.set(uid, status); + if (status === "succeeded") apply?.(); + return Response.json({ taskUid: uid }, { status: 202 }); + } + + fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = new URL(String(input)); + const method = init?.method ?? "GET"; + const parts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent); + + if (parts[0] === "tasks" && method === "GET") { + const status = this.tasks.get(Number(parts[1])); + return status + ? Response.json({ + uid: Number(parts[1]), + status, + ...(status === "failed" ? { error: { code: "fake_failure" } } : {}), + }) + : new Response(null, { status: 404 }); + } + if (url.pathname === "/indexes" && method === "POST") { + const body = JSON.parse(String(init?.body)); + return this.task(() => this.indexes.set(body.uid, new Map())); + } + if (url.pathname === "/swap-indexes" && method === "POST") { + const [{ indexes }] = JSON.parse(String(init?.body)); + return this.task(() => { + const first = this.indexes.get(indexes[0]) ?? new Map(); + const second = this.indexes.get(indexes[1]) ?? new Map(); + this.indexes.set(indexes[0], second); + this.indexes.set(indexes[1], first); + this.swaps++; + }); + } + if (parts[0] !== "indexes" || !parts[1]) { + return new Response(null, { status: 404 }); + } + const index = parts[1]; + if (parts.length === 2 && method === "GET") { + return this.indexes.has(index) + ? Response.json({ uid: index, primaryKey: "id" }) + : new Response(null, { status: 404 }); + } + if (parts[2] === "settings" && method === "PATCH") { + return this.task(); + } + if (parts[2] === "documents" && parts.length === 3 && method === "POST") { + const documents = JSON.parse(String(init?.body)); + return this.task(() => { + const target = this.indexes.get(index)!; + for (const document of documents) target.set(document.id, document); + }); + } + if ( + parts[2] === "documents" && + parts[3] === "delete-batch" && + method === "POST" + ) { + const ids = JSON.parse(String(init?.body)); + return this.task(() => { + const target = this.indexes.get(index)!; + for (const id of ids) target.delete(id); + }); + } + if (parts[2] === "documents" && parts[3] && method === "GET") { + const document = this.indexes.get(index)?.get(parts[3]); + return document + ? Response.json(document) + : new Response(null, { status: 404 }); + } + return new Response(null, { status: 404 }); + }; +} + +const env: AtmoMeilisearchEnv = { + MEILI_URL: "https://meili.example/", + MEILI_KEY: "secret", + MEILI_EVENTS_INDEX: "events", +}; + +function record(options: { + uri?: string; + name?: string; + status?: string; + locations?: unknown[]; +} = {}): CurrentRecord { + const uri = options.uri ?? "at://did:plc:alice/community.lexicon.calendar.event/one"; + return { + uri, + did: "did:plc:alice", + collection: "community.lexicon.calendar.event", + rkey: uri.split("/").at(-1)!, + cid: "cid-one", + value: { + name: options.name ?? "One", + createdAt: "2026-01-01T00:00:00Z", + startsAt: "2026-02-01T00:00:00Z", + ...(options.status ? { status: options.status } : {}), + ...(options.locations ? { locations: options.locations } : {}), + }, + timeUs: 1, + indexedAt: 2, + }; +} + +const context = { + env, + signal: new AbortController().signal, + attempt: 1, +}; + +describe("atmo Meilisearch reference consumer", () => { + it("normalizes stable document IDs, discoverability, and direct geo", () => { + const current = record({ + locations: [ + { + $type: "community.lexicon.location.geo", + latitude: "52.52", + longitude: "13.405", + }, + ], + }); + expect(eventDocumentId(current.uri)).toMatch(/^[a-zA-Z0-9_-]+$/); + expect(eventGeo(current.value)).toEqual({ lat: 52.52, lng: 13.405 }); + expect(eventDocument(current)).toMatchObject({ + id: eventDocumentId(current.uri), + kind: "event", + name: "One", + _geo: { lat: 52.52, lng: 13.405 }, + }); + expect( + defaultEventDiscoverability( + record({ + status: "community.lexicon.calendar.event#cancelled", + }), + ), + ).toBe(false); + }); + + it("writes snapshot/tail state, removes hidden and absent rows, and activates idempotently", async () => { + const fake = new FakeMeili(); + const runtime = createAtmoMeilisearchRuntime({ + fetch: fake.fetch as typeof fetch, + taskPollMs: 1, + maxTaskPolls: 3, + }); + const token = "generation-one"; + const snapshot: SnapshotDeliveryPage = { + kind: "snapshot", + consumerId: "search", + generation: "log-generation", + bootstrapToken: token, + collection: "community.lexicon.calendar.event", + fromUri: null, + throughUri: record().uri, + pageId: "page-one", + records: [record()], + attempt: 1, + leaseExpiresAt: Date.now() + 1_000, + }; + await runtime.changeBootstraps.search!.snapshot(snapshot, context); + const candidate = candidateIndex("events", token); + expect(fake.indexes.get(candidate)?.get(eventDocumentId(record().uri))).toMatchObject({ + name: "One", + }); + + const hidden = record({ + uri: "at://did:plc:alice/community.lexicon.calendar.event/hidden", + status: "community.lexicon.calendar.event#cancelled", + }); + fake.indexes.get(candidate)!.set(eventDocumentId(hidden.uri), { + id: eventDocumentId(hidden.uri), + kind: "event", + }); + const absentUri = "at://did:plc:alice/community.lexicon.calendar.event/absent"; + fake.indexes.get(candidate)!.set(eventDocumentId(absentUri), { + id: eventDocumentId(absentUri), + kind: "event", + }); + const batch: DeliveryBatch = { + consumerId: "search", + cursor: { generation: "log-generation", from: "0", through: "1" }, + changes: [], + currentRecords: [hidden], + absentUris: [absentUri], + destinationToken: token, + }; + await runtime.deliveries.search!(batch, context); + expect(fake.indexes.get(candidate)?.has(eventDocumentId(hidden.uri))).toBe(false); + expect(fake.indexes.get(candidate)?.has(eventDocumentId(absentUri))).toBe(false); + + const activation: ActivationDelivery = { + kind: "activation", + consumerId: "search", + generation: "log-generation", + bootstrapToken: token, + target: "1", + attempt: 1, + leaseExpiresAt: Date.now() + 1_000, + }; + await runtime.changeBootstraps.search!.activate(activation, context); + await runtime.changeBootstraps.search!.activate(activation, context); + expect(fake.swaps).toBe(1); + expect(fake.indexes.get("events")?.get("__contrail_generation__")).toMatchObject({ + generation: token, + }); + }); + + it("waits for task success and rejects a failed asynchronous task", async () => { + const fake = new FakeMeili(); + const runtime = createAtmoMeilisearchRuntime({ + fetch: fake.fetch as typeof fetch, + taskPollMs: 1, + maxTaskPolls: 3, + }); + const token = "generation-failure"; + const snapshot: SnapshotDeliveryPage = { + kind: "snapshot", + consumerId: "search", + generation: "log-generation", + bootstrapToken: token, + collection: "community.lexicon.calendar.event", + fromUri: null, + throughUri: record().uri, + pageId: "page-one", + records: [], + attempt: 1, + leaseExpiresAt: Date.now() + 1_000, + }; + await runtime.changeBootstraps.search!.snapshot(snapshot, context); + fake.failNext = true; + await expect( + runtime.deliveries.search!( + { + consumerId: "search", + cursor: { generation: "log-generation", from: "0", through: "1" }, + changes: [], + currentRecords: [record()], + absentUris: [], + destinationToken: token, + }, + context, + ), + ).rejects.toThrow("fake_failure"); + }); +}); diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 4875240..636684f 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -94,6 +94,8 @@ export interface DeliveryBatch { changes: RecordChange[]; currentRecords: CurrentRecord[]; absentUris: string[]; + /** Generation-scoped destination token during current bootstrap catch-up. */ + destinationToken?: string; } export interface ChangeFailure { @@ -736,6 +738,9 @@ export async function hydrateChanges( changes: claim.changes, currentRecords, absentUris, + ...(claim.bootstrapToken + ? { destinationToken: claim.bootstrapToken } + : {}), }; } diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index 9571bfe..435ede2 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -168,6 +168,7 @@ describe("current-state change consumer bootstrap", () => { bootstrapToken: status.token, }); const delivery = await hydrateChanges(db, withSearch, tail!); + expect(delivery.destinationToken).toBe(status.token); expect(delivery.currentRecords.map((record) => record.rkey).sort()).toEqual([ "a", "c", -- 2.51.2 From add067ae764f95d605335a1b6698e6b2b3d53d85 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:42:06 +0200 Subject: [PATCH 07/34] Stabilize change log operations and upgrades --- apps/atmo-rsvp/tests/contract.test.ts | 23 +++++++++- packages/contrail/README.md | 4 +- packages/contrail/src/cli/commands/changes.ts | 9 +++- packages/contrail/src/core/change-log.ts | 39 ++++++++++++++++ packages/contrail/src/index.ts | 7 ++- packages/contrail/tests/change-log.test.ts | 45 +++++++++++++++++++ 6 files changed, 123 insertions(+), 4 deletions(-) diff --git a/apps/atmo-rsvp/tests/contract.test.ts b/apps/atmo-rsvp/tests/contract.test.ts index bfea7e3..9f0030f 100644 --- a/apps/atmo-rsvp/tests/contract.test.ts +++ b/apps/atmo-rsvp/tests/contract.test.ts @@ -3,7 +3,11 @@ import { describePublicService } from "@atmo-dev/contrail"; import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createWorker } from "@atmo-dev/contrail/worker"; import { lexicons } from "../lexicons/generated"; -import { config } from "../src/contrail.config"; +import { config, searchGenerationConfig } from "../src/contrail.config"; +import { + createAtmoMeilisearchRuntime, + type AtmoMeilisearchEnv, +} from "../src/meilisearch"; const EXPECTED_METHODS = [ "rsvp.atmo.event.getRecord", @@ -71,6 +75,23 @@ describe("api.atmo.rsvp public contract", () => { publicService: { endpoint: "https://api.atmo.rsvp" }, }), ).not.toThrow(); + expect(searchGenerationConfig.changes?.consumers.search).toEqual({ + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "current", + requiredForActivation: true, + }); + const search = createAtmoMeilisearchRuntime(); + expect(() => + createWorker>( + searchGenerationConfig, + { + lexicons, + publicService: { endpoint: "https://api.atmo.rsvp" }, + ...search, + }, + ), + ).not.toThrow(); }); it("publishes its service DID and permits browser auth headers", async () => { diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 6eac71f..22ac887 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -151,7 +151,9 @@ Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and r `initial: "current"` uses a durable snapshot-plus-tail coordinator. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. -A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. +A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Status includes a conservative projection/head/batch/ack write plan. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. + +Backups retain the database's change-log generation ID, positions, leases, bootstrap token, and consumer progress. Restoring that backup as the same generation is resumable and may redeliver an in-flight lease after expiry. Do not clone it as a new deployment generation while reusing destination cursors: build a fresh projection database and run current-state bootstrap. Restoring projection tables without the matching change tables is an unrecoverable delivery gap and must enter reset/bootstrap rather than silently continuing numeric positions. ## Local development diff --git a/packages/contrail/src/cli/commands/changes.ts b/packages/contrail/src/cli/commands/changes.ts index 1dfd205..0d608d0 100644 --- a/packages/contrail/src/cli/commands/changes.ts +++ b/packages/contrail/src/cli/commands/changes.ts @@ -1,5 +1,6 @@ import type { CAC } from "cac"; import { Contrail } from "../../contrail.js"; +import { getChangeLogCostPlan } from "../../core/change-log.js"; import type { Database } from "../../core/types.js"; import { resolveAndLoadConfig, @@ -143,8 +144,9 @@ export function registerChanges(cli: CAC): void { } const status = await contrail.changes.status(db); + const cost = getChangeLogCostPlan(contrail.config); if (commandOptions.json) { - console.log(JSON.stringify(status, null, 2)); + console.log(JSON.stringify({ ...status, cost }, null, 2)); return; } if (!status.enabled || !status.state) { @@ -156,6 +158,11 @@ export function registerChanges(cli: CAC): void { `floor=${status.state.retainedFloor} rows=${status.rows} ` + `changes=${status.changes} bytes=${status.bytes}`, ); + console.log( + ` write plan: projection=${cost.projectionStateWrites} ` + + `head=${cost.changeHeadWrites} batch=${cost.changeBatchWrites} ` + + `ack=${cost.acknowledgementWrites}`, + ); for (const item of status.consumers) { const retry = item.nextAttemptAt === null diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index b50a551..7db6713 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -16,6 +16,45 @@ import { export const MAX_CHANGE_BATCH_CHANGES = 500; export const MAX_CHANGE_BATCH_BYTES = 512_000; +export interface ChangeLogCostPlan { + enabled: boolean; + consumers: number; + coveragePairs: number; + projectionStateWrites: number; + changeHeadWrites: number; + changeBatchWrites: number; + acknowledgementWrites: number; + /** Total expected rows written by one relevant projection transaction. */ + relevantProjectionWrites: number; +} + +/** Conservative write-amplification report for one bounded projection batch. */ +export function getChangeLogCostPlan( + config: ContrailConfig, + mutationUris = 50, +): ChangeLogCostPlan { + if (!Number.isSafeInteger(mutationUris) || mutationUris < 1 || mutationUris > 500) { + throw new TypeError("mutationUris must be an integer between 1 and 500"); + } + const enabled = changesEnabled(config); + // One serialized revision update plus one predecessor-check update per forty + // unique URIs. The idempotent singleton INSERT normally writes no row. + const projectionStateWrites = 1 + Math.ceil(mutationUris / 40); + const changeHeadWrites = enabled ? 1 : 0; + const changeBatchWrites = enabled ? 1 : 0; + return { + enabled, + consumers: Object.keys(config.changes?.consumers ?? {}).length, + coveragePairs: changeLogCoverage(config).length, + projectionStateWrites, + changeHeadWrites, + changeBatchWrites, + acknowledgementWrites: enabled ? 1 : 0, + relevantProjectionWrites: + projectionStateWrites + changeHeadWrites + changeBatchWrites, + }; +} + export interface RecordChange { id: string; kind: "record"; diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index dfd17b6..cd07755 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -30,11 +30,16 @@ export * from "./core/backfill"; export * from "./core/status"; export * from "./core/diagnostics"; export { + getChangeLogCostPlan, getChangeLogState, MAX_CHANGE_BATCH_BYTES, MAX_CHANGE_BATCH_CHANGES, } from "./core/change-log"; -export type { ChangeLogState, RecordChange } from "./core/change-log"; +export type { + ChangeLogCostPlan, + ChangeLogState, + RecordChange, +} from "./core/change-log"; export * from "./core/changes"; export * from "./core/change-bootstrap"; export * from "./core/delivery"; diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index 00ae632..bb5a3c3 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { Contrail, createIngestEvent, + getChangeLogCostPlan, getChangeLogState, ingestRecords, initSchema, @@ -146,6 +147,16 @@ describe("transactional projection change log", () => { it("has no change-log schema or writes when disabled", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config(); + expect(getChangeLogCostPlan(resolved, 50)).toEqual({ + enabled: false, + consumers: 0, + coveragePairs: 0, + projectionStateWrites: 3, + changeHeadWrites: 0, + changeBatchWrites: 0, + acknowledgementWrites: 0, + relevantProjectionWrites: 3, + }); await initSchema(db, resolved); const tables = await db @@ -169,6 +180,16 @@ describe("transactional projection change log", () => { it("initializes a fresh generation, registrations, and coverage ledger", async () => { const db = createSqliteDatabase(":memory:"); const resolved = loggedConfig(); + expect(getChangeLogCostPlan(resolved, 50)).toMatchObject({ + enabled: true, + consumers: 2, + coveragePairs: 2, + projectionStateWrites: 3, + changeHeadWrites: 1, + changeBatchWrites: 1, + acknowledgementWrites: 1, + relevantProjectionWrites: 5, + }); await initSchema(db, resolved); const state = await getChangeLogState(db); @@ -424,6 +445,30 @@ describe("transactional projection change log", () => { ); }); + it("upgrades retained pre-byte-count change batches without resetting consumers", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + await ingestRecords(db, [mutation({ sourceTime: 1 })], resolved); + await db + .prepare("ALTER TABLE change_batches DROP COLUMN encoded_bytes") + .run(); + await db + .prepare( + "UPDATE _contrail_meta SET value = 'old-change-schema' WHERE key = 'schema_fingerprint'", + ) + .run(); + + await initSchema(db, resolved); + const row = await db + .prepare("SELECT encoded_bytes, changes_json FROM change_batches") + .first<{ encoded_bytes: number; changes_json: string }>(); + expect(row?.encoded_bytes).toBe( + new TextEncoder().encode(row!.changes_json).byteLength, + ); + expect((await getChangeLogState(db))?.head).toBe("1"); + }); + it("retries a losing overlapping writer from fresh durable state", async () => { const real = createSqliteDatabase(":memory:"); const resolved = loggedConfig(); -- 2.51.2 From e70b0af5a16c083226888fe64a1d556c33575e41 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:56:30 +0200 Subject: [PATCH 08/34] Fix change bootstrap delivery invariants --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 2 +- packages/contrail/src/core/change-log.ts | 27 ++++++++------- packages/contrail/src/core/changes.ts | 6 ++-- packages/contrail/src/core/types.ts | 21 +++++++++--- .../contrail/tests/change-bootstrap.test.ts | 5 ++- packages/contrail/tests/change-log.test.ts | 34 +++++++++++++++++++ 7 files changed, 74 insertions(+), 23 deletions(-) diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 08d7e18..7318ebc 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Current-state consumers require both projection phases, and candidate destination tokens are scoped strictly to bootstrap deliveries. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 22ac887..947a24d 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -149,7 +149,7 @@ if (claim) { Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and resolve delete/recreate races from newest canonical state. Consumers lease and progress independently; irrelevant position ranges advance without invoking a handler. Delivery is intentionally at least once—a destination success followed by an acknowledgement crash causes duplicate delivery. Handlers must be idempotent by stable record/document key. -`initial: "current"` uses a durable snapshot-plus-tail coordinator. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. +`initial: "current"` uses a durable snapshot-plus-tail coordinator and must observe both `historical` and `live` phases so every mutation racing the keyset scan is retained for reconciliation. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Snapshot and bootstrap-tail deliveries carry the candidate destination token; ordinary deliveries after activation do not. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Status includes a conservative projection/head/batch/ack write plan. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 7db6713..1220a3a 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -363,7 +363,7 @@ export async function initializeChangeLog( const statements: Statement[] = []; for (const [consumerId, consumer] of Object.entries( config.changes?.consumers ?? {}, - ).sort(([left], [right]) => left.localeCompare(right))) { + ).sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))) { const initialReady = consumer.initial === "current" ? "pending" : "ready"; statements.push( db @@ -444,17 +444,17 @@ async function assertChangeLogDefinition( FROM change_consumers ORDER BY consumer_id`, ) .all(); - const expectedConsumers = Object.entries(config.changes?.consumers ?? {}).sort( - ([left], [right]) => left.localeCompare(right), - ); + const expectedConsumers = Object.entries(config.changes?.consumers ?? {}); if (consumers.results.length !== expectedConsumers.length) { throw new Error("Durable change consumer registration is incomplete"); } - for (let index = 0; index < expectedConsumers.length; index++) { - const [id, expected] = expectedConsumers[index]!; - const actual = consumers.results[index]!; + const consumersById = new Map( + consumers.results.map((consumer) => [consumer.consumer_id, consumer]), + ); + for (const [id, expected] of expectedConsumers) { + const actual = consumersById.get(id); if ( - actual.consumer_id !== id || + !actual || actual.generation_id !== state.generation_id || actual.configured_collections_json !== canonicalCollections(expected.collections) || actual.configured_phases_json !== canonicalPhases(changeConsumerPhases(expected)) || @@ -477,13 +477,14 @@ async function assertChangeLogDefinition( if (coverage.results.length !== expectedCoverage.length) { throw new Error("Durable change-log coverage is incomplete"); } - for (let index = 0; index < expectedCoverage.length; index++) { - const actual = coverage.results[index]!; - const expected = expectedCoverage[index]!; + const coverageByPair = new Map( + coverage.results.map((item) => [`${item.collection}\0${item.phase}`, item]), + ); + for (const expected of expectedCoverage) { + const actual = coverageByPair.get(`${expected.collection}\0${expected.phase}`); if ( + !actual || actual.generation_id !== state.generation_id || - actual.collection !== expected.collection || - actual.phase !== expected.phase || Number(actual.from_position) !== 0 || actual.through_position !== null ) { diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 636684f..3627147 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -631,9 +631,9 @@ async function claimChangeRange( changes: [...coalesced.values()], attempt: Number(leased.attempts) + 1, leaseExpiresAt, - ...(leased.bootstrap_token === null - ? {} - : { bootstrapToken: leased.bootstrap_token }), + ...(bootstrap && leased.bootstrap_token !== null + ? { bootstrapToken: leased.bootstrap_token } + : {}), ...(bootstrapTarget === null ? {} : { bootstrapTarget }), leaseOwner: owner, }; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index bb5860e..0544dde 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -258,7 +258,7 @@ export type ChangeConsumerInitialMode = "current" | "future" | "history"; export interface ChangeConsumerConfig { /** Exact configured collection NSIDs. Short aliases are deliberately rejected. */ collections: string[]; - /** Projection phases to observe. Defaults to both historical and live. */ + /** Projection phases to observe. Defaults to both; `initial: "current"` requires both. */ phases?: ProjectionPhase[]; /** How the consumer establishes its first durable position. */ initial: ChangeConsumerInitialMode; @@ -686,6 +686,11 @@ const MAX_CHANGE_CONSUMER_COLLECTIONS = 64; const MAX_CHANGE_COVERAGE_PAIRS = 256; const MAX_CHANGE_DEFINITIONS_BYTES = 64 * 1_024; +/** Locale-independent order for durable definitions compared across runtimes. */ +function compareCanonicalText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + /** Whether this configuration requires the optional transactional change log. */ export function changesEnabled(config: ContrailConfig): boolean { return Object.keys(config.changes?.consumers ?? {}).length > 0; @@ -712,8 +717,8 @@ export function changeLogCoverage( } return [...pairs.values()].sort( (left, right) => - left.collection.localeCompare(right.collection) || - left.phase.localeCompare(right.phase), + compareCanonicalText(left.collection, right.collection) || + compareCanonicalText(left.phase, right.phase), ); } @@ -721,7 +726,7 @@ export function changeLogCoverage( export function canonicalChangeDefinitions(config: ContrailConfig): string { return JSON.stringify( Object.entries(config.changes?.consumers ?? {}) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareCanonicalText(left, right)) .map(([id, consumer]) => ({ id, collections: [...consumer.collections].sort(), @@ -831,6 +836,14 @@ export function validateConfig(config: ContrailConfig): void { `Change consumer "${id}" requires unique historical/live phases`, ); } + if ( + consumer.initial === "current" && + !(phases.includes("historical") && phases.includes("live")) + ) { + throw new Error( + `Current-state change consumer "${id}" must observe both historical and live phases`, + ); + } if (!(["current", "future", "history"] as string[]).includes(consumer.initial)) { throw new Error(`Change consumer "${id}" has an invalid initial mode`); } diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index 435ede2..8a23e1b 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -206,6 +206,9 @@ describe("current-state change consumer bootstrap", () => { await apply(db, withSearch, event({ rkey: "d", time: 6 })); const ordinary = await claimChanges(db, "search", { now: 400 }); expect(ordinary).toMatchObject({ from: "2", through: "3" }); + expect(ordinary).not.toHaveProperty("bootstrapToken"); + const ordinaryDelivery = await hydrateChanges(db, withSearch, ordinary!); + expect(ordinaryDelivery).not.toHaveProperty("destinationToken"); }); it("persists snapshot failure backoff and resumes the same page", async () => { @@ -243,7 +246,7 @@ describe("current-state change consumer bootstrap", () => { await initSchema(db, eventOnly); const expanded = config({ keeper: { collections: [EVENT], phases: ["live"], initial: "history" }, - notes: { collections: [NOTE], phases: ["live"], initial: "current" }, + notes: { collections: [NOTE], initial: "current" }, }); await expect(initSchema(db, expanded)).rejects.toThrow( "expands collection/phase coverage", diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index bb5a3c3..17f3303 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -142,6 +142,21 @@ describe("transactional projection change log", () => { }, }), ).toThrow("unique historical/live phases"); + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + search: { + collections: [EVENT], + phases: ["live"], + initial: "current", + }, + }, + }, + }), + ).toThrow("must observe both historical and live phases"); }); it("has no change-log schema or writes when disabled", async () => { @@ -412,6 +427,25 @@ describe("transactional projection change log", () => { ); }); + it("matches durable consumer IDs independently of locale sort order", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + changes: { + consumers: { + a_: { collections: [EVENT], initial: "future" }, + "a-": { collections: [EVENT], initial: "future" }, + }, + }, + }); + + await initSchema(db, resolved); + await expect(initSchema(db, resolved)).resolves.toBeUndefined(); + const rows = await db + .prepare("SELECT consumer_id FROM change_consumers ORDER BY consumer_id") + .all<{ consumer_id: string }>(); + expect(rows.results.map((row) => row.consumer_id)).toEqual(["a-", "a_"]); + }); + it("fails closed for unsafe enable, disable, and definition changes", async () => { const populated = createSqliteDatabase(":memory:"); const disabled = config(); -- 2.51.2 From 92fedb06edacec6bcea7af0ec303b80b64211d74 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:07:55 +0200 Subject: [PATCH 09/34] scheduled ingestion bounds --- .changeset/bounded-scheduled-ingest.md | 5 + docs/01-indexing.md | 16 +- packages/contrail/README.md | 9 +- packages/contrail/src/contrail.ts | 15 +- packages/contrail/src/core/db/index.ts | 2 +- packages/contrail/src/core/db/records.ts | 41 + packages/contrail/src/core/db/schema.ts | 23 +- packages/contrail/src/core/ingest.ts | 46 +- packages/contrail/src/core/jetstream.ts | 836 +++++++++++++----- packages/contrail/src/core/types.ts | 9 +- packages/contrail/src/worker/index.ts | 5 +- packages/contrail/tests/ingest-hang.test.ts | 279 +++++- .../contrail/tests/scheduled-ingest.test.ts | 323 +++++++ .../tests/schema-fingerprint-gate.test.ts | 20 +- 14 files changed, 1391 insertions(+), 238 deletions(-) create mode 100644 .changeset/bounded-scheduled-ingest.md create mode 100644 packages/contrail/tests/scheduled-ingest.test.ts diff --git a/.changeset/bounded-scheduled-ingest.md b/.changeset/bounded-scheduled-ingest.md new file mode 100644 index 0000000..3cb5619 --- /dev/null +++ b/.changeset/bounded-scheduled-ingest.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": patch +--- + +Bound scheduled Jetstream cycles by retained candidate count and serialized bytes, drop exact transport observations before admission, preserve same-timestamp observations across capped restarts, capture empty initial cursors safely, reject rollback-prone endpoint pools in scheduled mode, and emit one bounded aggregate cycle summary. diff --git a/docs/01-indexing.md b/docs/01-indexing.md index f4117ba..001cae2 100644 --- a/docs/01-indexing.md +++ b/docs/01-indexing.md @@ -106,7 +106,21 @@ async scheduled(_ev, env, ctx) { } ``` -`ingest()` connects to Jetstream, streams events since the saved cursor, stops when caught up. Running every minute is fine — the next fire resumes where this one left off. Each cycle is bounded, so it can't blow past the Worker time limit. +`ingest()` connects to Jetstream, streams events since the saved cursor, and stops at the live edge or the first scheduled collection threshold. Defaults are 25 seconds, 250 retained unique commit candidates, and 4 MiB of serialized candidate data. Byte accounting uses the UTF-8 record body plus a fixed 512-byte event-metadata allowance; the threshold-crossing candidate is retained, while deletes use only the allowance. Exact transport duplicates and source-filtered or identity events do not consume candidate/byte limits, but remain accounted for checkpointing. + +Override the limits when profiling a deployment: + +```ts +await contrail.ingest({ + maxDrainMs: 20_000, + maxCandidates: 200, + maxSerializedBytes: 3 * 1024 * 1024, +}, env.DB); +``` + +Running every minute is fine—the next fire resumes from the last fully accounted cursor. Contrail stores bounded exact-observation hashes at the current microsecond cursor and resumes one microsecond earlier, so a cap can split equal-cursor items without skipping or recounting them. The cycle summary reports its stop reason, work counters, thresholds, safe cursor, and committed database sub-batches. + +Scheduled ingestion requires exactly one pinned Jetstream endpoint. Atcute intentionally rolls pooled connections back ten seconds on each new subscription; in a dense overlap that replay can consume every scheduled cap without moving forward. Use `runPersistent()` for multi-endpoint failover. Persistent ingestion is otherwise unchanged and does not inherit the scheduled count or byte defaults. **Local dev:** wrangler's cron scheduler only runs in deployed production. For local dev use `pnpm contrail dev` — it runs `wrangler dev --test-scheduled`, fires `/__scheduled` on your configured cron interval, and offers to start or resume backfill whenever known work remains. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 8a1e40d..7e042c1 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -61,6 +61,13 @@ HTTP routes expose the same pipeline, including relationship/reference hydration ```ts await contrail.ingest(); // bounded Jetstream cycle +// Optional per-cycle overrides (defaults: 25s, 250 candidates, 4 MiB). +await contrail.ingest({ + maxDrainMs: 20_000, + maxCandidates: 200, + maxSerializedBytes: 3 * 1024 * 1024, +}); + await contrail.runPersistent({ signal: abortController.signal, batchSize: 50, @@ -70,7 +77,7 @@ await contrail.runPersistent({ After a write to a user's PDS, `contrail.notify(uri)` can fetch the authoritative record immediately. Only an authoritative not-found response deletes local state; rate limits, server errors, timeouts, malformed responses, and network failures leave it unchanged. Authentication and abuse controls for the public HTTP operation remain under design. -Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact yielded cursor in the same transaction. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. +Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Scheduled collection drops exact Jetstream observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity and source-filtered observations consume neither candidate nor byte limits, but their yielded cursors remain accounted. Bounded exact-observation hashes at the current microsecond cursor let restarts replay one microsecond of overlap without skipping equal-cursor siblings or recounting earlier ones. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact accounted cursor in the same transaction. Scheduled mode requires one pinned Jetstream endpoint; use `runPersistent()` for an Atcute failover pool. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. ## Local development diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index bba7572..82f03c5 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -22,8 +22,10 @@ import { } from "./core/db/records"; import { createIngestState, + resolveScheduledIngestBudget, runIngestCycle, type IngestState, + type ScheduledIngestOptions, } from "./core/jetstream"; import { backfillPending, @@ -115,16 +117,17 @@ export class Contrail { } /** Run one ingestion cycle: catches up records from Jetstream and — when - * `config.labels` is set — labels from each configured labeler in parallel. - * Both share the same `timeoutMs` budget; they're independent network - * operations so concurrency is free. */ - async ingest(options?: { timeoutMs?: number }, db?: Database): Promise { + * `config.labels` is set — labels from each configured labeler in parallel. + * Scheduled record collection is independently bounded by drain time, + * retained candidates, and serialized bytes. */ + async ingest(options?: ScheduledIngestOptions, db?: Database): Promise { const d = this.getDb(db); + const budget = resolveScheduledIngestBudget(options); const tasks: Promise[] = [ - runIngestCycle(d, this.config, options?.timeoutMs, this._ingestState), + runIngestCycle(d, this.config, budget, this._ingestState), ]; if (this.config.labels) { - tasks.push(runLabelIngestCycle(d, this.config, options?.timeoutMs)); + tasks.push(runLabelIngestCycle(d, this.config, budget.maxDrainMs)); } await Promise.all(tasks); } diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index c772276..770eb3c 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,6 +1,6 @@ export { initSchema, CONTRAIL_SCHEMA_VERSION } from "./schema"; export { getMeta, setMeta, getMetaNumber } from "./meta"; export { optimizeDatabase } from "./optimize"; -export { assertServingSourceCompatibility, getLastCursor, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; +export { assertServingSourceCompatibility, getLastCursor, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult, ServingSourcePosition } from "./records"; export type { RecordSource } from "../types"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index e06cac0..04206ce 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -663,10 +663,50 @@ export function saveCursorStatement( .bind(timeUs); } +/** Exact source observations already accounted for at the current coarse + * cursor. Scheduled Jetstream resumes one microsecond earlier and uses these + * hashes to avoid both skipping same-cursor siblings and recounting prior ones. */ +export async function getCursorObservations( + db: Database, + timeUs: number, +): Promise> { + const rows = await db + .prepare( + "SELECT observation FROM cursor_observations WHERE time_us = ?", + ) + .bind(timeUs) + .all<{ observation: string }>(); + return new Set((rows.results ?? []).map((row) => row.observation)); +} + +/** Statements that atomically retire observations behind the monotonic cursor + * and union observations accounted for at its current timestamp. Inserts are + * conditional so an older concurrent cycle cannot attach hashes to a newer + * checkpoint. */ +export function saveCursorObservationStatements( + db: Database, + timeUs: number, + observations: Iterable, +): Statement[] { + return [ + db.prepare( + "DELETE FROM cursor_observations WHERE time_us < (SELECT time_us FROM cursor WHERE id = 1)", + ), + ...[...new Set(observations)].map((observation) => + db + .prepare( + "INSERT INTO cursor_observations (time_us, observation) SELECT ?, ? WHERE (SELECT time_us FROM cursor WHERE id = 1) = ? ON CONFLICT(time_us, observation) DO NOTHING", + ) + .bind(timeUs, observation, timeUs), + ), + ]; +} + export async function saveCursor( db: Database, timeUs: number, orderedSource?: OrderedSourceConfig, + observations: Iterable = [], ): Promise { const statements = [saveCursorStatement(db, timeUs)]; if (orderedSource) { @@ -674,6 +714,7 @@ export async function saveCursor( saveOrderedSourcePositionStatement(db, orderedSource, timeUs), ); } + statements.push(...saveCursorObservationStatements(db, timeUs, observations)); await db.batch(statements); } diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 95fa42d..77cc78a 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -702,16 +702,32 @@ export async function initSchema( const indexes = buildDynamicIndexes(config, dialect); const feeds = buildFeedTables(config, dialect); const fts = buildFtsTables(config, dialect); - const fingerprint = schemaFingerprint(config, dialect, { + const baseFingerprint = schemaFingerprint(config, dialect, { base, collections, indexes, feeds, fts, }); - - if ((await getMeta(db, SCHEMA_FINGERPRINT_KEY)) === fingerprint) { + const fingerprint = `${baseFingerprint}:cursor-observations-v1`; + const cursorObservationsDdl = `CREATE TABLE IF NOT EXISTS cursor_observations ( + time_us ${dialect.bigintType} NOT NULL, + observation TEXT NOT NULL, + PRIMARY KEY (time_us, observation) + )`; + const storedFingerprint = await getMeta(db, SCHEMA_FINGERPRINT_KEY); + + if (storedFingerprint === fingerprint) { + for (const apply of options.extraSchemas ?? []) await apply(db); + return; + } + if (storedFingerprint === baseFingerprint) { + // This independent additive table must not force an otherwise-current + // deployment through the FTS rebuild path. Upgrade the prior fingerprint + // directly after its idempotent DDL succeeds. + await runIdempotentDdl(db, cursorObservationsDdl); for (const apply of options.extraSchemas ?? []) await apply(db); + await setMeta(db, SCHEMA_FINGERPRINT_KEY, fingerprint); return; } @@ -725,6 +741,7 @@ export async function initSchema( } } + await runIdempotentDdl(db, cursorObservationsDdl); await applyFtsTables(db, config, dialect); const hasFeeds = !!(config.feeds && Object.keys(config.feeds).length > 0); diff --git a/packages/contrail/src/core/ingest.ts b/packages/contrail/src/core/ingest.ts index 48527cf..defbb11 100644 --- a/packages/contrail/src/core/ingest.ts +++ b/packages/contrail/src/core/ingest.ts @@ -3,6 +3,7 @@ import type { ContrailConfig, Database, IngestEvent, + Logger, MutationSource, Statement, } from "./types"; @@ -99,6 +100,13 @@ export function recordTimeUs( return microseconds > fallbackUs ? fallbackUs : microseconds; } +export interface IngestWarningSamples { + /** Fixed maximum number of warning strings retained by the caller. */ + maxSamples: number; + samples: string[]; + omitted: number; +} + export interface IngestRecordsOptions { skipReplayDetection?: boolean; skipFeedFanout?: boolean; @@ -117,6 +125,8 @@ export interface IngestRecordsOptions { /** @internal Aggregate private diagnostics for one bulk run. The caller * flushes this bounded object once after concurrent page processing. */ aggregateDiagnostics?: IngestDiagnosticCounts; + /** Collect bounded warning details instead of logging per-record lines. */ + warningSamples?: IngestWarningSamples; } export interface IngestDropCounts { @@ -140,6 +150,22 @@ export interface IngestRecordsResult { discoveredDids: string[]; } +function emitIngestWarning( + logger: Pick, + samples: IngestWarningSamples | undefined, + message: string, +): void { + if (!samples) { + logger.warn(message); + return; + } + if (samples.samples.length >= samples.maxSamples) { + samples.omitted++; + return; + } + samples.samples.push(message.slice(0, 320)); +} + /** * The single admission and projection path for records from every source. * @@ -178,7 +204,9 @@ export async function ingestRecords( const shortName = resolveCollectionKey(config, event.collection); if (!shortName) { dropped.unknownCollection++; - logger.warn( + emitIngestWarning( + logger, + options.warningSamples, `[ingest] drop unknown collection: ${event.operation} ${event.uri} collection=${event.collection}`, ); continue; @@ -187,7 +215,11 @@ export async function ingestRecords( event.operation === "delete" ? null : parseRecord(event.record); if (event.operation !== "delete" && !record) { dropped.invalidRecord++; - logger.warn(`[ingest] drop invalid record: ${event.uri}`); + emitIngestWarning( + logger, + options.warningSamples, + `[ingest] drop invalid record: ${event.uri}`, + ); continue; } candidates.push({ event, shortName, record }); @@ -249,7 +281,11 @@ export async function ingestRecords( try { keep = filter(record); } catch (error) { - logger.warn(`[ingest] recordFilter threw for ${event.uri}: ${error}`); + emitIngestWarning( + logger, + options.warningSamples, + `[ingest] recordFilter threw for ${event.uri}: ${error}`, + ); } if (!keep) { dropped.recordFilter++; @@ -270,7 +306,9 @@ export async function ingestRecords( dropped.cidEncoding + dropped.missingCid; if (validationDropTotal > 0 && !options.aggregateDiagnostics) { - logger.warn( + emitIngestWarning( + logger, + options.warningSamples, `[ingest] dropped ${validationDropTotal} record(s) during validation ` + `(lexicon=${dropped.lexiconValidation}, cid_mismatch=${dropped.cidMismatch}, ` + `cid_encoding=${dropped.cidEncoding}, missing_cid=${dropped.missingCid})`, diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index b29c8d1..5e953aa 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -1,6 +1,7 @@ import { JetstreamSubscription } from "@atcute/jetstream"; import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; import { + DEFAULT_JETSTREAMS, getCollectionNsids, getDependentNsids, jetstreamUrlOption, @@ -10,12 +11,121 @@ import { optimizeIntervalMs, optimizeAnalysisLimit, } from "./types"; -import { initSchema, getLastCursor, saveCursor, saveCursorStatement, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; -import { createIngestEvent, ingestRecords, recordTimeUs } from "./ingest"; +import { initSchema, getLastCursor, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; +import { + createIngestEvent, + ingestRecords, + recordTimeUs, + type IngestDropCounts, + type IngestWarningSamples, +} from "./ingest"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; const BATCH_SIZE = 50; + +/** Fixed estimate for the URI, source position, revision, CID, and other + * metadata retained beside each serialized record body. Deletes consume only + * this allowance. The byte budget is intentionally an admission threshold: the + * candidate that reaches it is retained, bounding overshoot to one record plus + * this allowance. */ +export const SCHEDULED_INGEST_METADATA_BYTES = 512; + +/** Conservative defaults for one D1/Worker scheduled drain. Persistent + * ingestion has its own streaming lifecycle and does not use these limits. */ +export const DEFAULT_SCHEDULED_INGEST_BUDGET = Object.freeze({ + maxDrainMs: 25_000, + maxCandidates: 250, + maxSerializedBytes: 4 * 1024 * 1024, +}) satisfies ScheduledIngestBudget; + +export interface ScheduledIngestBudget { + maxDrainMs: number; + maxCandidates: number; + maxSerializedBytes: number; +} + +export interface ScheduledIngestOptions { + /** Maximum wall time spent requesting source items. Default: 25 seconds. */ + maxDrainMs?: number; + /** Maximum unique commit candidates retained by one drain. Default: 250. */ + maxCandidates?: number; + /** UTF-8 record bytes plus metadata allowances. Default: 4 MiB. */ + maxSerializedBytes?: number; + /** @deprecated Compatibility alias for maxDrainMs. */ + timeoutMs?: number; +} + +export type ScheduledIngestStopReason = + | "head" + | "idle" + | "count" + | "bytes" + | "drain-time" + | "cancelled"; + +export interface ScheduledIngestCollectionStats { + observedSourceItems: number; + commitObservations: number; + identityObservations: number; + retainedCandidates: number; + exactDuplicatesDropped: number; + cursorBoundaryDuplicatesDropped: number; + resumeOverlapDropped: number; + sourceScopeFiltered: number; + sourceInconsistencies: number; + serializedCandidateBytes: number; + startingCursor: number | null; + lastAccountedCursor: number | null; + safeEndingCursor: number | null; + stopReason: ScheduledIngestStopReason; + connections: number; + connectionCloses: number; + connectionErrors: number; + diagnosticSamples: string[]; + diagnosticSamplesOmitted: number; +} + +function positiveInteger(value: number, label: string): number { + if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) { + throw new TypeError(`${label} must be a positive finite integer`); + } + return value; +} + +/** Resolve and validate every scheduled collection threshold. */ +export function resolveScheduledIngestBudget( + value?: ScheduledIngestOptions | ScheduledIngestBudget | number, +): ScheduledIngestBudget { + if (typeof value === "number") { + return { + ...DEFAULT_SCHEDULED_INGEST_BUDGET, + maxDrainMs: positiveInteger(value, "maxDrainMs"), + }; + } + const options = (value ?? {}) as ScheduledIngestOptions; + if (options.timeoutMs !== undefined) { + positiveInteger(options.timeoutMs, "timeoutMs"); + } + return { + maxDrainMs: positiveInteger( + options.maxDrainMs ?? + options.timeoutMs ?? + DEFAULT_SCHEDULED_INGEST_BUDGET.maxDrainMs, + "maxDrainMs", + ), + maxCandidates: positiveInteger( + options.maxCandidates ?? DEFAULT_SCHEDULED_INGEST_BUDGET.maxCandidates, + "maxCandidates", + ), + maxSerializedBytes: positiveInteger( + options.maxSerializedBytes ?? + DEFAULT_SCHEDULED_INGEST_BUDGET.maxSerializedBytes, + "maxSerializedBytes", + ), + }; +} + /** Distinct actors pruned per ingest tick by the rolling feed sweep. Each * actor costs a handful of index-backed O(cap) deletes, so this bounds the * prune's per-tick CPU regardless of how large feed_items grows. */ @@ -161,17 +271,15 @@ function getLogger(config: ContrailConfig): Logger { const INGEST_TIMEOUT = Symbol("ingest-timeout"); /** Await the iterator's next value, but give up after `ms`. Without this a - * quiet Jetstream (the async iterator blocks forever waiting for an event that - * never arrives) holds the cycle past its safety timeout until the caller's - * hard timeout kills the isolate — so the batch and cursor are never written. */ + * quiet Jetstream can hold a scheduled drain past its deadline. */ function nextWithDeadline( iterator: AsyncIterator, - ms: number + ms: number, ): Promise | typeof INGEST_TIMEOUT> { let timer: ReturnType; const next = iterator.next(); - // If the timeout wins this race the next() promise stays pending; swallow a - // later rejection so it can't surface as an unhandled rejection. + // The transport currently has no structural cancellation seam. If the timer + // wins, swallow a later rejection while the iterator is closed best-effort. next.catch(() => {}); const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve(INGEST_TIMEOUT), ms); @@ -179,71 +287,257 @@ function nextWithDeadline( return Promise.race([next, timeout]).finally(() => clearTimeout(timer)); } +const utf8 = new TextEncoder(); +const MAX_DIAGNOSTIC_SAMPLES = 5; +const MAX_DIAGNOSTIC_SAMPLE_LENGTH = 320; + +function addDiagnosticSample( + stats: Pick< + ScheduledIngestCollectionStats, + "diagnosticSamples" | "diagnosticSamplesOmitted" + >, + message: string, +): void { + if (stats.diagnosticSamples.length >= MAX_DIAGNOSTIC_SAMPLES) { + stats.diagnosticSamplesOmitted++; + return; + } + stats.diagnosticSamples.push(message.slice(0, MAX_DIAGNOSTIC_SAMPLE_LENGTH)); +} + +/** JSON normalization used only for transport-observation fingerprints. The + * Jetstream payload has already been decoded from JSON, so sorting object keys + * makes semantically identical payloads stable across decoder/property order. */ +function normalizedJson(value: unknown): string { + if (value === null || typeof value !== "object") { + const encoded = JSON.stringify(value); + return encoded === undefined ? "null" : encoded; + } + if (Array.isArray(value)) { + return `[${value.map((item) => normalizedJson(item)).join(",")}]`; + } + const fields: string[] = []; + for (const key of Object.keys(value as Record).sort()) { + const field = (value as Record)[key]; + if (field === undefined) continue; + fields.push(`${JSON.stringify(key)}:${normalizedJson(field)}`); + } + return `{${fields.join(",")}}`; +} + +async function observationHash(value: string): Promise { + const subtle = ( + globalThis as typeof globalThis & { + crypto?: { + subtle?: { + digest( + algorithm: string, + data: Uint8Array, + ): Promise; + }; + }; + } + ).crypto?.subtle; + if (!subtle) throw new Error("Web Crypto SHA-256 is required for ingestion"); + const digest = await subtle.digest("SHA-256", utf8.encode(value)); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +} + +interface JetstreamCommitObservation { + /** One logical Jetstream source slot, before source ID/epoch prefixing. */ + key: string; + /** Stable operation/CID/normalized-payload identity for that slot. */ + fingerprint: string; + uri: string; +} + +/** Jetstream owns the transport-specific observation identity. Core collection + * adds the configured source ID and epoch before cycle-local deduplication. */ +function jetstreamCommitObservation( + event: { + did: string; + time_us: number; + commit: { + rev?: string; + collection: string; + rkey: string; + operation: string; + cid?: string; + record?: unknown; + }; + }, +): JetstreamCommitObservation { + const { commit } = event; + const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; + return { + key: JSON.stringify([event.time_us, uri, commit.rev ?? null]), + fingerprint: JSON.stringify([ + commit.operation, + commit.cid ?? null, + commit.operation === "delete" ? null : normalizedJson(commit.record), + ]), + uri, + }; +} + export async function ingestEvents( config: ContrailConfig, cursor: number | null, - safetyTimeoutMs: number = 25_000, - knownDids?: Set + budgetInput: ScheduledIngestOptions | ScheduledIngestBudget | number = + DEFAULT_SCHEDULED_INGEST_BUDGET, + knownDids?: Set, + startingCursorObservations: ReadonlySet = new Set(), ): Promise<{ events: IngestEvent[]; lastCursor: number | null; + cursorObservations: Set; identityUpdates: Map; + stats: ScheduledIngestCollectionStats; }> { - const log = getLogger(config); + const budget = resolveScheduledIngestBudget(budgetInput); const startTimeUs = Date.now() * 1000; - const deadline = Date.now() + safetyTimeoutMs; + const deadline = Date.now() + budget.maxDrainMs; const collected: IngestEvent[] = []; const collections = getCollectionNsids(config); const dependentCollections = new Set(getDependentNsids(config)); const provisionalKnownDids = knownDids ? new Set(knownDids) : undefined; - const urls = config.jetstreams ?? []; - - let totalCommits = 0; - let filteredUnknownDid = 0; - const filteredDidSamples = new Set(); - let lastYieldedTimeUs: number | null = null; - let firstYieldedTimeUs: number | null = null; - let connectCount = 0; - const seenUris = new Map(); // uri -> time_us of first occurrence - const duplicateUris: string[] = []; + const urls = config.jetstreams ?? DEFAULT_JETSTREAMS; + const sourceId = config.orderedSource?.source ?? "jetstream"; + const sourceEpoch = config.orderedSource?.epoch ?? null; + + const seenObservations = new Map>(); const identityUpdates = new Map(); + const stats: ScheduledIngestCollectionStats = { + observedSourceItems: 0, + commitObservations: 0, + identityObservations: 0, + retainedCandidates: 0, + exactDuplicatesDropped: 0, + cursorBoundaryDuplicatesDropped: 0, + resumeOverlapDropped: 0, + sourceScopeFiltered: 0, + sourceInconsistencies: 0, + serializedCandidateBytes: 0, + startingCursor: cursor, + lastAccountedCursor: null, + safeEndingCursor: cursor, + stopReason: "idle", + connections: 0, + connectionCloses: 0, + connectionErrors: 0, + diagnosticSamples: [], + diagnosticSamplesOmitted: 0, + }; + // A durable timestamp cursor is resumed one microsecond earlier. This works + // with inclusive and exclusive timestamp APIs: all observations at the + // coarse cursor are replayed, while persisted hashes suppress only the exact + // items already accounted for there. It also protects a captured empty-start + // cursor from a later item that happens to share its microsecond. + const requestedCursor = + cursor === null ? null : Math.max(0, cursor - 1); const subscription = new JetstreamSubscription({ // A single-instance config is handed over as a string so @atcute skips its - // array-only first-connect cursor rollback (see jetstreamUrlOption). On the - // cron model that rollback would otherwise re-ingest 10s every cycle. + // array-only first-connect cursor rollback (see jetstreamUrlOption). url: jetstreamUrlOption(urls), wantedCollections: collections, - ...(cursor !== null ? { cursor } : {}), + ...(requestedCursor !== null ? { cursor: requestedCursor } : {}), onConnectionOpen() { - connectCount++; - log.log( - `[ingest] connected to Jetstream #${connectCount} (url=${urls.join("|")}, cursor=${cursor ?? "none"}, wanted=${collections.join(",")})` - ); + stats.connections++; }, - onConnectionClose(event) { - log.log( - `[ingest] disconnected from Jetstream: ${event.code} ${event.reason}` - ); + onConnectionClose() { + stats.connectionCloses++; }, onConnectionError(event) { - log.error("[ingest] Jetstream error:", event.error); + stats.connectionErrors++; + addDiagnosticSample(stats, `Jetstream connection error: ${String(event.error)}`); }, }); + // Capture Atcute's constructor cursor before iteration can buffer frames and + // move it ahead. With no durable cursor this is the subscription's effective + // lower bound and must be persisted even when the first drain stays empty. + const effectiveStartCursor = cursor ?? subscription.cursor ?? null; + stats.safeEndingCursor = effectiveStartCursor; + let boundaryCursor = effectiveStartCursor; + let boundaryObservations = new Set(startingCursorObservations); + // Only hashes newly accounted for this cycle are returned for insertion. The + // database unions them with existing boundary rows, keeping each DB batch + // bounded even if many cycles share one coarse cursor. + let cursorObservations = new Set(); + const singleEndpoint = urls.length === 1; + const iterator = subscription[Symbol.asyncIterator](); type Ev = typeof subscription extends AsyncIterable ? V : never; - // Collect or skip a single event. Filtering uses early `return` rather than - // the loop's `continue` so the loop's exit checks still run after a filtered - // event (a stream of all-filtered events must not skip the deadline). - const handleEvent = (event: Ev): void => { + const accountSourceItem = async ( + event: Ev, + identity: string, + ): Promise => { + // The scheduled path requires one pinned endpoint. Its one-microsecond + // boundary overlap is known-complete below the durable cursor and must not + // consume candidate/byte budgets on every restart. + if ( + singleEndpoint && + effectiveStartCursor !== null && + event.time_us < effectiveStartCursor + ) { + stats.resumeOverlapDropped++; + return true; + } + + const hash = await observationHash( + JSON.stringify([sourceId, sourceEpoch, identity]), + ); + if (boundaryCursor === null || event.time_us > boundaryCursor) { + boundaryCursor = event.time_us; + boundaryObservations = new Set([hash]); + cursorObservations = new Set([hash]); + } else if (event.time_us === boundaryCursor) { + const alreadyAccounted = boundaryObservations.has(hash); + boundaryObservations.add(hash); + if (!alreadyAccounted) cursorObservations.add(hash); + } + + if ( + effectiveStartCursor !== null && + event.time_us === effectiveStartCursor && + startingCursorObservations.has(hash) + ) { + stats.cursorBoundaryDuplicatesDropped++; + return true; + } + return false; + }; + + // Fully account for one yielded item before the loop considers any stop + // threshold. Exact boundary observations are suppressed before evolving + // source-scope policy; ordinary cheap filters still precede cycle dedupe. + const handleEvent = async (event: Ev): Promise => { if (event.kind === "commit") { const { commit } = event; - totalCommits++; - - const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; + stats.commitObservations++; + const observation = jetstreamCommitObservation(event); + const dedupeKey = JSON.stringify([ + sourceId, + sourceEpoch, + observation.key, + ]); + if ( + await accountSourceItem( + event, + JSON.stringify([ + "commit", + observation.key, + observation.fingerprint, + ]), + ) + ) { + return; + } if ( provisionalKnownDids && @@ -257,89 +551,104 @@ export async function ingestEvents( provisionalKnownDids && !provisionalKnownDids.has(event.did) ) { - filteredUnknownDid++; - if (filteredDidSamples.size < 10) filteredDidSamples.add(event.did); + stats.sourceScopeFiltered++; return; } - const prev = seenUris.get(uri); - if (prev !== undefined) { - duplicateUris.push(uri); - log.warn( - `[ingest] DUPLICATE in cycle: ${uri} first time_us=${prev}, again=${event.time_us}, delta=${event.time_us - prev}us` + const fingerprints = seenObservations.get(dedupeKey); + if (fingerprints?.has(observation.fingerprint)) { + stats.exactDuplicatesDropped++; + return; + } + if (fingerprints) { + stats.sourceInconsistencies++; + addDiagnosticSample( + stats, + `source slot changed payload: ${observation.uri} time_us=${event.time_us}`, ); + fingerprints.add(observation.fingerprint); } else { - seenUris.set(uri, event.time_us); + seenObservations.set(dedupeKey, new Set([observation.fingerprint])); } - collected.push( - createIngestEvent({ - did: event.did, - timeUs: - commit.operation === "delete" - ? event.time_us - : recordTimeUs( - commit.record, - commit.collection, - config, - event.time_us, - ), - collection: commit.collection, - operation: commit.operation, - rkey: commit.rkey, - cid: commit.operation === "delete" ? null : commit.cid, - value: commit.operation === "delete" ? undefined : commit.record, - source: { - id: config.orderedSource?.source ?? "jetstream", - ...(config.orderedSource - ? { epoch: config.orderedSource.epoch } - : {}), - time_us: event.time_us, - revision: commit.rev, - cursor: String(event.time_us), - }, - }), - ); - - log.log( - `[ingest] candidate: ${commit.operation} ${uri} time_us=${event.time_us}` - ); - + const candidate = createIngestEvent({ + did: event.did, + timeUs: + commit.operation === "delete" + ? event.time_us + : recordTimeUs( + commit.record, + commit.collection, + config, + event.time_us, + ), + collection: commit.collection, + operation: commit.operation, + rkey: commit.rkey, + cid: commit.operation === "delete" ? null : commit.cid, + value: commit.operation === "delete" ? undefined : commit.record, + source: { + id: sourceId, + ...(sourceEpoch === null ? {} : { epoch: sourceEpoch }), + time_us: event.time_us, + revision: commit.rev, + cursor: String(event.time_us), + }, + }); + collected.push(candidate); + stats.retainedCandidates++; + stats.serializedCandidateBytes += + SCHEDULED_INGEST_METADATA_BYTES + + (candidate.record === null ? 0 : utf8.encode(candidate.record).byteLength); } else if (event.kind === "identity") { + stats.identityObservations++; + if (await accountSourceItem(event, normalizedJson(event))) return; identityUpdates.set(event.did, event.identity.handle); + } else { + await accountSourceItem(event, normalizedJson(event)); } }; for (;;) { - // Run the exit checks BEFORE awaiting the next event and regardless of - // whether the previous event was filtered — otherwise a quiet stream blocks - // forever and an all-filtered flood never reaches the deadline check. - if (Date.now() >= deadline) { - log.log( - `[ingest] safety timeout reached, stopping (deadline=${deadline}, collected=${collected.length})` - ); + // Check the deadline before requesting another item. A hot iterator must not + // get one extra next() after any threshold has been reached. + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + stats.stopReason = "drain-time"; break; } - const step = await nextWithDeadline(iterator, Math.max(0, deadline - Date.now())); + const step = await nextWithDeadline(iterator, remainingMs); if (step === INGEST_TIMEOUT) { - log.log( - `[ingest] safety timeout reached, stopping (deadline=${deadline}, collected=${collected.length})` - ); + stats.stopReason = "drain-time"; + break; + } + if (step.done) { + stats.stopReason = "idle"; break; } - if (step.done) break; const event = step.value; + stats.observedSourceItems++; + + await handleEvent(event); + // handleEvent either retained, filtered, deduplicated, or deliberately + // handled the item. Only now is its cursor safe to checkpoint. Keep the + // representable checkpoint monotonic across deliberate overlap replay. + stats.lastAccountedCursor = Math.max( + stats.lastAccountedCursor ?? event.time_us, + event.time_us, + ); - if (firstYieldedTimeUs === null) firstYieldedTimeUs = event.time_us; - lastYieldedTimeUs = event.time_us; - - handleEvent(event); - + if (stats.retainedCandidates >= budget.maxCandidates) { + stats.stopReason = "count"; + break; + } + if (stats.serializedCandidateBytes >= budget.maxSerializedBytes) { + stats.stopReason = "bytes"; + break; + } if (event.time_us >= startTimeUs) { - log.log( - `[ingest] caught up to present, stopping (last time_us=${event.time_us}, startTimeUs=${startTimeUs})` - ); + stats.stopReason = "head"; break; } } @@ -348,74 +657,98 @@ export async function ingestEvents( // iterator's return on a quiet stream could itself block (the hang we fix). Promise.resolve(iterator.return?.()).catch(() => {}); - if (filteredUnknownDid > 0) { - const sample = [...filteredDidSamples].join(", "); - log.log( - `[ingest] ${filteredUnknownDid} events filtered (unknown did). sample dids: ${sample}` - ); - } - const subscriptionCursor = subscription.cursor || null; - // @atcute may advance its internal cursor when an event enters its buffer, - // before the async iterator yields that event. Persist only through the last - // event Contrail actually observed; anything buffered is deliberately replayed. + // Never read Atcute's cursor again here: it may have moved when a frame was + // buffered but not yielded. The constructor cursor captured above and the + // maximum fully-accounted yielded cursor are the only safe positions. const lastCursor = - lastYieldedTimeUs === null - ? subscriptionCursor - : subscriptionCursor === null - ? lastYieldedTimeUs - : Math.min(subscriptionCursor, lastYieldedTimeUs); - - const cursorGap = - subscriptionCursor !== null && lastYieldedTimeUs !== null - ? subscriptionCursor - lastYieldedTimeUs - : null; - - // Detect the library's internal cursor rollback (picks a different URL → rolls - // back 10s → first event comes in BEFORE the cursor we asked it to start from). - const rolledBackUs = - cursor !== null && firstYieldedTimeUs !== null && firstYieldedTimeUs < cursor - ? cursor - firstYieldedTimeUs - : 0; - - log.log( - `[ingest] jetstream loop done. commits_seen=${totalCommits}, filtered=${filteredUnknownDid}, candidates=${collected.length}, dupes=${duplicateUris.length}, connects=${connectCount}, first_yielded=${firstYieldedTimeUs ?? "none"}, last_yielded=${lastYieldedTimeUs ?? "none"}, subscription_cursor=${subscriptionCursor ?? "none"}, safe_cursor=${lastCursor ?? "none"}, cursor_gap=${cursorGap ?? "n/a"}us, rolled_back=${rolledBackUs}us` - ); - - if (cursorGap !== null && cursorGap > 1000) { - log.warn( - `[ingest] CURSOR GAP: subscription cursor is ${cursorGap}us (${Math.floor( - cursorGap / 1000 - )}ms) ahead of last yielded event — saving safe_cursor=${lastCursor ?? "none"}; buffered events will replay` - ); + effectiveStartCursor === null + ? stats.lastAccountedCursor + : Math.max( + effectiveStartCursor, + stats.lastAccountedCursor ?? effectiveStartCursor, + ); + stats.safeEndingCursor = lastCursor; + if (lastCursor !== boundaryCursor) { + // This occurs only for an older deliberate overlap. Leave the durable + // boundary unchanged rather than attaching old hashes to a newer cursor. + cursorObservations = new Set(); } - if (connectCount > 1) { - if (urls.length > 1) { - // Multi-instance pool: each reconnect picks a URL at random, and @atcute - // rolls the cursor back up to 10s on a fresh instance to absorb clock skew. - log.warn( - `[ingest] RECONNECTED ${connectCount} times during cycle across a ${urls.length}-instance pool — each reconnect picks a URL at random and may roll the cursor back up to 10s (rolled_back=${rolledBackUs}us this cycle)` - ); - } else { - // Single fixed instance (see jetstreamUrlOption): reconnects resume on the - // same instance from the saved cursor, so there is no rollback. - log.log( - `[ingest] reconnected ${connectCount} times during cycle to the single fixed instance — no cursor rollback (rolled_back=${rolledBackUs}us)` - ); - } + return { + events: collected, + lastCursor, + cursorObservations, + identityUpdates, + stats, + }; +} + +function emptyDropCounts(): IngestDropCounts { + return { + unknownCollection: 0, + invalidRecord: 0, + lexiconValidation: 0, + cidMismatch: 0, + cidEncoding: 0, + missingCid: 0, + recordFilter: 0, + unknownActor: 0, + unknownSubject: 0, + superseded: 0, + }; +} + +function addDropCounts(target: IngestDropCounts, value: IngestDropCounts): void { + for (const key of Object.keys(target) as Array) { + target[key] += value[key]; } +} - return { events: collected, lastCursor, identityUpdates }; +function admissionFilteredCount(dropped: IngestDropCounts): number { + return Object.entries(dropped).reduce( + (total, [key, value]) => key === "superseded" ? total : total + value, + 0, + ); +} + +function runtimeCpuUsage(): (() => number) | null { + const runtimeProcess = ( + globalThis as typeof globalThis & { + process?: { + cpuUsage?: (previous?: { user: number; system: number }) => { + user: number; + system: number; + }; + }; + } + ).process; + if (!runtimeProcess?.cpuUsage) return null; + const start = runtimeProcess.cpuUsage(); + return () => { + const elapsed = runtimeProcess.cpuUsage!(start); + return Math.round((elapsed.user + elapsed.system) / 100) / 10; + }; } // Run a full ingest cycle: init schema, load cursor, ingest, apply, save cursor export async function runIngestCycle( db: Database, config: ContrailConfig, - timeoutMs: number = 25_000, + budgetInput: ScheduledIngestOptions | ScheduledIngestBudget | number = + DEFAULT_SCHEDULED_INGEST_BUDGET, state?: IngestState, ): Promise { const log = getLogger(config); + const budget = resolveScheduledIngestBudget(budgetInput); + const scheduledUrls = config.jetstreams ?? DEFAULT_JETSTREAMS; + if (scheduledUrls.length !== 1) { + throw new TypeError( + "scheduled ingestion requires exactly one pinned Jetstream endpoint; " + + "use runPersistent() for a failover pool", + ); + } + const wallStartedAt = Date.now(); + const finishCpuUsage = runtimeCpuUsage(); const s = state ?? createIngestState(); if (!s.schemaInitialized) { @@ -424,15 +757,8 @@ export async function runIngestCycle( } const cursor = await getLastCursor(db); - const collections = getCollectionNsids(config); - const nowUs = Date.now() * 1000; - const lagMs = cursor !== null ? Math.floor((nowUs - cursor) / 1000) : null; - - log.log( - `[ingest] starting cycle. cursor=${cursor ?? "none"}${ - lagMs !== null ? ` (lag=${lagMs}ms)` : "" - }, timeout=${timeoutMs}ms, collections=${collections.join(", ")}` - ); + const startingCursorObservations = + cursor === null ? new Set() : await getCursorObservations(db, cursor); // Load known DIDs for filtering dependent collections const dependentCollections = getDependentNsids(config); @@ -441,44 +767,44 @@ export async function runIngestCycle( if (dependentCollections.length > 0) { if (s.cachedKnownDids) { knownDids = s.cachedKnownDids; - log.log(`Using cached known DIDs (${knownDids.size} users)`); } else { const result = await db .prepare("SELECT did FROM identities") .all<{ did: string }>(); knownDids = new Set((result.results ?? []).map((r) => r.did)); s.cachedKnownDids = knownDids; - log.log(`Loaded ${knownDids.size} known DIDs from database`); } } - const { events, lastCursor, identityUpdates } = await ingestEvents( + const { + events, + lastCursor, + cursorObservations, + identityUpdates, + stats, + } = await ingestEvents( config, cursor, - timeoutMs, - knownDids + budget, + knownDids, + startingCursorObservations, ); - if (events.length > 0) { - const breakdown: Record = {}; - for (const e of events) { - const key = `${e.collection}:${e.operation}`; - breakdown[key] = (breakdown[key] ?? 0) + 1; - } - log.log( - `[ingest] received ${events.length} events. breakdown=${JSON.stringify(breakdown)}` - ); - } else { - log.log(`[ingest] received 0 events from Jetstream`); - } - const accepted: IngestEvent[] = []; const newlyKnownDids: string[] = []; + const dropped = emptyDropCounts(); + let databaseSubBatchesCommitted = 0; + const warningSamples: IngestWarningSamples = { + maxSamples: MAX_DIAGNOSTIC_SAMPLES, + samples: stats.diagnosticSamples, + omitted: stats.diagnosticSamplesOmitted, + }; for (let i = 0; i < events.length; i += BATCH_SIZE) { const batch = events.slice(i, i + BATCH_SIZE); const isFinalBatch = i + BATCH_SIZE >= events.length; const result = await ingestRecords(db, batch, config, { knownDids, + warningSamples, // Earlier batches may commit without moving the cursor. A crash replays // them safely; the final batch atomically commits the exact source cursor. trailingStatements: @@ -494,10 +820,17 @@ export async function runIngestCycle( ), ] : []), + ...saveCursorObservationStatements( + db, + lastCursor, + cursorObservations, + ), ] : undefined, }); + databaseSubBatchesCommitted++; accepted.push(...result.accepted); + addDropCounts(dropped, result.dropped); if (knownDids) { for (const did of result.discoveredDids) { knownDids.add(did); @@ -506,54 +839,97 @@ export async function runIngestCycle( } } - // Apply handle changes from #identity events. UPDATE-only, so unknown - // DIDs are no-ops — we don't want to create partial rows lacking PDS. - if (identityUpdates.size > 0) { - for (const [did, handle] of identityUpdates) { - try { - await applyIdentityEvent(db, did, handle); - } catch (err) { - log.warn(`[ingest] identity update failed for ${did}: ${err}`); + // Apply handle changes from #identity events. UPDATE-only, so unknown DIDs + // are no-ops. Failures are sampled into the bounded cycle summary. + let identityUpdateFailures = 0; + for (const [did, handle] of identityUpdates) { + try { + await applyIdentityEvent(db, did, handle); + } catch (error) { + identityUpdateFailures++; + if (warningSamples.samples.length < warningSamples.maxSamples) { + warningSamples.samples.push( + `identity update failed for ${did}: ${String(error)}`.slice( + 0, + MAX_DIAGNOSTIC_SAMPLE_LENGTH, + ), + ); + } else { + warningSamples.omitted++; } } - log.log(`[ingest] applied ${identityUpdates.size} identity event(s)`); } // A commit batch saves its source cursor in the projection transaction above. - // An identity-only or fully filtered stream has no canonical record batch, so - // its cursor can advance independently after best-effort identity handling. - if (lastCursor !== null && events.length === 0) { - await saveCursor(db, lastCursor, config.orderedSource); - } - if (lastCursor !== null) { - log.log( - `[ingest] saved cursor=${lastCursor} (advanced ${ - cursor !== null ? lastCursor - cursor : "n/a" - }us, atomic=${events.length > 0})`, + // Identity-only, filtered-only, or otherwise candidate-free accounted ranges + // checkpoint only after best-effort identity handling. No observed item means + // there is no new source work to save, even if the subscription buffered ahead. + if ( + lastCursor !== null && + events.length === 0 && + // Save an accounted range, or capture Atcute's constructor cursor for an + // empty first drain so the next cron cannot silently start later. + (stats.lastAccountedCursor !== null || cursor === null) + ) { + await saveCursor( + db, + lastCursor, + config.orderedSource, + cursorObservations, ); - } else { - log.log(`[ingest] no cursor returned from subscription; not saving`); + databaseSubBatchesCommitted++; } // Refresh stale/missing identities for DIDs in this batch (best-effort; runs // after the cursor save so its network latency can't strand forward progress). const uniqueDids = [...new Set(accepted.map((e) => e.did))]; + let identityRefreshFailures = 0; if (uniqueDids.length > 0) { try { await refreshStaleIdentities(db, uniqueDids, config); - } catch (err) { - log.warn(`Identity refresh failed: ${err}`); + } catch (error) { + identityRefreshFailures++; + if (warningSamples.samples.length < warningSamples.maxSamples) { + warningSamples.samples.push( + `identity refresh failed: ${String(error)}`.slice( + 0, + MAX_DIAGNOSTIC_SAMPLE_LENGTH, + ), + ); + } else { + warningSamples.omitted++; + } } } - // Newly-discovered DIDs: ask Constellation for back-edges so they - // immediately appear in existing followers' feeds (best-effort, opt-out). + // Newly-discovered DIDs: ask Constellation for back-edges. Suppress helper + // per-subject logs and report bounded aggregate results in the cycle summary. + let constellationFailures = 0; + let constellationInserted = 0; if (config.feeds && newlyKnownDids.length > 0) { + const quietConfig = { + ...config, + logger: { log() {}, warn() {}, error() {} }, + }; for (const subj of newlyKnownDids) { try { - await backfillFollowersFromConstellation(db, config, subj); - } catch (err) { - log.warn(`[constellation] subject=${subj} failed: ${err}`); + constellationInserted += await backfillFollowersFromConstellation( + db, + quietConfig, + subj, + ); + } catch (error) { + constellationFailures++; + if (warningSamples.samples.length < warningSamples.maxSamples) { + warningSamples.samples.push( + `constellation failed for ${subj}: ${String(error)}`.slice( + 0, + MAX_DIAGNOSTIC_SAMPLE_LENGTH, + ), + ); + } else { + warningSamples.omitted++; + } } } } @@ -586,5 +962,41 @@ export async function runIngestCycle( // config.maintenance.optimize is set). await maybeOptimize(db, config, log); - log.log(`[ingest] cycle complete. stored=${accepted.length}`); + const summary = { + observed_source_items: stats.observedSourceItems, + commit_observations: stats.commitObservations, + identity_observations: stats.identityObservations, + retained_candidates: stats.retainedCandidates, + exact_duplicates_dropped: stats.exactDuplicatesDropped, + cursor_boundary_duplicates_dropped: + stats.cursorBoundaryDuplicatesDropped, + resume_overlap_dropped: stats.resumeOverlapDropped, + source_scope_filtered: stats.sourceScopeFiltered, + admission_policy_filtered: admissionFilteredCount(dropped), + candidates_superseded: dropped.superseded, + source_inconsistencies: stats.sourceInconsistencies, + serialized_candidate_bytes: stats.serializedCandidateBytes, + max_candidates: budget.maxCandidates, + max_serialized_bytes: budget.maxSerializedBytes, + max_drain_ms: budget.maxDrainMs, + starting_cursor: cursor, + last_accounted_cursor: stats.lastAccountedCursor, + safe_ending_cursor: stats.safeEndingCursor, + stop_reason: stats.stopReason, + database_sub_batches_committed: databaseSubBatchesCommitted, + projected_mutations_accepted: accepted.length, + identity_updates_attempted: identityUpdates.size, + identity_update_failures: identityUpdateFailures, + identity_refresh_failures: identityRefreshFailures, + constellation_inserted: constellationInserted, + constellation_failures: constellationFailures, + connections: stats.connections, + connection_closes: stats.connectionCloses, + connection_errors: stats.connectionErrors, + wall_ms: Date.now() - wallStartedAt, + cpu_ms: finishCpuUsage?.() ?? null, + diagnostic_samples: warningSamples.samples, + diagnostic_samples_omitted: warningSamples.omitted, + }; + log.log(`[ingest] cycle summary ${JSON.stringify(summary)}`); } diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index c472054..f46691e 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -275,10 +275,11 @@ export interface ContrailConfig { /** Jetstream endpoints to ingest from (defaults to {@link DEFAULT_JETSTREAMS}). * Prefer a single endpoint: one instance has no clock skew, so `@atcute` takes * no cursor rollback (see {@link jetstreamUrlOption}) — important for the cron - * model, which rebuilds the subscription every cycle. Use 2+ only for failover - * across interchangeable endpoints, and ideally only with a persistent - * connection (`runPersistent`), where the per-switch 10s skew rollback fires - * about once rather than every cycle. */ + * model, which rebuilds the subscription every cycle. Scheduled ingestion + * requires exactly one pinned endpoint because a pooled connection rolls its + * timestamp cursor back on every new cycle and cannot make bounded progress + * through a dense overlap. Use 2+ only with `runPersistent()`, where the + * connection and its failover cursor remain long-lived. */ jetstreams?: string[]; /** Identity of the ordered source consumed by live ingestion. Its opaque * cursor is persisted atomically with projected mutations and may be exposed diff --git a/packages/contrail/src/worker/index.ts b/packages/contrail/src/worker/index.ts index a75f0a0..17f97ff 100644 --- a/packages/contrail/src/worker/index.ts +++ b/packages/contrail/src/worker/index.ts @@ -18,6 +18,7 @@ import { Contrail } from "../contrail.js"; import { createHandler } from "../server.js"; import type { ContrailConfig, Database } from "../core/types.js"; import type { BackfillRetryOptions } from "../core/backfill.js"; +import type { ScheduledIngestOptions } from "../core/jetstream.js"; import { normalizePublicServiceEndpoint, validatePublicServiceAuthEndpoint, @@ -33,6 +34,8 @@ export interface CreateWorkerOptions { lexicons?: object[]; /** Enable stable discovery and Lexicon routes for anonymous remote clients. */ publicService?: PublicServiceOptions; + /** Count, byte, and drain-time limits for each scheduled Jetstream cycle. */ + scheduledIngest?: ScheduledIngestOptions; /** Bounded pending-account retry slice after each scheduled ingest. Enabled * by default; pass `false` to disable or options to tune its budget. */ backfillRetries?: BackfillRetryOptions | false; @@ -84,7 +87,7 @@ export function createWorker( // failures. A database lease prevents overlap with a manual backfill. ctx.waitUntil( (async () => { - await contrail.ingest({}, db); + await contrail.ingest(options.scheduledIngest, db); if (options.backfillRetries !== false) { await contrail.retryBackfill(options.backfillRetries, db); } diff --git a/packages/contrail/tests/ingest-hang.test.ts b/packages/contrail/tests/ingest-hang.test.ts index 8d7ee34..2c7bebd 100644 --- a/packages/contrail/tests/ingest-hang.test.ts +++ b/packages/contrail/tests/ingest-hang.test.ts @@ -27,7 +27,11 @@ vi.mock("@atcute/jetstream", () => { return { JetstreamSubscription: MockJetstreamSubscription }; }); -import { ingestEvents } from "../src/index"; +import { + ingestEvents, + resolveScheduledIngestBudget, + SCHEDULED_INGEST_METADATA_BYTES, +} from "../src/index"; import { resolveConfig } from "../src/index"; import type { ContrailConfig } from "../src/index"; @@ -38,22 +42,44 @@ function commitEvent( collection: string, time_us: number, rkey: string, + options: { + revision?: string; + record?: Record; + cid?: string; + } = {}, ) { return { kind: "commit" as const, time_us, did, commit: { - rev: String(time_us), + rev: options.revision ?? String(time_us), collection, operation: "create" as const, rkey, - cid: "bafy" + rkey, - record: { name: "Test Event", startsAt: "2026-04-01T10:00:00Z", mode: "online" }, + cid: options.cid ?? "bafy" + rkey, + record: options.record ?? { + name: "Test Event", + startsAt: "2026-04-01T10:00:00Z", + mode: "online", + }, }, }; } +function budget(overrides: Partial<{ + maxDrainMs: number; + maxCandidates: number; + maxSerializedBytes: number; +}> = {}) { + return { + maxDrainMs: 5_000, + maxCandidates: 100, + maxSerializedBytes: 1024 * 1024, + ...overrides, + }; +} + /** A discoverable-only config — events flow straight into `collected`. */ function discoverableConfig(): ContrailConfig { return { @@ -127,6 +153,7 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { }); // The replayed cursor must come back so the caller can persist it. expect(result.lastCursor).toBe(1_000_000); + expect(result.stats.stopReason).toBe("drain-time"); }); it("never checkpoints past the last yielded event when the subscription buffers ahead", async () => { @@ -152,6 +179,7 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { expect(result.events).toHaveLength(1); expect(result.lastCursor).toBe(1_000_000); + expect(result.stats.lastAccountedCursor).toBe(1_000_000); }); it("returns by the safety timeout even when every arriving event is filtered out", async () => { @@ -177,6 +205,8 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { // ...but the cursor still advanced, so the caller persists forward progress. expect(result.lastCursor).not.toBeNull(); expect(result.lastCursor).toBeGreaterThan(1_000_000); + expect(result.stats.stopReason).toBe("drain-time"); + expect(result.stats.lastAccountedCursor).toBe(result.lastCursor); } finally { jetstream.abort = true; } @@ -237,6 +267,7 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { expect(result.events.map((e) => e.rkey)).toEqual(["hist", "live"]); expect(result.lastCursor).toBe(liveUs); + expect(result.stats.stopReason).toBe("head"); }); it("caught-up break fires even on a filtered live event, deferring a following kept event to the next cycle", async () => { @@ -315,4 +346,244 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { expect(result.identityUpdates.get("did:plc:author")).toBe("alice.test"); expect(result.events).toHaveLength(0); // identity events are not record commits }); + + it("validates every scheduled work threshold", () => { + expect(() => resolveScheduledIngestBudget({ maxCandidates: 0 })).toThrow( + "maxCandidates must be a positive finite integer", + ); + expect(() => + resolveScheduledIngestBudget({ maxSerializedBytes: Number.NaN }), + ).toThrow("maxSerializedBytes must be a positive finite integer"); + expect(() => resolveScheduledIngestBudget({ maxDrainMs: 1.5 })).toThrow( + "maxDrainMs must be a positive finite integer", + ); + }); + + it("stops an infinite hot stream at exactly maxCandidates without another next()", async () => { + let produced = 0; + jetstream.script = async function* (self) { + for (let index = 0; !jetstream.abort; index++) { + produced++; + self.cursor = 1_000_000 + index; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + self.cursor, + `hot-${index}`, + ); + } + }; + + const result = await ingestEvents( + discoverableConfig(), + 999_999, + budget({ maxCandidates: 3 }), + ); + + expect(result.events).toHaveLength(3); + expect(produced).toBe(3); + expect(result.stats).toMatchObject({ + observedSourceItems: 3, + retainedCandidates: 3, + stopReason: "count", + lastAccountedCursor: 1_000_002, + safeEndingCursor: 1_000_002, + }); + }); + + it("retains the byte-threshold-crossing candidate and does not split equal cursors", async () => { + const firstRecord = { value: "a" }; + const secondRecord = { value: "bbbb" }; + const encoder = new TextEncoder(); + const firstBytes = + SCHEDULED_INGEST_METADATA_BYTES + + encoder.encode(JSON.stringify(firstRecord)).byteLength; + const secondBytes = + SCHEDULED_INGEST_METADATA_BYTES + + encoder.encode(JSON.stringify(secondRecord)).byteLength; + const byteThreshold = firstBytes + 1; + let produced = 0; + jetstream.script = async function* (self) { + self.cursor = 2_000_000; + produced++; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 2_000_000, + "same-cursor-1", + { record: firstRecord }, + ); + produced++; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 2_000_000, + "same-cursor-2", + { record: secondRecord }, + ); + produced++; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 2_000_001, + "not-requested", + ); + }; + + const result = await ingestEvents( + discoverableConfig(), + 999_999, + budget({ maxSerializedBytes: byteThreshold }), + ); + + expect(result.events.map((event) => event.rkey)).toEqual([ + "same-cursor-1", + "same-cursor-2", + ]); + expect(produced).toBe(2); + expect(result.stats.stopReason).toBe("bytes"); + expect(result.stats.serializedCandidateBytes).toBe(firstBytes + secondBytes); + expect(result.stats.serializedCandidateBytes - byteThreshold).toBeLessThanOrEqual( + secondBytes, + ); + expect(result.lastCursor).toBe(2_000_000); + }); + + it("drops only exact source observations and retains genuine revisions", async () => { + jetstream.script = async function* (self) { + self.cursor = 3_000_000; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 3_000_000, + "record", + { revision: "rev-1", record: { b: 2, a: 1 } }, + ); + // Same source slot and normalized payload, despite object key order. + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 3_000_000, + "record", + { revision: "rev-1", record: { a: 1, b: 2 } }, + ); + self.cursor = 3_000_001; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 3_000_001, + "record", + { revision: "rev-2", record: { a: 2, b: 2 } }, + ); + }; + + const result = await ingestEvents( + discoverableConfig(), + 999_999, + budget(), + ); + + expect(result.events.map((event) => event.source?.revision)).toEqual([ + "rev-1", + "rev-2", + ]); + expect(result.stats.exactDuplicatesDropped).toBe(1); + expect(result.stats.retainedCandidates).toBe(2); + expect(result.stats.lastAccountedCursor).toBe(3_000_001); + }); + + it("advances the accounted cursor through filtered and duplicate observations", async () => { + const knownDids = new Set(); + jetstream.script = async function* (self) { + self.cursor = 4_000_000; + const original = commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 4_000_000, + "kept", + ); + yield original; + yield original; + self.cursor = 4_000_001; + yield commitEvent( + "did:plc:stranger", + "app.bsky.graph.follow", + 4_000_001, + "filtered", + ); + }; + + const result = await ingestEvents( + dependentConfig(), + 999_999, + budget(), + knownDids, + ); + + expect(result.events).toHaveLength(1); + expect(result.stats.exactDuplicatesDropped).toBe(1); + expect(result.stats.sourceScopeFiltered).toBe(1); + expect(result.stats.observedSourceItems).toBe(3); + expect(result.lastCursor).toBe(4_000_001); + }); + + it("never regresses the safe cursor when a pooled transport replays older overlap", async () => { + const startingCursor = 6_000_000; + jetstream.script = async function* (self) { + self.cursor = startingCursor - 500; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + startingCursor - 500, + "rolled-back", + ); + }; + const config = { + ...discoverableConfig(), + jetstreams: ["wss://one.test", "wss://two.test"], + }; + + const result = await ingestEvents( + config, + startingCursor, + budget({ maxCandidates: 1 }), + ); + + expect(result.events).toHaveLength(1); + expect(result.stats.lastAccountedCursor).toBe(startingCursor - 500); + expect(result.lastCursor).toBe(startingCursor); + expect(result.stats.safeEndingCursor).toBe(startingCursor); + }); + + it("keeps source-inconsistency diagnostics and normal logs bounded", async () => { + const logs: unknown[][] = []; + const config = { + ...discoverableConfig(), + logger: { + log: (...args: unknown[]) => logs.push(args), + warn: (...args: unknown[]) => logs.push(args), + error: (...args: unknown[]) => logs.push(args), + }, + }; + jetstream.script = async function* (self) { + self.cursor = 5_000_000; + for (let index = 0; index < 50; index++) { + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + 5_000_000, + "inconsistent", + { revision: "same", record: { value: index } }, + ); + } + }; + + const result = await ingestEvents(config, 999_999, budget()); + + expect(result.events).toHaveLength(50); + expect(result.stats.sourceInconsistencies).toBe(49); + expect(result.stats.diagnosticSamples).toHaveLength(5); + expect(result.stats.diagnosticSamplesOmitted).toBe(44); + expect(logs).toHaveLength(0); + }); }); diff --git a/packages/contrail/tests/scheduled-ingest.test.ts b/packages/contrail/tests/scheduled-ingest.test.ts new file mode 100644 index 0000000..12d9391 --- /dev/null +++ b/packages/contrail/tests/scheduled-ingest.test.ts @@ -0,0 +1,323 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const source = vi.hoisted(() => ({ + initialCursor: 0, + requestedCursors: [] as Array, + events: [] as Array<{ + kind: "commit"; + time_us: number; + did: string; + commit: { + rev: string; + collection: string; + operation: "create"; + rkey: string; + cid: string; + record: unknown; + }; + }>, +})); + +vi.mock("@atcute/jetstream", () => { + class MockJetstreamSubscription { + cursor: number | null; + private readonly start: number | null; + + constructor(options: { cursor?: number }) { + this.start = options.cursor ?? source.initialCursor; + this.cursor = this.start; + source.requestedCursors.push(this.start); + } + + async *[Symbol.asyncIterator]() { + for (const event of source.events) { + // Model the supported source's resume coordinate: only events after the + // committed cursor are delivered on the next scheduled connection. + if (this.start !== null && event.time_us <= this.start) continue; + this.cursor = event.time_us; + yield event; + } + } + } + + return { JetstreamSubscription: MockJetstreamSubscription }; +}); + +import { + getLastCursor, + resolveConfig, + runIngestCycle, + type Logger, +} from "../src/index"; +import { createTestDb } from "./helpers"; + +const COLLECTION = "com.example.event"; + +function commit(time_us: number, rkey: string) { + return { + kind: "commit" as const, + time_us, + did: "actor", + commit: { + rev: String(time_us), + collection: COLLECTION, + operation: "create" as const, + rkey, + cid: `bafy-${rkey}`, + record: { $type: COLLECTION, value: rkey }, + }, + }; +} + +function config(logger: Logger, dependent = false) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + constellation: false, + collections: dependent + ? { + event: { collection: COLLECTION }, + follow: { collection: "app.bsky.graph.follow", discover: false }, + } + : { event: { collection: COLLECTION } }, + logger, + }); +} + +function logger() { + const lines: Array<{ level: "log" | "warn" | "error"; text: string }> = []; + return { + lines, + value: { + log: (...values: unknown[]) => + lines.push({ level: "log" as const, text: values.map(String).join(" ") }), + warn: (...values: unknown[]) => + lines.push({ level: "warn" as const, text: values.map(String).join(" ") }), + error: (...values: unknown[]) => + lines.push({ level: "error" as const, text: values.map(String).join(" ") }), + }, + }; +} + +const budget = { + maxDrainMs: 5_000, + maxCandidates: 2, + maxSerializedBytes: 1024 * 1024, +}; + +describe("bounded scheduled ingest cycles", () => { + beforeEach(() => { + source.initialCursor = 0; + source.requestedCursors = []; + source.events = []; + }); + + it("commits a capped cycle cursor and converges after restart", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value); + const first = commit(1_000_001, "one"); + source.events = [first, first, commit(1_000_002, "two"), commit(1_000_003, "three")]; + + await runIngestCycle(db, configured, budget); + + expect(await getLastCursor(db)).toBe(1_000_002); + expect( + (await db.prepare(`SELECT rkey FROM records_event ORDER BY time_us`).all<{ rkey: string }>()) + .results?.map((row) => row.rkey), + ).toEqual(["one", "two"]); + + const firstSummary = output.lines.filter((line) => + line.text.startsWith("[ingest] cycle summary "), + ); + expect(firstSummary).toHaveLength(1); + expect(firstSummary[0]?.text).toContain('"stop_reason":"count"'); + expect(firstSummary[0]?.text).toContain('"exact_duplicates_dropped":1'); + expect(output.lines.some((line) => /candidate:|DUPLICATE/.test(line.text))).toBe(false); + + await runIngestCycle(db, configured, budget); + + expect(await getLastCursor(db)).toBe(1_000_003); + expect( + (await db.prepare(`SELECT rkey FROM records_event ORDER BY time_us`).all<{ rkey: string }>()) + .results?.map((row) => row.rkey), + ).toEqual(["one", "two", "three"]); + }); + + it("resumes before a capped coarse cursor without skipping or recounting siblings", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value); + source.events = [ + commit(1_100_000, "same-cursor-a"), + commit(1_100_000, "same-cursor-b"), + commit(1_100_001, "after"), + ]; + const oneCandidate = { ...budget, maxCandidates: 1 }; + + await runIngestCycle(db, configured, oneCandidate); + expect(await getLastCursor(db)).toBe(1_100_000); + expect( + (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) + .results?.map((row) => row.rkey), + ).toEqual(["same-cursor-a"]); + + await runIngestCycle(db, configured, oneCandidate); + expect(await getLastCursor(db)).toBe(1_100_000); + expect( + (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) + .results?.map((row) => row.rkey), + ).toEqual(["same-cursor-a", "same-cursor-b"]); + + await runIngestCycle(db, configured, oneCandidate); + expect(await getLastCursor(db)).toBe(1_100_001); + expect( + (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) + .results?.map((row) => row.rkey), + ).toEqual(["after", "same-cursor-a", "same-cursor-b"]); + expect(source.requestedCursors).toEqual([0, 1_099_999, 1_099_999]); + + const summaries = output.lines + .filter((line) => line.text.startsWith("[ingest] cycle summary ")) + .map((line) => + JSON.parse(line.text.slice("[ingest] cycle summary ".length)) as { + retained_candidates: number; + cursor_boundary_duplicates_dropped: number; + }, + ); + expect(summaries.map((summary) => summary.retained_candidates)).toEqual([ + 1, + 1, + 1, + ]); + expect(summaries[1]?.cursor_boundary_duplicates_dropped).toBe(1); + expect(summaries[2]?.cursor_boundary_duplicates_dropped).toBe(2); + }); + + it("persists Atcute's captured initial cursor after an empty first drain", async () => { + const db = createTestDb(); + const output = logger(); + source.initialCursor = 7_000_000; + + await runIngestCycle(db, config(output.value), budget); + + expect(await getLastCursor(db)).toBe(7_000_000); + const summary = output.lines.find((line) => + line.text.startsWith("[ingest] cycle summary "), + ); + expect(summary?.text).toContain('"starting_cursor":null'); + expect(summary?.text).toContain('"safe_ending_cursor":7000000'); + + // An event sharing the captured microsecond is not skipped by an exclusive + // resume API: the next cycle requests one microsecond earlier. + source.events = [commit(7_000_000, "between-empty-cycles")]; + await runIngestCycle(db, config(output.value), budget); + expect(source.requestedCursors).toEqual([7_000_000, 6_999_999]); + expect( + await db + .prepare("SELECT rkey FROM records_event WHERE rkey = ?") + .bind("between-empty-cycles") + .first<{ rkey: string }>(), + ).toEqual({ rkey: "between-empty-cycles" }); + }); + + it("rejects endpoint pools for scheduled ingestion before rollback can starve the cap", async () => { + const db = createTestDb(); + const output = logger(); + const configured = resolveConfig({ + ...config(output.value), + jetstreams: ["wss://one.test", "wss://two.test"], + }); + + await expect(runIngestCycle(db, configured, budget)).rejects.toThrow( + "scheduled ingestion requires exactly one pinned Jetstream endpoint", + ); + expect(source.requestedCursors).toHaveLength(0); + }); + + it("keeps admission diagnostics bounded as source volume grows", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value); + source.events = Array.from({ length: 20 }, (_, index) => ({ + ...commit(1_250_000 + index, `invalid-${index}`), + commit: { + ...commit(1_250_000 + index, `invalid-${index}`).commit, + record: `not-an-object-${index}`, + }, + })); + + await runIngestCycle(db, configured, { + ...budget, + maxCandidates: 20, + }); + + expect(output.lines.filter((line) => line.level === "warn")).toHaveLength(0); + const summaryLine = output.lines.find((line) => + line.text.startsWith("[ingest] cycle summary "), + ); + const summary = JSON.parse( + summaryLine!.text.slice("[ingest] cycle summary ".length), + ) as { + admission_policy_filtered: number; + diagnostic_samples: string[]; + diagnostic_samples_omitted: number; + }; + expect(summary.admission_policy_filtered).toBe(20); + expect(summary.diagnostic_samples).toHaveLength(5); + expect(summary.diagnostic_samples_omitted).toBe(15); + }); + + it("retains and orders two genuine revisions of one URI", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value); + source.events = [ + commit(1_500_001, "same"), + { + ...commit(1_500_002, "same"), + commit: { + ...commit(1_500_002, "same").commit, + cid: "bafy-newer", + record: { $type: COLLECTION, value: "newer" }, + }, + }, + ]; + + await runIngestCycle(db, configured, budget); + + const row = await db + .prepare("SELECT cid, record FROM records_event WHERE rkey = ?") + .bind("same") + .first<{ cid: string; record: string }>(); + expect(row?.cid).toBe("bafy-newer"); + expect(JSON.parse(row!.record)).toMatchObject({ value: "newer" }); + expect(await getLastCursor(db)).toBe(1_500_002); + }); + + it("checkpoints a filtered-only accounted range", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value, true); + source.events = [ + { + ...commit(2_000_001, "filtered"), + did: "unknown-actor", + commit: { + ...commit(2_000_001, "filtered").commit, + collection: "app.bsky.graph.follow", + }, + }, + ]; + + await runIngestCycle(db, configured, budget); + + expect(await getLastCursor(db)).toBe(2_000_001); + const summary = output.lines.find((line) => + line.text.startsWith("[ingest] cycle summary "), + ); + expect(summary?.text).toContain('"retained_candidates":0'); + expect(summary?.text).toContain('"source_scope_filtered":1'); + expect(summary?.text).toContain('"safe_ending_cursor":2000001'); + }); +}); diff --git a/packages/contrail/tests/schema-fingerprint-gate.test.ts b/packages/contrail/tests/schema-fingerprint-gate.test.ts index bd31c61..64b768f 100644 --- a/packages/contrail/tests/schema-fingerprint-gate.test.ts +++ b/packages/contrail/tests/schema-fingerprint-gate.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema, getMeta } from "../src/index"; +import { initSchema, getMeta, setMeta } from "../src/index"; import { resolveConfig } from "../src/index"; import type { Database, Statement } from "../src/index"; @@ -53,6 +53,24 @@ describe("schema fingerprint gate", () => { expect(await getMeta(real, "schema_fingerprint")).toBe(fp); // unchanged }); + it("installs cursor observations additively without rebuilding current projections", async () => { + const real = createSqliteDatabase(":memory:"); + await initSchema(real, CONFIG); + const current = (await getMeta(real, "schema_fingerprint"))!; + const previous = current.replace(/:cursor-observations-v1$/, ""); + await real.prepare("DROP TABLE cursor_observations").run(); + await setMeta(real, "schema_fingerprint", previous); + + const upgraded = recordingDb(real); + await initSchema(upgraded.db, CONFIG); + + expect( + upgraded.prepares.some((sql) => /CREATE TABLE IF NOT EXISTS cursor_observations/.test(sql)), + ).toBe(true); + expect(upgraded.prepares.some((sql) => /fts_event|records_event/.test(sql))).toBe(false); + expect(await getMeta(real, "schema_fingerprint")).toBe(current); + }); + it("re-applies when the generated schema changes (fingerprint busts)", async () => { const real = createSqliteDatabase(":memory:"); await initSchema(real, CONFIG); -- 2.51.2 From 95edda0ebc4c1087c0c744d69f480742cedf2f51 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:09:32 +0200 Subject: [PATCH 10/34] Scope test build dependency to Contrail --- packages/contrail/turbo.json | 9 +++++++++ turbo.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 packages/contrail/turbo.json diff --git a/packages/contrail/turbo.json b/packages/contrail/turbo.json new file mode 100644 index 0000000..8649e73 --- /dev/null +++ b/packages/contrail/turbo.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://turbo.build/schema.json", + "extends": ["//"], + "tasks": { + "test": { + "dependsOn": ["^build", "build"] + } + } +} diff --git a/turbo.json b/turbo.json index 2279a64..94b9292 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,7 @@ "dependsOn": ["^build"] }, "test": { - "dependsOn": ["^build", "build"], + "dependsOn": ["^build"], "outputs": [] }, "dev": { -- 2.51.2 From b707deff64d79ddd66bd85b1196450b70fb44b10 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:19:01 +0200 Subject: [PATCH 11/34] integrate feedback --- .../contrail/src/core/change-bootstrap.ts | 6 +- packages/contrail/src/core/changes.ts | 6 +- packages/contrail/src/core/delivery.ts | 2 +- .../contrail/tests/change-bootstrap.test.ts | 5 +- .../contrail/tests/change-consumers.test.ts | 10 ++- packages/contrail/tests/delivery.test.ts | 81 +++++++++++++++++++ 6 files changed, 100 insertions(+), 10 deletions(-) diff --git a/packages/contrail/src/core/change-bootstrap.ts b/packages/contrail/src/core/change-bootstrap.ts index 0cc6a97..2115bfc 100644 --- a/packages/contrail/src/core/change-bootstrap.ts +++ b/packages/contrail/src/core/change-bootstrap.ts @@ -443,6 +443,8 @@ async function failBootstrapLease( ) { throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); } + // Expiry permits reclamation; owner equality remains the CAS guard. Preserve + // failure/backoff when slow work expires but no replacement has claimed it. const failed = await db .prepare( `UPDATE change_consumers @@ -451,7 +453,6 @@ async function failBootstrapLease( last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_token = ? AND lease_owner = ? - AND lease_expires_at > ? RETURNING consumer_id`, ) .bind( @@ -463,12 +464,11 @@ async function failBootstrapLease( claim.generation, claim.bootstrapToken, claim.leaseOwner, - timestamp, ) .first<{ consumer_id: string }>(); if (!failed) { throw new ChangeLeaseLostError( - `Current bootstrap claim for ${claim.consumerId} is stale or expired`, + `Current bootstrap claim for ${claim.consumerId} is stale or no longer owned`, ); } } diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 3627147..0008639 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -854,6 +854,8 @@ export async function failChanges( ) { throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); } + // Expiry permits reclamation; owner equality remains the CAS guard. Preserve + // failure/backoff when slow work expires but no replacement has claimed it. const failed = await db .prepare( `UPDATE change_consumers @@ -862,7 +864,6 @@ export async function failChanges( last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? - AND lease_expires_at > ? RETURNING attempts, next_attempt_at`, ) .bind( @@ -874,12 +875,11 @@ export async function failChanges( claim.generation, claim.from, claim.leaseOwner, - now, ) .first<{ attempts: number | string; next_attempt_at: number | string | null }>(); if (!failed) { throw new ChangeLeaseLostError( - `Change claim for ${claim.consumerId} is stale or expired`, + `Change claim for ${claim.consumerId} is stale or no longer owned`, ); } return { diff --git a/packages/contrail/src/core/delivery.ts b/packages/contrail/src/core/delivery.ts index 65a2283..4a0ba26 100644 --- a/packages/contrail/src/core/delivery.ts +++ b/packages/contrail/src/core/delivery.ts @@ -314,7 +314,7 @@ async function runCurrent( if (before.state === "activating") { const claim = await state.changes.claimActivation( consumerId, - { now: state.clock() }, + { leaseMs: state.claim.leaseMs, now: state.clock() }, state.db, ); if (!claim) return "empty"; diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index 8a23e1b..c4733aa 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -211,7 +211,7 @@ describe("current-state change consumer bootstrap", () => { expect(ordinaryDelivery).not.toHaveProperty("destinationToken"); }); - it("persists snapshot failure backoff and resumes the same page", async () => { + it("persists expired snapshot failure backoff and resumes the same page", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config({ keeper, @@ -220,13 +220,14 @@ describe("current-state change consumer bootstrap", () => { await initSchema(db, resolved); await apply(db, resolved, event({ rkey: "a", time: 1 })); const page = await claimCurrentSnapshotPage(db, resolved, "search", { + leaseMs: 10, now: 100, }); await failCurrentSnapshotPage( db, page!, { code: "destination_unavailable", nextAttemptAt: 200 }, - { now: 101 }, + { now: 111 }, ); expect( await claimCurrentSnapshotPage(db, resolved, "search", { now: 150 }), diff --git a/packages/contrail/tests/change-consumers.test.ts b/packages/contrail/tests/change-consumers.test.ts index 71b6184..f1d1528 100644 --- a/packages/contrail/tests/change-consumers.test.ts +++ b/packages/contrail/tests/change-consumers.test.ts @@ -246,7 +246,7 @@ describe("durable change consumers", () => { }); }); - it("allows only one concurrent lease and rejects an expired owner", async () => { + it("allows only one concurrent lease and rejects a replaced owner", async () => { const db = createSqliteDatabase(":memory:"); const config = readyEventConsumer(); await initSchema(db, config); @@ -269,6 +269,14 @@ describe("durable change consumers", () => { await expect( acknowledgeChanges(db, stale, { now: 152 }), ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await expect( + failChanges( + db, + stale, + { code: "stale_failure", nextAttemptAt: 300 }, + { now: 152 }, + ), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); await acknowledgeChanges(db, replacement!, { now: 152 }); }); diff --git a/packages/contrail/tests/delivery.test.ts b/packages/contrail/tests/delivery.test.ts index 2835fe5..df983e3 100644 --- a/packages/contrail/tests/delivery.test.ts +++ b/packages/contrail/tests/delivery.test.ts @@ -192,6 +192,46 @@ describe("change delivery runtime", () => { }); }); + it("persists handler failure after the claim lease expires", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + webhook: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + + let current = 100; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { + webhook: async () => { + current = 151; + throw new Error("slow destination failure"); + }, + }, + runtime: { + maxRounds: 1, + claim: { leaseMs: 50 }, + baseRetryMs: 100, + maxRetryMs: 100, + jitter: 0, + clock: () => current, + }, + }); + + expect(result).toMatchObject({ delivered: 0, failures: 1 }); + expect((await getChangesStatus(db)).consumers[0]).toMatchObject({ + position: "0", + attempts: 1, + nextAttemptAt: 251, + lastErrorCode: "handler_error", + }); + }); + it("drives current snapshot, catch-up, and idempotent activation", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config({ @@ -261,6 +301,47 @@ describe("change delivery runtime", () => { }); }); + it("uses the configured lease for a slow activation", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + search: { collections: [EVENT], initial: "current" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + + let current = 100; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { search: async () => {} }, + bootstraps: { + search: { + snapshot: async () => {}, + activate: async (activation) => { + expect(activation.leaseExpiresAt).toBe(40_100); + current = 31_100; + }, + }, + }, + runtime: { + maxRounds: 6, + maxDurationMs: 60_000, + claim: { leaseMs: 40_000 }, + jitter: 0, + clock: () => current, + }, + }); + + expect(result).toMatchObject({ activations: 1, failures: 0 }); + expect(await contrail.changes.bootstrapStatus("search")).toMatchObject({ + state: "ready", + position: "1", + }); + }); + it("aborts destination work at the runtime deadline", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config({ -- 2.51.2 From 746367bc587d0b1c2752c2a7fd7cda2e0c9235fb Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:58:43 +0200 Subject: [PATCH 12/34] small fixes, update docs --- README.md | 17 +- apps/atmo-rsvp/README.md | 2 - apps/cloudflare-workers/README.md | 2 +- docs/00-getting-started.md | 32 ++ docs/01-client.md | 34 ++ docs/01-indexing.md | 184 ----------- docs/02-configure-and-query.md | 83 +++++ docs/02-querying.md | 195 ----------- docs/03-deploy-cloudflare.md | 101 ++++++ docs/04-feeds.md | 122 ------- docs/09-labels.md | 142 -------- docs/advanced/README.md | 9 + docs/advanced/feeds.md | 41 +++ docs/advanced/labels.md | 37 +++ docs/advanced/outbox.md | 94 ++++++ docs/frameworks/sveltekit-cloudflare.md | 213 ------------ docs/public-services/api-atmo-rsvp.md | 246 -------------- docs/public-services/creating.md | 311 ------------------ docs/public-services/using.md | 164 --------- packages/contrail/README.md | 1 - .../contrail/src/core/change-bootstrap.ts | 22 +- packages/contrail/src/core/change-log.ts | 1 + packages/contrail/src/core/changes.ts | 109 +++++- packages/contrail/src/core/db/schema.ts | 6 + .../contrail/tests/change-bootstrap.test.ts | 40 +++ .../contrail/tests/change-consumers.test.ts | 28 +- packages/contrail/tests/change-log.test.ts | 14 +- 27 files changed, 629 insertions(+), 1621 deletions(-) create mode 100644 docs/00-getting-started.md create mode 100644 docs/01-client.md delete mode 100644 docs/01-indexing.md create mode 100644 docs/02-configure-and-query.md delete mode 100644 docs/02-querying.md create mode 100644 docs/03-deploy-cloudflare.md delete mode 100644 docs/04-feeds.md delete mode 100644 docs/09-labels.md create mode 100644 docs/advanced/README.md create mode 100644 docs/advanced/feeds.md create mode 100644 docs/advanced/labels.md create mode 100644 docs/advanced/outbox.md delete mode 100644 docs/frameworks/sveltekit-cloudflare.md delete mode 100644 docs/public-services/api-atmo-rsvp.md delete mode 100644 docs/public-services/creating.md delete mode 100644 docs/public-services/using.md diff --git a/README.md b/README.md index 7c6c4f0..deaa28d 100644 --- a/README.md +++ b/README.md @@ -133,19 +133,16 @@ import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; ``` -The runnable [`apps/sqlite`](apps/sqlite) example wires the standard backfill CLI to a local SQLite file, including the optional Alluvium base/archive path. See [Indexing](docs/01-indexing.md) for adapter setup and [Querying](docs/02-querying.md) for the query and hydration model. +The runnable [`apps/sqlite`](apps/sqlite) example wires the standard backfill CLI to a local SQLite file, including the optional Alluvium base/archive path. ## Documentation -- [Indexing](docs/01-indexing.md) -- [Querying](docs/02-querying.md) -- [Feeds](docs/04-feeds.md) -- [Labels](docs/09-labels.md) -- Public Contrail services: - - [Creating a service](docs/public-services/creating.md) - - [Using a service](docs/public-services/using.md) - - [Example: api.atmo.rsvp](docs/public-services/api-atmo-rsvp.md) -- [SvelteKit + Cloudflare](docs/frameworks/sveltekit-cloudflare.md) +1. [Get started locally](docs/00-getting-started.md) +2. [Add the typed client](docs/01-client.md) +3. [Configure and query](docs/02-configure-and-query.md) +4. [Deploy to Cloudflare Workers](docs/03-deploy-cloudflare.md) + +See [advanced topics](docs/advanced/README.md) for other runtimes and features. ## Repository layout diff --git a/apps/atmo-rsvp/README.md b/apps/atmo-rsvp/README.md index bdb8cf5..83a75ae 100644 --- a/apps/atmo-rsvp/README.md +++ b/apps/atmo-rsvp/README.md @@ -92,5 +92,3 @@ pnpx @atmo-dev/contrail connect https://api.atmo.rsvp ``` That verifies the anonymous and service-auth contracts, verifies the canonical contract and Lexicon digests, writes a provider lock, installs provider-owned Lexicons, and runs Atcute TypeScript generation. Reconnecting an existing project requires `--update`. - -See [Example: api.atmo.rsvp](../../docs/public-services/api-atmo-rsvp.md) for the complete method, authentication, acquisition, and deployment walkthrough. diff --git a/apps/cloudflare-workers/README.md b/apps/cloudflare-workers/README.md index 114d5cf..7db07bc 100644 --- a/apps/cloudflare-workers/README.md +++ b/apps/cloudflare-workers/README.md @@ -71,4 +71,4 @@ This reports the durable state without exposing account DIDs or raw upstream err - **add a collection:** append to `collections` in `src/contrail.config.ts`; redeploy; `pnpm contrail backfill --remote` to backfill the new one. - **add full-text search:** `searchable: ["field1", "field2"]`, redeploy, no backfill needed (fts indexes repopulate on ingest). -- **add relations / references:** see [indexing docs](../../docs/01-indexing.md). +- **add relations / references:** see [configuration guide](../../docs/02-configure-and-query.md). diff --git a/docs/00-getting-started.md b/docs/00-getting-started.md new file mode 100644 index 0000000..2e5cad0 --- /dev/null +++ b/docs/00-getting-started.md @@ -0,0 +1,32 @@ +# Get started locally + +Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. + +Create an empty directory with one file: + +```ts +// contrail.config.ts +export default { + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { startsAt: { type: "range" } }, + }, + }, +}; +``` + +Then run: + +```bash +pnpx @atmo-dev/contrail dev +``` + +Contrail resolves the Lexicons, backfills existing records, follows new records, and serves a resumable SQLite AppView at `http://127.0.0.1:8787`. + +```bash +curl 'http://127.0.0.1:8787/xrpc/com.example.event.listRecords?limit=10' +``` + +Replace the collection and fields with your own. Next: [add the typed client to your app](./01-client.md). diff --git a/docs/01-client.md b/docs/01-client.md new file mode 100644 index 0000000..e498f93 --- /dev/null +++ b/docs/01-client.md @@ -0,0 +1,34 @@ +# Add the typed client + +With your [local AppView](./00-getting-started.md) set up, add Contrail and Atcute to your application: + +```bash +pnpm add @atmo-dev/contrail @atcute/client @atcute/lexicons +pnpx @atmo-dev/contrail connect ../my-appview +``` + +Point `connect` at the directory containing `contrail.config.ts`. It resolves the source and query Lexicons, then uses Atcute to generate a typed client in `src/contrail/`. + +Use it from your app: + +```ts +import { createLocalContrailClient } from "./contrail/index.js"; + +const contrail = createLocalContrailClient(); +const response = await contrail.get("com.example.event.listRecords", { + params: { + startsAtMin: new Date().toISOString(), + limit: 20, + }, +}); + +if (!response.ok) throw new Error(`Contrail returned ${response.status}`); + +for (const event of response.data.records) { + console.log(event.value.name, event.value.startsAt); +} +``` + +The method name, parameters, and response are all typed from the Lexicons. Re-run `connect` when the AppView config changes. + +Next: [configure filters, sorting, and hydration](./02-configure-and-query.md). To use an existing deployed AppView, pass its HTTPS URL to `contrail connect` instead of a config path. diff --git a/docs/01-indexing.md b/docs/01-indexing.md deleted file mode 100644 index f4117ba..0000000 --- a/docs/01-indexing.md +++ /dev/null @@ -1,184 +0,0 @@ -# Indexing - -Contrail's core job: mirror atproto records into your DB and expose them via XRPC. You describe what to index with a config object; everything else is automatic. - -## Collection shape - -A realistic two-collection example: events and RSVPs. RSVPs point at events via `subject.uri`; events expose per-status RSVP counts. - -```ts -collections: { - event: { - collection: "community.lexicon.calendar.event", // full NSID - queryable: { - mode: {}, // ?mode=online - startsAt: { type: "range" }, // ?startsAtMin=...&startsAtMax=... - }, - searchable: ["name", "description"], // FTS5 / tsvector - relations: { - rsvps: { - collection: "rsvp", // short name of the child collection - groupBy: "status", // field on the child record - groups: { - going: "community.lexicon.calendar.rsvp#going", - interested: "community.lexicon.calendar.rsvp#interested", - }, - }, - }, - }, - rsvp: { - collection: "community.lexicon.calendar.rsvp", - queryable: { status: {} }, - references: { - event: { collection: "event", field: "subject.uri" }, // RSVP's field → event's URI - }, - }, -} -``` - -- **queryable** — string equality or range, exposed as query params. -- **searchable** — FTS5 on D1/Postgres. Not available on `node:sqlite`. -- **relations** — many-to-one with materialized counts. The `event` collection gains `rsvpsCount`, `rsvpsGoingCount`, `rsvpsInterestedCount` columns — filter (`?rsvpsGoingCountMin=10`) and sort (`?sort=rsvpsGoingCount`) on them. Hydrate inline with `?hydrateRsvps=5`. -- **references** — forward lookups from child → parent. `?hydrateEvent=true` on an RSVP query embeds the referenced event record. - -## Backfill (historical data) - -Run once at setup to pull every record that exists today. - -```ts -await contrail.backfillAll({ concurrency: 100 }); // discover + backfill, logs progress -``` - -Under the hood this is two steps you can call separately if you want finer control: - -```ts -await contrail.discover(); // walk relays, register DIDs -await contrail.backfill({ - concurrency: 100, // identity resolution - pdsConcurrency: 20, // active PDS hosts - didsPerPds: 3, // accounts per active PDS -}); -``` - -`backfill()` picks up each account/collection at its saved PDS cursor. A row is marked complete only after the PDS listing reaches its end. Timeouts, failed identity resolution, `429`, and `5xx` responses leave the row pending with its last error. - -Each initial invocation attempts a failed account once by default, then gets out of the way. Failed rows retain their cursors and receive an exponential `next_retry_at`. Scheduled retries start at 15 minutes, double to a maximum of 48 hours, and stop after ten failed scheduled attempts. Cloudflare's scheduled Worker retries a small due slice after each live-ingest cycle; an explicit later invocation resets exhausted rows and forces another pass. `backfillAll()` returns a durable `status` summary alongside the number of discovered accounts and accepted records. - -Historical loading writes canonical records first, then rebuilds FTS and materialized relation counts with set-based SQL. A durable dirty marker keeps status `incomplete` if the process stops between those phases; the next manual or scheduled backfill repairs the projections before reporting readiness. Live ingestion and scheduled account retries continue maintaining both projections incrementally. - -### Workers CLI - -For Cloudflare Workers deploys, `@atmo-dev/contrail` ships a `contrail` bin that handles the `wrangler.getPlatformProxy` dance — no script file, no package.json alias needed: - -```bash -pnpm contrail backfill # local D1 (wrangler dev's bindings) -pnpm contrail backfill --remote # production D1 -``` - -Auto-detects configs at `contrail.config.ts`, `src/contrail.config.ts`, `src/lib/contrail.config.ts`, or `app/contrail.config.ts` (first match wins). Override with `--config `. Other flags include `--binding ` (default `DB`), `--concurrency ` for identity resolution (default 100), `--pds-concurrency ` (default 20), `--dids-per-pds ` (default 3), and `--max-attempts ` (default 1). Once every known account has either completed or received a deferred failure, the initial pass is complete and scheduled retries continue in the background. Interrupted or undiscovered work still reports the pass as incomplete. - -If you'd rather embed backfill inside your own script, `@atmo-dev/contrail/workers` exports the same logic as a function: - -```ts -import { backfillAll } from "@atmo-dev/contrail/workers"; -import { config } from "../src/contrail.config"; - -await backfillAll({ config, remote: process.argv.includes("--remote") }); -``` - -For node/postgres deploys, skip both — you already have a `db` in hand; just `await contrail.backfillAll({}, db)` directly. - -## Ingestion (ongoing new records) - -After the initial `backfillAll()`, keep the index fresh with new records as they're published. Pick the mode that matches your runtime. - -### Cron-driven (cloudflare workers) - -Workers can't hold long-lived connections, so run one catch-up cycle per cron fire: - -```ts -// wrangler.jsonc: "triggers": { "crons": ["*/1 * * * *"] } -async scheduled(_ev, env, ctx) { - ctx.waitUntil((async () => { - await contrail.ingest({}, env.DB); - await contrail.retryBackfill({}, env.DB); // small due slice - })()); -} -``` - -`ingest()` connects to Jetstream, streams events since the saved cursor, stops when caught up. Running every minute is fine — the next fire resumes where this one left off. Each cycle is bounded, so it can't blow past the Worker time limit. - -**Local dev:** wrangler's cron scheduler only runs in deployed production. For local dev use `pnpm contrail dev` — it runs `wrangler dev --test-scheduled`, fires `/__scheduled` on your configured cron interval, and offers to start or resume backfill whenever known work remains. - -### Persistent (node / any long-lived server) - -If your runtime can keep a socket open, skip the cron entirely: - -```ts -const ac = new AbortController(); -await contrail.runPersistent({ - batchSize: 50, // flush every N events (default: 50) - flushIntervalMs: 5000, // or every N ms, whichever first - signal: ac.signal, -}); -// ac.abort() flushes the current batch and saves the cursor before returning -``` - -One process, one socket, auto-reconnect on drops. Lower latency than cron mode (events land within seconds instead of up-to-a-minute), but needs a runtime that can run indefinitely. - -### Immediate (`notify()`) - -Use this when your own app writes to a user's PDS and needs the change indexed *now* — waiting for the next cron / Jetstream flush is too slow: - -```ts -await contrail.notify(uri); // one record -await contrail.notify([u1, u2, u3]); // batch, up to 25 -``` - -Fetches directly from the user's PDS and indexes synchronously. When Jetstream later delivers the same event, the duplicate is detected by CID and skipped. - -### Which one do I use? - -| | backfillAll | ingest | runPersistent | notify | -|---|---|---|---|---| -| when | once, at setup | every cron fire | start once, runs forever | per-write, on demand | -| runtime | local script | cloudflare workers | node / long-lived server | anywhere | -| scope | all historical records | events since last cursor | events since last cursor, live | specific URIs | -| latency | — | ~minute | ~seconds | immediate | - -Typical combos: -- **workers app:** `backfillAll()` once + `ingest()` on cron + optional `notify()` for self-writes -- **node server:** `backfillAll()` once + `runPersistent()` forever + optional `notify()` for self-writes - -## Recovery after an outage - -Normal ingestion resumes from its saved Jetstream cursor, so a short outage needs no special command: restart `ingest()` or `runPersistent()` and let it catch up. - -Contrail does not perform a full PDS sweep as a repair mechanism. Such a sweep is expensive, cannot discover repositories it never knew about, and cannot safely infer remote deletions after partial failures. If the saved cursor is older than the source's retained history, rebuild into a fresh database with `backfillAll()` rather than trusting a partial reconciliation. A replay-capable source and first-class projection rebuild command are planned follow-up work. - -Reading the indexed data — filters, sorts, hydration, search, pagination — has its own doc: [Querying](./02-querying.md). - -## Adapters - -| Adapter | Use when | FTS | -|---|---|---| -| Cloudflare D1 | Workers | ✅ | -| `@atmo-dev/contrail/sqlite` | Node 22+ local dev | ❌ | -| `@atmo-dev/contrail/postgres` | Node server | ✅ | - -```ts -import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; -const db = createPostgresDatabase(pool); -``` - -## Top-level config - -| Key | Default | | -|---|---|---| -| `namespace` | — | Reverse-domain for XRPC paths | -| `profiles` | `["app.bsky.actor.profile"]` | Profile NSIDs, auto-hydrated via `?profiles=true` | -| `jetstreams` | Bluesky | Jetstream URLs | -| `relays` | Bluesky | Relay URLs for discovery | -| `notify` | off | Prefer an in-process call or a secret string requiring `Bearer`; open `true` mode is not recommended | -| `feeds` | — | See [Feeds](./04-feeds.md) | -| `labels` | — | See [Labels](./09-labels.md) | diff --git a/docs/02-configure-and-query.md b/docs/02-configure-and-query.md new file mode 100644 index 0000000..2b5e27f --- /dev/null +++ b/docs/02-configure-and-query.md @@ -0,0 +1,83 @@ +# Configure and query + +Each entry in `collections` becomes typed `getRecord` and `listRecords` methods. Declare only the fields and relationships your app needs: + +```ts +// contrail.config.ts +export default { + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { + mode: {}, + startsAt: { type: "range" }, + }, + searchable: ["name", "description"], + relations: { + rsvps: { + collection: "rsvp", + groupBy: "status", + groups: { + going: "community.lexicon.calendar.rsvp#going", + }, + }, + }, + }, + rsvp: { + collection: "community.lexicon.calendar.rsvp", + queryable: { + status: {}, + "subject.uri": {}, + }, + references: { + event: { collection: "event", field: "subject.uri" }, + }, + }, + }, +}; +``` + +This produces the following client parameters: + +| Config | Query parameter | +|---|---| +| `mode: {}` | `mode` | +| `startsAt: { type: "range" }` | `startsAtMin`, `startsAtMax` | +| `searchable` | `search` | +| `relations.rsvps` | `rsvpsCountMin`, `hydrateRsvps` | +| `groups.going` | `rsvpsGoingCountMin` | +| `references.event` | `hydrateEvent` | + +Dotted fields become camel case: `subject.uri` becomes `subjectUri`. Every list method also supports `actor` (a DID or handle), `sort`, `order`, `limit`, `cursor`, and `profiles`. + +## Query from the client + +After changing the config, re-run `contrail connect` in your app. The generated Atcute client now knows the new parameters and response types: + +```ts +const response = await contrail.get("com.example.event.listRecords", { + params: { + mode: "in-person", + startsAtMin: new Date().toISOString(), + rsvpsGoingCountMin: 5, + sort: "startsAt", + order: "asc", + hydrateRsvps: 3, + profiles: true, + limit: 20, + }, +}); + +if (!response.ok) throw new Error(`Contrail returned ${response.status}`); + +const { records, profiles, cursor } = response.data; +``` + +Every record has `uri`, `cid`, and its original record body in `value`. Hydrated relations and references are added to that record; requested profiles are returned once in the top-level `profiles` array. + +Pass the returned opaque `cursor` into the same query to get the next page. `limit` defaults to 50 and may be 1–200. + +Full-text `search` works with D1 and PostgreSQL. The zero-config local SQLite AppView does not provide full-text search. + +Next: [deploy to Cloudflare Workers](./03-deploy-cloudflare.md). diff --git a/docs/02-querying.md b/docs/02-querying.md deleted file mode 100644 index 46dac29..0000000 --- a/docs/02-querying.md +++ /dev/null @@ -1,195 +0,0 @@ -# Querying - -Once [indexing](./01-indexing.md) is set up, every collection you declared gets a pair of XRPC endpoints under `/xrpc/{namespace}.{short}.*`: - -| Endpoint | Returns | -|---|---| -| `{namespace}.{short}.listRecords` | Paginated list with filters, sorts, hydration | -| `{namespace}.{short}.getRecord?uri=…` | Single record by AT-URI | - -Top-level methods include `{namespace}.getProfile`, `{namespace}.getCursor`, `{namespace}.notifyOfUpdate`, and optionally `{namespace}.getFeed` and `{namespace}.lexicons`. - -## HTTP (what most callers use) - -Every config field becomes a predictable URL param: - -``` -/xrpc/com.example.event.listRecords?mode=online&startsAtMin=2026-01-01&rsvpsGoingCountMin=10&sort=startsAt&order=asc&hydrateRsvps=5 -/xrpc/com.example.event.getRecord?uri=at://did:plc:.../...&hydrateRsvps=5 -``` - -| Config produces | URL param | -|---|---| -| `queryable: { field: {} }` | `?field=value` (equality) | -| `queryable: { field: { type: "range" } }` | `?fieldMin=…`, `?fieldMax=…` | -| `relations: { rel: {...} }` | `?relCountMin=N`, `?sort=relCount`, `?hydrateRel=N` | -| `relations: { rel: { groups: { going } } }` | `?relGoingCountMin=N`, `?sort=relGoingCount` | -| `references: { ref: {...} }` | `?hydrateRef=true` | - -Dotted field names become camelCase params — `queryable: { "subject.uri": {} }` → `?subjectUri=…`. - -## Operational status - -`GET /status` returns the current JSON overview. It includes indexed record totals, live-ingest freshness, and durable backfill state: - -- discovery source progress; -- mutually exclusive account totals for `complete`, `pending`, `retrying`, and `failed`; -- known-account completion percentage; -- the same mutually exclusive totals per collection; and -- scheduled/due account retries plus the next retry time. - -`state` is `running` while a manual or scheduled slice holds the backfill lease. It becomes `complete` after discovery and the initial pass finish, even when account-level `retrying` or `failed` counts are non-zero. `pending` means no failure has occurred yet, `retrying` means at least one attempt failed but automatic attempts remain, and `failed` means the ten-attempt automatic budget is exhausted. An explicit backfill resets that budget. `incomplete` means discovery or an initial account attempt has not finished. - -"Known" is deliberate: while relay discovery is incomplete, Contrail cannot honestly claim how many accounts remain undiscovered. `/health` remains a lightweight liveness response and does not claim that historical backfill is complete. - -`GET /xrpc/{namespace}.getCursor` returns the committed primary ordered-source position when `orderedSource` is configured. The `{ source, epoch, cursor }` tuple is opaque: compare complete tuples for equality only, and treat a source or epoch change as a full reset. A consumer that needs a stable query snapshot can read the position before and after its query and retry when the two positions differ. - -## Programmatic - -```ts -const { records, cursor } = await contrail.query("event", { - filters: { mode: "online" }, - rangeFilters: { startsAt: { min: "2026-01-01" } }, - countFilters: { rsvp: 10 }, // keyed by child collection short name - sort: { recordField: "startsAt", direction: "asc" }, - limit: 20, -}); -``` - -The programmatic shape doesn't use the URL param names — keys are the underlying field/collection identifiers: - -- `filters` / `rangeFilters` — keyed by the field name from your config (`startsAt`, `subject.uri`), not the camelCased URL param. -- `countFilters` — keyed by the target collection's short name for totals, or by the full `nsid#group` token for group counts. E.g., `{ rsvp: 10 }` for "at least 10 RSVPs total," or `{ "community.lexicon.calendar.rsvp#going": 10 }` for "at least 10 going." -- `sort` — `{ recordField, direction }` for field sorts, `{ countType, direction }` for count sorts (where `countType` is the same collection-short-name or `nsid#group` as above). Field sorts preserve the SQL value type, and records whose field is missing or `null` sort last in either direction. - -For count filters / sorts, the HTTP side is nicer than the programmatic side — consider going through `createHandler` + `fetch` even for in-process calls if you want the friendly names. Or use `createServerClient` from `@atmo-dev/contrail/server` for a typed XRPC client that runs in-process (no fetch roundtrip). - -## Pagination - -``` -?limit=25&cursor= -``` - -`cursor` is opaque — pass back whatever `listRecords` returned in its `cursor` field. `limit` is 1–200 (default 50). Cursors embed the complete ordering, including relevance rank for search and URI as the final unique tiebreaker. They also embed the sort kind, so a cursor from a `sort=startsAt` query is ignored by a `sort=rsvpsCount` query instead of silently returning wrong results. - -```ts -let cursor: string | undefined; -do { - const page = await contrail.query("event", { limit: 100, cursor }); - // process page.records - cursor = page.cursor; -} while (cursor); -``` - -## Hydration - -Each record response is a flat shape: - -```jsonc -{ - "uri": "at://did:plc:.../community.lexicon.calendar.event/...", - "cid": "...", - "value": { "name": "Rust meetup", "startsAt": "2026-03-16T...", ... }, - "rsvpsCount": 42, // from relations - "rsvpsGoingCount": 30, - // relations + references appear here only when hydrated -} -``` - -The `value` field carries the record body — same shape as atproto's `com.atproto.repo.listRecords#record`. `did`, `collection`, `rkey`, and `time_us` are also returned alongside as optional extras. - -### `?hydrateRel=N` (relations) - -Embeds the latest N child records per group, inline under the parent: - -``` -/xrpc/com.example.event.listRecords?hydrateRsvps=5 -``` - -Returns: - -```jsonc -{ - "records": [{ - "uri": "at://.../event/...", - "value": { "name": "..." }, - "rsvpsCount": 42, - "rsvps": { - "going": [ {uri, cid, value}, ... 5 items ], - "interested":[ {uri, cid, value}, ... 5 items ] - } - }] -} -``` - -Max 50 per group. For grouped relations you get one array per group value; for ungrouped relations just a flat array. - -### `?hydrateRef=true` (references) - -Embeds the single referenced parent record — useful for RSVP lists that need to show event details: - -``` -/xrpc/com.example.rsvp.listRecords?subjectUri=at://.../event/...&hydrateEvent=true -``` - -Each RSVP record in the response gains an `event: {uri, cid, value}` field. - -### `?profiles=true` - -Opt in to profile + handle hydration for every DID referenced in the result: - -``` -/xrpc/com.example.event.listRecords?profiles=true -``` - -Response grows a top-level `profiles` array, one entry per (DID, configured profile NSID): - -```jsonc -{ - "records": [...], - "profiles": [ - { - "did": "did:plc:alice...", - "handle": "alice.bsky.social", - "uri": "at://did:plc:alice.../app.bsky.actor.profile/self", - "cid": "...", - "collection": "app.bsky.actor.profile", - "rkey": "self", - "value": { /* profile record body */ } - } - ] -} -``` - -A DID with no profile record (or whose handle resolved but profile didn't) shows up as a bare `{ did, handle }` entry — `uri`/`cid`/`value` are omitted. With multiple profile NSIDs configured, you'll see one entry per (DID × NSID) that resolved. - -Which profile NSID(s) to hydrate from is configured at the top level of Contrail's config (`profiles`, defaults to `["app.bsky.actor.profile"]`). - -## Full-text search - -``` -?search=meetup -?search=meetup* -?search="rust meetup" -?search=rust OR typescript -``` - -Combinable with every other filter and sort. Backed by SQLite FTS5 (D1) or Postgres tsvector (Postgres adapter). Not available on `node:sqlite` — that adapter doesn't ship FTS5. - -When searching, results are ranked by relevance by default. Override with an explicit `sort` param. - -## Examples - -``` -# Upcoming events with 10+ going RSVPs, with RSVP records + profiles -?startsAtMin=2026-03-16&rsvpsGoingCountMin=10&hydrateRsvps=5&profiles=true - -# Events for a specific user (by handle — triggers on-demand backfill) -?actor=alice.bsky.social&profiles=true - -# RSVPs for one event, with the event record embedded -?subjectUri=at://did:plc:.../event/...&hydrateEvent=true&profiles=true - -# Search + filter + sort -?search=meetup&mode=online&sort=startsAt&order=asc -``` diff --git a/docs/03-deploy-cloudflare.md b/docs/03-deploy-cloudflare.md new file mode 100644 index 0000000..5252755 --- /dev/null +++ b/docs/03-deploy-cloudflare.md @@ -0,0 +1,101 @@ +# Deploy to Cloudflare Workers + +Turn the same local AppView into a public Worker backed by D1. + +## Install + +From the directory containing `contrail.config.ts`: + +```bash +pnpm init +pnpm add @atmo-dev/contrail +pnpm add -D @atcute/lex-cli wrangler typescript +``` + +Add an ordered source to the config. Keep its epoch stable for the lifetime of this database: + +```ts +// contrail.config.ts +import type { ContrailConfig } from "@atmo-dev/contrail"; + +const config: ContrailConfig = { + namespace: "com.example", + orderedSource: { + source: "jetstream", + epoch: "my-appview-v1", + }, + collections: { + // ...your existing collections + }, +}; + +export default config; +``` + +Generate the public query Lexicons and their referenced record Lexicons: + +```bash +pnpm contrail lexicons all --public +``` + +## Create the Worker + +```ts +// worker.ts +import { createWorker } from "@atmo-dev/contrail/worker"; +import config from "./contrail.config"; +import { lexicons } from "./lexicons/generated"; + +export default createWorker(config, { + lexicons, + publicService: { + endpoint: "https://my-appview.example.com", + }, +}); +``` + +Use the Worker's actual `workers.dev` or custom-domain URL as `endpoint`. + +Create `wrangler.jsonc`: + +```jsonc +{ + "name": "my-appview", + "main": "worker.ts", + "compatibility_date": "2025-12-25", + "observability": { "enabled": true }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "my-appview", + "database_id": "PASTE_DATABASE_ID_HERE" + } + ], + "triggers": { "crons": ["*/1 * * * *"] } +} +``` + +## Deploy and backfill + +```bash +pnpm wrangler d1 create my-appview # copy its ID into wrangler.jsonc +pnpm wrangler deploy +pnpm contrail backfill --remote +``` + +The one-minute cron keeps the AppView current. Check it with: + +```bash +curl https://my-appview.example.com/status +curl 'https://my-appview.example.com/xrpc/com.example.event.listRecords?limit=10' +``` + +Finally, point the application from the previous guide at the deployment: + +```bash +pnpx @atmo-dev/contrail connect https://my-appview.example.com +``` + +When the config changes, regenerate the Lexicons, deploy, backfill any new collections, and reconnect the client with `--update`. + +For optional outbox deliveries, feeds, and labels, see [advanced topics](./advanced/README.md). diff --git a/docs/04-feeds.md b/docs/04-feeds.md deleted file mode 100644 index 0f5c5e1..0000000 --- a/docs/04-feeds.md +++ /dev/null @@ -1,122 +0,0 @@ -# Feeds - -Personalized "what the people I follow are doing" timelines, fanned out at write time. Opt-in; no cost if you don't enable it. - -## Mental model - -> A feed is a (follow-collection, [target-collections]) pair, named by you. Every time someone an *actor* follows posts to a target collection, contrail inserts one row into `feed_items` for that actor. - -Reading a feed is a join through `feed_items` plus the standard pipeline (filters, sorts, hydration, references). The actor parameter on a read is *whose feed* you want — there is no anonymous feed read. - -## Enable - -```ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -const config: ContrailConfig = { - namespace: "com.example", - collections: { - follow: { collection: "app.bsky.graph.follow" }, - post: { collection: "app.bsky.feed.post", queryable: { /* ... */ } }, - }, - feeds: { - timeline: { - follow: "follow", // short name (key in `collections`), NOT the NSID - targets: ["post"], - maxItems: 500, // optional, default 200 - }, - }, -}; -``` - -Both the follow collection and every target collection must be declared in `collections`. Names in `feeds` are the **short names** (the keys of `collections`), not NSIDs. Config validation throws if you reference an unknown short name. - -## Follow-record shape - -The follow collection's record must have a `subject` field at the top level whose value is the followed DID. `app.bsky.graph.follow` matches this naturally: - -```json -{ "subject": "did:plc:abc...", "createdAt": "2026-01-01T00:00:00Z" } -``` - -Custom follow lexicons work as long as `subject` is the followed DID at JSON path `$.subject`. Contrail extracts via that path during ingest fan-out and during follow-event backfill. - -## Schema - -Two tables, one shared across all feeds: - -| Table | Purpose | -|---|---| -| `feed_items (actor, uri, collection, time_us)` | One row per (viewer, target record). Primary key `(actor, uri)` so a single target record can appear in many feeds. | -| `feed_backfills (actor, feed, completed)` | Marker so first-read backfill only runs once per (actor, feed). | - -Indexes: `(actor, collection, time_us DESC)` and `(actor, time_us DESC)` on `feed_items`, plus a JSON `subject` index on each follow collection's records table for the fan-out join. - -## Read - -``` -GET /xrpc/{namespace}.getFeed?feed=timeline&actor=&limit=50 -``` - -| Param | Meaning | -|---|---| -| `feed` | Feed name from `config.feeds` (required) | -| `actor` | Whose feed — DID or handle (required) | -| `collection` | Restrict to one target collection's short name (default: first in `targets`) | -| `limit`, `cursor`, filters from the target's `queryable`, hydration flags, sort/order | Same as `listRecords` on the target collection | - -The `actor` parameter is **whose feed** you're reading, not a filter on record creator. Feeds are always per-user. - -```ts -const feed = await fetch( - `/xrpc/com.example.getFeed?feed=timeline&actor=${did}&limit=50&profiles=true` -).then((r) => r.json()); -// feed.records — target records by users `actor` follows, newest first -// feed.profiles — hydrated profile records for record authors -``` - -## How fan-out works - -Three moments: - -1. **A target write.** Someone followed by N actors posts to a target collection. Contrail inserts N `feed_items` rows in one statement (`INSERT … SELECT … FROM WHERE subject = ?`). Cost is linear in N — there is no max-followers cap; a viral author with 1M followers is 1M inserts. - -2. **A follow write.** An actor follows a new user. Contrail backfills the most recent **100** target records from that user into the new follower's feed. The 100 is hardcoded in `core/router/feed.ts` — separate from the per-feed `maxItems` cap, and not tunable per feed today. - -3. **First read for an (actor, feed) pair.** Contrail backfills the actor's follow records from their PDS (so their `feed_items` rows can be computed), then populates `feed_items` from existing target records by users they already follow. Marked complete in `feed_backfills` so it runs once per pair. - -## Pruning - -Feeds are capped: each actor keeps at most `maxItems` rows per target collection (default 200, newest first). Older rows past the cap are deleted by a background cleanup that piggybacks on ingestion — there is no separate prune job. - -A few terms used below: - -- **Tick** — one cycle of the ingest loop. In cron mode the worker wakes on a schedule (e.g. once a minute) and each wake-up is a tick; in the persistent loop it's each batch flush. -- **Sweep** — the cleanup that walks `feed_items` actor by actor and deletes whatever is over an actor's cap. -- **Slice** — a sweep doesn't scan the whole table at once. Each tick it handles a chunk of up to `FEED_PRUNE_SWEEP_ACTORS` actors (default 500). That chunk is one slice. -- **Cursor / full pass** — a bookmark for the last actor a slice stopped on, so the next slice resumes after it instead of restarting. When the cursor reaches the last actor it *wraps* back to the start; one start-to-end trip is a *full pass*. - -**When the sweep runs.** A feed can only go over its cap right after a feed-mutating record (a target fan-out or a follow backfill) is applied, so the sweep is skipped entirely on ticks that ingested nothing feed-relevant. It runs when the current tick — a cron run, a persistent-loop flush, or a `notifyOfUpdate` call — applied a feed-mutating record. As a safety net it also runs on a recovery interval (`FEED_PRUNE_RECOVERY_INTERVAL_MS`, 6h), so rows that went over cap without a fresh ingest (a lowered cap, a bulk import) still get cleaned up — including on a stream that is otherwise idle. - -Doing one slice per tick keeps each tick's cost flat no matter how big the table grows. The recovery timer measures from the last *completed full pass* (not the last slice): a fresh pass becomes due one recovery interval after the previous one finished, then advances a slice per tick until the cursor wraps. So a full pass *completes* roughly every `recovery interval + lap time`, where lap time is `ceil(actors / FEED_PRUNE_SWEEP_ACTORS)` ticks — e.g. with 100k actors and one-minute cron ticks, ~6h + ~3h20m. That keeps the whole table draining on a bounded cadence; it is not a hard "fully clean every 6h" guarantee. Raise `FEED_PRUNE_SWEEP_ACTORS` if you need the lap time shorter at large actor counts. - -**Fan-out isn't cleaned up instantly.** A slice cleans up whatever actors the cursor lands on next — not specifically the actors whose feeds just changed. So when a popular author posts and fans out to many followers: a follower the cursor *hasn't reached yet* this pass is trimmed later in the same pass (soon), but a follower the cursor has *already passed* waits for the next pass — and on a quiet stream the next pass only starts on the recovery interval. So the worst case for an over-cap follower is roughly one recovery interval (`FEED_PRUNE_RECOVERY_INTERVAL_MS`, 6h), not the next tick. - -This is on purpose: an author can have unboundedly many followers, and trimming every one on the spot would either overrun the per-tick request budget (one delete per follower) or overrun D1's per-query CPU limit (one big delete over all of them, which can reset the shared Durable Object). `feed_items` is just a cache, so a follower sitting a little over cap for up to an interval does no harm. Deployments with fewer than `FEED_PRUNE_SWEEP_ACTORS` (500) distinct feed actors clean the whole table on every triggered tick, so they never see this lag at all. Pruning the touched actors directly (instead of the rolling cursor) would remove the lag but trade the bounded per-tick cost for cost proportional to fan-out size; see the issue tracker for that trade-off. - -## Deletes - -Deleting a target record removes its `feed_items` rows across all actors. Deleting a follow record currently does not retroactively prune the feed_items inserted during the original follow backfill — they age out via the global pruner instead. - -## XRPCs - -- `{namespace}.getFeed` — read - -That's it. Feeds are read-only over XRPC; writes to follow / target collections happen through `com.atproto.repo.putRecord` on the user's PDS as normal, and Jetstream ingestion drives the fan-out. - -## What's not here - -- No per-feed prune cap; the global pruner uses the largest `maxItems` across all feeds. -- The 100-record backfill on a new follow is hardcoded — not tunable per feed. -- No max-followers cap on target writes — a target record by a user with 1M followers means 1M `feed_items` inserts. For apps expecting that scale, partition feeds or rate-limit upstream. -- Feeds contain only public target records. diff --git a/docs/09-labels.md b/docs/09-labels.md deleted file mode 100644 index d34b702..0000000 --- a/docs/09-labels.md +++ /dev/null @@ -1,142 +0,0 @@ -# Labels - -Atproto-native moderation hydration. Subscribe to one or more labelers, index their labels, and attach them to records and profiles in your XRPC responses. Opt-in; zero cost if you don't enable it. - -## Mental model - -> A **label** is a `(src, uri, val)` triple authored by a labeler DID. A **labeler** is a regular atproto account that publishes signed annotations about other accounts and records via `com.atproto.label.subscribeLabels`. - -- One contrail deployment can subscribe to many labelers. -- The caller of your XRPC picks which subset to honor per request via the `atproto-accept-labelers` header (or `?labelers=` query param when headers are awkward — SSE/WS). -- Labels hydrate onto every `listRecords`, `getRecord`, `getProfile`, and `?profiles=true` response without changing your collection config. -- This module only consumes labels. Producing them — your appview emitting its own labels — is a separate question. See *Future work* below. - -## Enable - -```ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -const config: ContrailConfig = { - namespace: "com.example", - collections: { /* ... */ }, - labels: { - sources: [ - { did: "did:plc:ar7c4by46qjdydhdevvrndac" }, // bsky moderation - { did: "did:plc:newsmast" }, - ], - }, -}; -``` - -`initSchema` creates a `labels` table and a `labeler_cursors` table. Both live on the main DB; nothing per-collection. - -## Caller selection - -Per request, contrail picks accepted labelers in this order: - -1. `atproto-accept-labelers: did:plc:a, did:plc:b` — the spec's HTTP header. -2. `?labelers=did:plc:a,did:plc:b` — fallback for transports that can't set headers easily. -3. `config.labels.defaults` — operator policy. -4. Every entry in `config.labels.sources`. - -The list is intersected with what's actually configured (unknowns dropped — only labelers we've subscribed to have rows to hydrate from) and capped at `maxPerRequest` (default 20). Contrail echoes the applied set back via `atproto-content-labelers`. - -``` -GET /xrpc/com.example.event.listRecords - atproto-accept-labelers: did:plc:ar7c4by46qjdydhdevvrndac -``` - -→ - -```jsonc -// Response: atproto-content-labelers: did:plc:ar7c4by46qjdydhdevvrndac -{ - "records": [ - { - "uri": "at://did:plc:.../com.example.event/...", - "value": { /* ... */ }, - "labels": [ - { - "src": "did:plc:ar7c4by46qjdydhdevvrndac", - "uri": "at://did:plc:.../com.example.event/...", - "val": "spam", - "cts": "2026-04-25T00:00:00.000Z" - } - ] - } - ] -} -``` - -`labels` matches `com.atproto.label.defs#label` field-for-field — pass it straight to atproto SDK moderation helpers. - -### `defaults: []` - -Set defaults to an empty array if you want strict opt-in: callers that send no header / param see no labels at all. - -## Hydration semantics - -For each `(src, uri, val)` tuple visible to the caller, hydration picks the row with the highest `cts`. If that row has `neg=true`, the label is treated as retracted and dropped. Expired rows (`exp` past `now`) are filtered at the SQL level. CID-pinned labels apply only when the indexed record's CID matches. - -Account-level labels (subject = bare DID) hydrate onto profiles. They appear inside each `ProfileEntry.labels` of the `profiles` array on `?profiles=true` responses, and on `getProfile`. - -## Ingestion - -`com.atproto.label.subscribeLabels` is a per-labeler WebSocket firehose with a CBOR frame envelope. Contrail mirrors its existing Jetstream pipeline: - -| Mode | Function | When | -|---|---|---| -| Cron-driven | `contrail.ingestLabels()` | Cloudflare Workers — one drain per cron tick | -| Persistent | `contrail.runPersistentLabels()` | Node / long-lived servers — one socket per labeler, auto-reconnect | -| One-shot backfill | `pnpm contrail backfill --only labels [--remote]` | Local script, drains until each labeler reports caught up. (`pnpm contrail backfill` runs both records and labels.) | - -When `config.labels` is set, the bundled `createWorker` already calls `ingestLabels()` from `scheduled()` alongside `ingest()` — no boilerplate. - -```ts -// node / long-lived -const ac = new AbortController(); -await Promise.all([ - contrail.runPersistent({ signal: ac.signal }), - contrail.runPersistentLabels({ signal: ac.signal }), -]); -``` - -Per-labeler cursors live in `labeler_cursors` (`{did, cursor, endpoint, resolved_at}`). Endpoints are resolved from the DID doc's `service[id="#atproto_labeler"]` and cached for 6h. On `#info { name: "OutdatedCursor" }` frames, contrail resets the cursor to `0` so the next cycle re-backfills. - -### `backfill: false` - -Per source. Default: backfill from `cursor=0` on first sight. Set `false` to start at "now" — useful for very chatty labelers where you don't need history. - -```ts -labels: { - sources: [{ did: "did:plc:somenoisylabeler", backfill: false }], -} -``` - -## Storage - -```sql -CREATE TABLE labels ( - src TEXT NOT NULL, -- labeler DID - uri TEXT NOT NULL, -- subject: at://... or did:... - val TEXT NOT NULL, -- label value - cid TEXT, -- optional record-version pin - neg INTEGER NOT NULL DEFAULT 0, - exp INTEGER, -- expiry, unix sec - cts INTEGER NOT NULL, -- creation time, unix sec - sig BLOB, -- signature bytes (stored, not verified in v1) - PRIMARY KEY (src, uri, val, cts) -); -``` - -The PK includes `cts`, so a `neg=true` retraction is a *new row* that replaces the previous decision via the read-time collapse rule above — never an in-place mutation. This matches the spec, tolerates out-of-order delivery, and survives a labeler that flip-flops. - -## What's not here - -- **Signature verification.** `sig` is stored if the labeler supplies it, but contrail does not verify it in v1. Document as TODO; most appviews skip it. -- **Outbound `subscribeLabels`.** Contrail consumes labels but does not act as a labeler or republish them. -- **Label definitions / preferences UX.** Custom label names, blur behaviors, severity, and per-user preference state belong on the *client*, fetched directly from each labeler. Contrail intentionally stays out of this. - -## Design - -Follows the [atproto label spec](https://atproto.com/specs/label) literally. The wire format on responses matches `com.atproto.label.defs#label` so existing atproto SDKs can consume it directly. Storage is the data model normalized into rows; ingestion mirrors Jetstream both in code shape and in operator UX. diff --git a/docs/advanced/README.md b/docs/advanced/README.md new file mode 100644 index 0000000..49e9282 --- /dev/null +++ b/docs/advanced/README.md @@ -0,0 +1,9 @@ +# Advanced topics + +Start with the four-page happy path: [local AppView](../00-getting-started.md), [typed client](../01-client.md), [configuration and queries](../02-configure-and-query.md), and [Cloudflare deployment](../03-deploy-cloudflare.md). + +Optional features: + +- [Outbox](./outbox.md) +- [Feeds](./feeds.md) +- [Labels](./labels.md) diff --git a/docs/advanced/feeds.md b/docs/advanced/feeds.md new file mode 100644 index 0000000..6e57cf6 --- /dev/null +++ b/docs/advanced/feeds.md @@ -0,0 +1,41 @@ +# Feeds + +Feeds provide a per-user timeline of records authored by people that user follows. + +## Configure + +```ts +export default { + namespace: "com.example", + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + feeds: { + network: { + targets: [{ collection: "event", maxItems: 200 }], + }, + }, +}; +``` + +The default follow collection is `app.bsky.graph.follow`. Contrail adds it internally with `discover: false`; declare a different collection and set `follow` only when your app uses another follow record type. + +## Query + +```ts +const response = await contrail.get("com.example.getFeed", { + params: { + feed: "network", + actor: signedInDid, + collection: "community.lexicon.calendar.event", + profiles: true, + limit: 20, + }, +}); +``` + +`actor` means “whose feed,” not “record author.” It accepts a DID or handle. Filters, sorting, pagination, and hydration work like `listRecords` for the selected target collection. + +The first read starts a bounded backfill of that actor's follows, so its initial result may be partial. New target records are then fanned out during normal ingestion. Old items are pruned to each target's `maxItems` cap. + +Fan-out cost grows with an author's number of indexed followers. Feeds are therefore a projection for bounded application communities, not a replacement for a network-wide timeline service. diff --git a/docs/advanced/labels.md b/docs/advanced/labels.md new file mode 100644 index 0000000..a636329 --- /dev/null +++ b/docs/advanced/labels.md @@ -0,0 +1,37 @@ +# Labels + +Contrail can subscribe to AT Protocol labelers and attach their labels to records and profiles. + +## Configure + +```ts +export default { + namespace: "com.example", + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + labels: { + sources: [ + { did: "did:plc:ar7c4by46qjdydhdevvrndac" }, + ], + }, +}; +``` + +Normal `contrail backfill`, Worker cron ingestion, and `runPersistent()` include configured labelers automatically. + +## Select labelers + +A request chooses labelers with the standard header: + +```text +Atproto-Accept-Labelers: did:plc:ar7c4by46qjdydhdevvrndac +``` + +Use `?labelers=did:plc:...` when setting a header is inconvenient. Without either, Contrail uses `labels.defaults`, or all configured sources when defaults are omitted. Set `defaults: []` to require callers to opt in. + +Selected labels appear as `record.labels`. Account labels appear on hydrated profile entries. The response's `Atproto-Content-Labelers` header reports which configured labelers were applied. + +Contrail drops expired labels, applies CID-pinned labels only to the matching record version, and treats newer `neg: true` labels as retractions. + +Label signatures are stored but are not currently verified. Contrail consumes labels; it does not publish a label stream or provide moderation-preference UI. diff --git a/docs/advanced/outbox.md b/docs/advanced/outbox.md new file mode 100644 index 0000000..db069e1 --- /dev/null +++ b/docs/advanced/outbox.md @@ -0,0 +1,94 @@ +# Outbox + +> Experimental. Enable the outbox only on a fresh, empty Contrail database. + +The outbox delivers indexed record changes to external projections such as search indexes, webhooks, or caches. Contrail appends each change in the same database transaction as the canonical record and source cursor; destination failures never roll back ingestion. + +## Configure a consumer + +```ts +const config: ContrailConfig = { + namespace: "com.example", + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "history", + }, + }, + }, +}; +``` + +Collections are full NSIDs, not config short names. Omitting `phases` includes both historical backfill and live ingestion. + +## Deliver from a Worker + +Add one handler for every configured consumer: + +```ts +type Env = { SEARCH_ENDPOINT: string }; + +export default createWorker(config, { + deliveries: { + search: async (batch, { env, signal }) => { + const response = await fetch(env.SEARCH_ENDPOINT, { + method: "POST", + signal, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + cursor: batch.cursor, + upserts: batch.currentRecords, + deletes: batch.absentUris, + }), + }); + if (!response.ok) throw new Error(`Search returned ${response.status}`); + }, + }, +}); +``` + +`createWorker` runs bounded delivery rounds after scheduled ingestion. Throwing retries the batch with backoff; returning successfully acknowledges it. + +Delivery is **at least once**. A destination may apply a batch before the acknowledgement fails, so handlers must be idempotent. Upsert and delete by record URI rather than incrementing counters. + +Claims coalesce repeated changes to the same URI. `currentRecords` contains the latest indexed values when the batch is delivered; `absentUris` contains records that are currently deleted. + +## Initial state + +| `initial` | Starts with | +|---|---| +| `history` | All retained historical and live changes | +| `future` | Changes written after the consumer is registered | +| `current` | A current-state snapshot, a fixed catch-up tail, then atomic destination activation | + +`current` is intended for building a candidate index without a read gap. It additionally requires matching `changeBootstraps` snapshot and activation handlers. + +For a long-lived Node process, run delivery beside ingestion: + +```ts +await Promise.all([ + contrail.runPersistent({ signal }), + contrail.runPersistentDeliveries({ + env, + deliveries: { search: deliverSearch }, + runtime: { signal }, + }), +]); +``` + +## Operate + +```bash +pnpm contrail changes status --remote +pnpm contrail changes retry search --remote +pnpm contrail changes prune --remote +``` + +Status reports each consumer's position, backlog, lease, and retry state. Pruning never passes the slowest durable consumer. `changes skip` is an explicit audited data-loss operation and should be reserved for recovery. + +Once enabled, ordinary startup fails closed if a consumer is removed or changed incompatibly. Adding a consumer is safe only when its collection/phase coverage was already retained; otherwise build a fresh database generation. diff --git a/docs/frameworks/sveltekit-cloudflare.md b/docs/frameworks/sveltekit-cloudflare.md deleted file mode 100644 index dd384dd..0000000 --- a/docs/frameworks/sveltekit-cloudflare.md +++ /dev/null @@ -1,213 +0,0 @@ -# SvelteKit + Cloudflare Workers - -How to add contrail to an existing SvelteKit project deployed on Cloudflare Workers (via `@sveltejs/adapter-cloudflare`). Gives you XRPC endpoints alongside your pages, Jetstream ingestion on cron, and a typed in-process client for server loaders. - -Assumes you already have a SvelteKit app with `@sveltejs/adapter-cloudflare` and a D1 binding. If you don't, [`apps/sveltekit-cloudflare-workers`](https://github.com/flo-bit/contrail/tree/main/apps/sveltekit-cloudflare-workers) is a complete starting point. - -## Install - -```bash -pnpm add @atmo-dev/contrail @atcute/client -``` - -## Project layout - -``` -src/ - lib/ - contrail.config.ts # your config — auto-detected by the CLI - contrail/ - index.ts # Contrail instance + ensureInit + server client - routes/ - xrpc/[...path]/+server.ts # mounts all contrail XRPC endpoints - api/cron/+server.ts # hit by the cron trigger (see below) -wrangler.jsonc -``` - -## 1. Declare the config - -```ts -// src/lib/contrail.config.ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -export const config: ContrailConfig = { - namespace: "com.example", - collections: { - event: { - collection: "community.lexicon.calendar.event", - queryable: { startsAt: { type: "range" } }, - searchable: ["name", "description"], - }, - }, -}; -``` - -## 2. The Contrail instance - -```ts -// src/lib/contrail/index.ts -import { Contrail } from "@atmo-dev/contrail"; -import { createHandler, createServerClient } from "@atmo-dev/contrail/server"; -import type { Client } from "@atcute/client"; -import { config } from "../contrail.config"; - -export const contrail = new Contrail(config); - -let initialized = false; -export async function ensureInit(db: D1Database) { - if (!initialized) { await contrail.init(db); initialized = true; } -} - -const handle = createHandler(contrail); - -/** In-process XRPC client for loaders and actions. */ -export function getServerClient(db: D1Database): Client { - return createServerClient(async (req) => { - await ensureInit(db); - return handle(req, db) as Promise; - }); -} -``` - -Why the lazy `ensureInit`: Workers cold-start many times; doing schema init on the first request keeps the boot path fast and means `contrail.init()` doesn't need top-level `await` (which the adapter doesn't love). - -## 3. Mount the XRPC routes - -One catch-all that forwards to contrail's handler: - -```ts -// src/routes/xrpc/[...path]/+server.ts -import type { RequestHandler } from "./$types"; -import { createHandler } from "@atmo-dev/contrail/server"; -import { contrail, ensureInit } from "$lib/contrail"; - -const handle = createHandler(contrail); - -async function h(req: Request, platform: App.Platform | undefined) { - const db = platform!.env.DB; - await ensureInit(db); - return handle(req, db) as Promise; -} - -export const GET: RequestHandler = ({ request, platform }) => h(request, platform); -export const POST: RequestHandler = ({ request, platform }) => h(request, platform); -``` - -Now every `com.example.*.listRecords` / `com.example.*.getRecord` / `com.example.notifyOfUpdate` / etc. is served under `/xrpc/...`. - -## 4. Using the typed client in loaders - -```ts -// src/routes/+page.server.ts -import { getServerClient } from "$lib/contrail"; -import type { PageServerLoad } from "./$types"; - -export const load: PageServerLoad = async ({ platform }) => { - const rpc = getServerClient(platform!.env.DB); - const res = await rpc.get("com.example.event.listRecords", { - params: { startsAtMin: "2026-01-01", limit: 20 }, - }); - return { events: res.ok ? res.data.records : [] }; -}; -``` - -`createServerClient` bypasses the network — the loader runs Contrail's public XRPC handler in-process. - -## 5. Cron ingest — the workaround - -SvelteKit's `@sveltejs/adapter-cloudflare` doesn't expose a `scheduled()` export on the generated worker ([issue #4841](https://github.com/sveltejs/kit/issues/4841)). The fix is an HTTP endpoint that does the ingest, plus a post-build patch on `_worker.js` that appends a `scheduled` handler calling it. The patch is what `contrail append-scheduled` does. - -**Endpoint:** - -```ts -// src/routes/api/cron/+server.ts -import type { RequestHandler } from "./$types"; -import { contrail, ensureInit } from "$lib/contrail"; - -export const POST: RequestHandler = async ({ request, platform }) => { - if (request.headers.get("X-Cron-Secret") !== platform!.env.CRON_SECRET) { - return new Response("Unauthorized", { status: 401 }); - } - const db = platform!.env.DB; - await ensureInit(db); - await contrail.ingest({}, db); - return new Response("OK"); -}; -``` - -**Wire `contrail append-scheduled` into your `build` script:** - -```jsonc -// package.json -"scripts": { - "build": "vite build && contrail append-scheduled" -} -``` - -`contrail append-scheduled` patches `.svelte-kit/cloudflare/_worker.js` to append a `scheduled()` export that POSTs to `/api/cron` with `env.CRON_SECRET`. Override with `--worker `, `--cron-path `, or `--secret-env ` if your project diverges. - -`CRON_SECRET` is any random string — generate one, set it as a secret with `wrangler secret put CRON_SECRET`. The cron handler self-auths with it so nobody external can trigger your ingest. - -## 6. Wrangler config - -```jsonc -// wrangler.jsonc -{ - "main": ".svelte-kit/cloudflare/_worker.js", - "compatibility_date": "2025-12-25", - "compatibility_flags": ["nodejs_compat_v2"], - "assets": { "binding": "ASSETS", "directory": ".svelte-kit/cloudflare" }, - "d1_databases": [ - { "binding": "DB", "database_name": "yourapp", "database_id": "..." } - ], - "triggers": { "crons": ["*/1 * * * *"] } -} -``` - -Type the D1 binding in `src/app.d.ts`: - -```ts -declare global { - namespace App { - interface Platform { - env: { - DB: D1Database; - CRON_SECRET: string; - // ...other bindings - }; - } - } -} -``` - -## 7. Deploy + backfill - -```bash -pnpm wrangler d1 create yourapp # copy the id into wrangler.jsonc -pnpm build && pnpm wrangler deploy -pnpm wrangler secret put CRON_SECRET # paste any random string -pnpm contrail backfill --remote # one-time historical backfill -``` - -From now on: - -- Pages and XRPC endpoints are served under your domain. -- The cron fires every minute, hitting `/api/cron`, which runs `contrail.ingest()`. -- Loaders that need live data use `getServerClient()` for zero-overhead typed calls. -- After a short outage, ingestion resumes from its saved cursor. If source history has expired, rebuild into a fresh database with `pnpm backfill:remote`. - -## Where to go next - -- [Indexing](../01-indexing.md) — config options, adapter choices -- [Querying](../02-querying.md) — filters, sorts, hydration, search -- [Feeds](../04-feeds.md) — personalized timelines via follow + target collections -- [Labels](../09-labels.md) — moderation label hydration - -Use Atcute directly for Lexicon pulling, validation, and TypeScript generation. - -## Common gotchas - -- **Top-level await in `$lib/contrail/index.ts`** will fail to bundle — use the lazy `ensureInit` pattern above. -- **`ensureInit` is per-isolate, not global.** Cloudflare cold-starts spin new isolates; each one pays one init call on its first request. `contrail.init()` is idempotent so this is safe, just not instant. -- **SvelteKit's `adapter-cloudflare` regenerates `_worker.js` on every build**, so `contrail append-scheduled` has to run *after* `vite build`. Don't try to put it in `prebuild`. -- **`D1Database` type in platform env** needs `@cloudflare/workers-types` in `devDependencies` and `types` in your tsconfig. diff --git a/docs/public-services/api-atmo-rsvp.md b/docs/public-services/api-atmo-rsvp.md deleted file mode 100644 index cfb2345..0000000 --- a/docs/public-services/api-atmo-rsvp.md +++ /dev/null @@ -1,246 +0,0 @@ -# Example: api.atmo.rsvp - -[`https://api.atmo.rsvp`](https://api.atmo.rsvp) is a public Contrail read-through service for AT Protocol calendar events and RSVPs. It demonstrates anonymous collection queries, profile hydration, a personalized network feed, authenticated update notifications, verified remote discovery, and an immutable D1 deployment generation. - -## Discovery - -```text -https://api.atmo.rsvp/.well-known/contrail -https://api.atmo.rsvp/.well-known/did.json -https://api.atmo.rsvp/lexicons -https://api.atmo.rsvp/status -``` - -The XRPC namespace is DNS-authoritative: - -```text -rsvp.atmo.* -``` - -The base service DID and exact service-auth audience are: - -```text -service DID: did:web:api.atmo.rsvp -audience: did:web:api.atmo.rsvp#contrail -``` - -The fragmented service reference is the OAuth and JWT audience. The base DID identifies the DID document published by the API. The API does not use either value to sign user records and does not act as a PDS. - -## Anonymous methods - -```text -rsvp.atmo.getCursor -rsvp.atmo.getProfile -rsvp.atmo.event.getRecord -rsvp.atmo.event.listRecords -rsvp.atmo.rsvp.getRecord -rsvp.atmo.rsvp.listRecords -``` - -Event queries support equality and date-range filtering, full-text search over names and descriptions, stable keyset pagination, RSVP relation counts, RSVP hydration, and actor profile hydration. - -RSVP queries support status, subject URI, and creation-time filtering. They can hydrate the referenced event and actor profiles. - -`getProfile` resolves an actor and reads their indexed `app.bsky.actor.profile` record. Missing public profile data may be fetched from that actor's PDS as part of the read-through request. - -## Protected methods - -Discovery lists these separately under AT Protocol service auth: - -```text -rsvp.atmo.getFeed -rsvp.atmo.notifyOfUpdate -``` - -The module generated by `contrail connect` exposes the required OAuth permission: - -```ts -import { contrail } from "./contrail/index.js"; - -export const scopes = ["atproto", contrail.scope]; -// contrail.scope is: -// rpc?aud=did:web:api.atmo.rsvp%23contrail&lxm=rsvp.atmo.getFeed&lxm=rsvp.atmo.notifyOfUpdate -``` - -After login, derive one client from the user's existing authenticated AT Protocol client: - -```ts -const client = contrail.authenticated(authenticatedClient, { - onNotificationError(error, { uris }) { - console.warn("Contrail notification failed", uris, error); - }, -}); -``` - -Advertised service methods route to Contrail; other methods route to the PDS. Protected methods automatically obtain and cache exact method-bound tokens. - -Missing, expired, wrong-audience, wrong-method, and invalid-signature tokens receive `401` with a `WWW-Authenticate` challenge. - -## Personalized network feed - -The configured feed is: - -```text -feed=network -``` - -It contains recent events and RSVPs authored by actors followed by the signed-in user. Per-actor projection caps are: - -| Collection | Maximum retained items | -|---|---:| -| `community.lexicon.calendar.event` | 100 | -| `community.lexicon.calendar.rsvp` | 250 | - -A typed query looks like: - -```ts -const response = await client.get( - "rsvp.atmo.getFeed", - { - params: { - feed: "network", - actor: signedInDid, - collection: "community.lexicon.calendar.event", - profiles: true, - limit: 20, - }, - }, -); -``` - -The requested actor must resolve to the token issuer. Service auth therefore prevents one authenticated account from creating or refreshing arbitrary personalized feed projections for other actors. - -The underlying `app.bsky.graph.follow` records are internal. The service does not advertise raw follow collection methods. It indexes follows authored by known actors only when the follow subject is already in the service's acquisition scope. Constellation enrichment helps connect newly observed calendar authors to existing in-scope followers. - -The feed endpoint may start a bounded background follow backfill the first time an actor requests their feed. Until that finishes, an initial response may be empty or partial. The operation remains a read-through cache fill rather than a network-wide social graph crawl. - -## Immediate update notification - -Successful event and RSVP writes through the combined client automatically notify Contrail: - -```ts -const response = await client.post("com.atproto.repo.createRecord", { - input: { - repo: signedInDid, - collection: "community.lexicon.calendar.event", - record: event, - }, -}); -``` - -The original PDS response is returned unchanged. Notification failures are nonfatal and reported through `onNotificationError`. The protected `rsvp.atmo.notifyOfUpdate` procedure remains available for explicit batches or records written elsewhere. - -The endpoint enforces all of the following: - -- at most 25 URIs per request; -- every URI is a canonical record AT URI; -- every URI belongs to the service-token issuer; -- the collection is tracked by this deployment; -- the record body and CID are fetched from the issuer's current PDS; and -- only an explicit XRPC `RecordNotFound` response is treated as deletion. - -Transient DNS, identity, network, timeout, PDS, or malformed-response failures preserve existing indexed state and are returned as bounded per-record errors to the authenticated caller. - -## Profiles - -The deployment indexes: - -```text -app.bsky.actor.profile -``` - -but keeps the underlying profile collection methods internal. Profiles are exposed through: - -- `rsvp.atmo.getProfile`; -- `profiles=true` on event and RSVP reads; and -- `profiles=true` on network feed reads. - -This avoids a redundant raw profile-record API while still providing typed display names, handles, avatars, and profile values alongside calendar data. - -## Acquisition scope - -Relay discovery starts from: - -```text -community.lexicon.calendar.event -community.lexicon.calendar.rsvp -``` - -Profiles and follows are dependent collections. They do not independently discover every Bluesky repository. - -The service is intentionally a shared, possibly incomplete cache. Unavailable identities and PDSes stay visibly pending, retrying, or failed instead of being silently marked complete. Unknown dependent subjects are scope exclusions and do not create tombstones. - -## Runtime validation policy - -This deployment publishes verified API and record Lexicons for discovery and TypeScript generation, but does not enable Contrail's optional runtime record/CID validation during ingestion. - -That distinction is deliberate. A matched benchmark using the current calendar Lexicons rejected 12,291 historical records, reducing indexed events from roughly 14,600 to 4,800 and RSVPs from roughly 6,300 to 3,800. Those records include historical shapes that predate the current published definitions, so enabling latest-schema validation would silently discard most of the useful archive. - -Under the compatibility policy: - -- provider and consumer contracts are still canonical and digest-verified; -- generated Atcute types describe the current expected response values, but are not a runtime guarantee for every historical record; -- startup still rejects an inconsistent advertised API; -- records and CIDs are fetched from authoritative sources rather than accepted from callers; but -- ingestion does not reject records based on runtime Lexicon or canonical-CID checks. - -A future strict deployment needs version-aware historical schemas or an explicitly looser response value, rather than pretending the compatibility loss does not exist. - -## Ordered source position - -The primary ordered source is one pinned Jetstream endpoint with an operator-owned continuity epoch: - -```json -{ - "source": "jetstream", - "epoch": "api-atmo-rsvp-primary-2026-08" -} -``` - -`rsvp.atmo.getCursor` returns the currently committed opaque position. Consumers compare the complete source, epoch, and cursor for equality only. A source or epoch change requires a full refetch. - -Backfills do not send historical notifications. Jetstream projection advances the serving position atomically with accepted live mutations. - -## Production generation - -The current expanded generation was built in native SQLite before activation. Its initial canonical projection contained approximately: - -| Collection | Records | -|---|---:| -| Events | 14,751 | -| RSVPs | 6,299 | -| Profiles | 1,408 | -| Scoped follows | 65,798 | - -The exact live totals change as Jetstream and read-through acquisition continue. - -The provisioning process: - -1. captured a replay boundary before relay discovery; -2. ran the resumable native-SQLite backfill; -3. retained 43 unavailable accounts as explicit scheduled retries; -4. replayed Jetstream to the present; -5. imported canonical tables into a fresh D1 database; -6. rebuilt FTS and materialized RSVP counts; -7. verified all visible rows against durable record versions; -8. exercised discovery, profiles, search, CORS, and protected-route rejection through a candidate Worker; and -9. activated the new Worker and D1 binding together. - -The previous D1 generation remains separate for rollback. No percentage traffic split is used between databases with independent serving positions. - -## Connect a consumer - -```bash -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp -``` - -The provider lock records: - -- the HTTPS endpoint; -- `rsvp.atmo` namespace; -- anonymous methods; -- protected methods and their audience; -- the content-addressed Lexicon digest; and -- the provider-owned Lexicon directory. - -See [Using a public Contrail service](./using.md) for a framework-neutral consumer walkthrough. diff --git a/docs/public-services/creating.md b/docs/public-services/creating.md deleted file mode 100644 index a520a0c..0000000 --- a/docs/public-services/creating.md +++ /dev/null @@ -1,311 +0,0 @@ -# Creating a public Contrail service - -A public Contrail service lets independent applications query one Contrail AppView from a stable HTTPS origin. The provider chooses the indexed collections, projections, query methods, and authentication policy. Consumers discover that API surface, verify its Lexicons, generate local TypeScript types, and make ordinary XRPC requests. - -Public service mode does not turn Contrail into a PDS. Records remain in their authors' repositories, and applications still authenticate users and publish writes through those users' PDSes. - -## Define the index - -Start with a normal Contrail configuration: - -```ts -// src/contrail.config.ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -export const config: ContrailConfig = { - namespace: "events.example", - orderedSource: { - source: "jetstream", - epoch: "primary-2026-08", - }, - collections: { - event: { - collection: "community.lexicon.calendar.event", - queryable: { - mode: {}, - startsAt: { type: "range" }, - }, - searchable: ["name", "description"], - }, - }, -}; -``` - -The namespace becomes the prefix of generated methods such as: - -```text -events.example.getCursor -events.example.event.getRecord -events.example.event.listRecords -``` - -Use one stable `orderedSource.epoch` for one continuity history. Change the epoch when the Jetstream endpoint set, retention assumptions, or cursor meaning changes. Consumers treat a source or epoch change as a full-refetch boundary. - -## Generate the public Lexicons - -Add Atcute's generator configuration: - -```js -// lex.config.js -import { defineLexiconConfig } from "@atcute/lex-cli"; - -export default defineLexiconConfig({ - generate: { - files: [ - "lexicons/custom/**/*.json", - "lexicons/pulled/**/*.json", - "lexicons/generated/**/*.json", - ], - outdir: "src/lexicon-types/", - }, -}); -``` - -Generate the provider API, pull referenced record Lexicons, and generate TypeScript types: - -```bash -pnpm contrail lexicons all --public -``` - -Check generated drift in CI: - -```bash -pnpm contrail lexicons check --public -``` - -The public surface includes anonymous queries plus explicitly configured service-auth queries and procedures. Private full-surface procedures are not included merely because they exist in application code. - -## Publish discovery from a Worker - -Pass the generated documents and canonical HTTPS origin to `createWorker`: - -```ts -// src/worker.ts -import { createWorker } from "@atmo-dev/contrail/worker"; -import { lexicons } from "../lexicons/generated"; -import { config } from "./contrail.config"; - -export default createWorker(config, { - lexicons, - publicService: { - endpoint: "https://api.example.com", - }, -}); -``` - -This publishes: - -```text -GET /.well-known/contrail -GET /lexicons -GET /lexicons/ -GET /status -``` - -The version-2 discovery manifest contains the endpoint, namespace, methods, collections, service-auth declaration, and a content-addressed Lexicon bundle. It does not hash the complete method set. Startup still fails when advertised methods, capabilities, and bundled Lexicons disagree, and the immutable Lexicon URL retains its digest. - -The public `/status` response contains aggregate readiness and freshness information. It omits DIDs, record bodies, source cursors, raw upstream errors, and other private operational details. - -## Anonymous read-through methods - -Collection queries, `getCursor`, profiles, feeds, and authored custom queries are anonymous unless explicitly protected. An anonymous query may still improve the shared cache by: - -- resolving an actor; -- fetching a missing public record from its PDS; -- populating profile or feed projections; or -- running a trusted custom query handler. - -This is read-through acquisition, not a caller-controlled write API. Only configured collections and trusted provider code can affect the projection. - -## Protecting feeds and notifications with service auth - -AT Protocol service auth lets any suitably authorized AT Protocol client call selected methods without distributing a shared application secret. - -```ts -export const config: ContrailConfig = { - namespace: "events.example", - notify: true, - serviceAuth: { - audience: "did:web:api.example.com#contrail", - methods: ["getFeed", "notifyOfUpdate"], - }, - collections, - feeds: { - network: { - targets: [{ collection: "event", maxItems: 100 }], - }, - }, -}; -``` - -The provider verifies: - -- the JWT signature against the issuer DID's `#atproto` key; -- the exact fragmented service audience; -- the token's exact `lxm` method claim; -- expiration and maximum token age; and -- the route-specific ownership rule. - -For protected feeds, the requested actor must resolve to the token issuer. For `notifyOfUpdate`, every submitted AT URI must belong to the token issuer. Notify still fetches the current authoritative record from that issuer's PDS; callers never submit a record body for Contrail to trust. - -The default PLC/`did:web` resolver keeps a bounded five-minute in-process cache and deduplicates concurrent lookups. Signature failure forces an uncached refresh so key rotation does not remain hidden behind a stale entry. Deployments can still provide their own resolver policy. - -The authenticated methods are listed separately from anonymous methods in discovery. Their query or procedure Lexicons remain in the provider bundle, so consumers still get generated types. - -## Start from owned source - -A consumer developed alongside the provider can generate its initial API surface before any deployment exists: - -```bash -pnpx @atmo-dev/contrail connect ./src/contrail.config.ts -# or discover the standard config beneath another project directory -pnpx @atmo-dev/contrail connect ../api -``` - -This compiles the config directly, writes generated Lexicons and types, and exports local/target client factories. It does not create or modify `contrail.lock.json`. Run the service separately: - -```bash -pnpx @atmo-dev/contrail dev --config ../api/src/contrail.config.ts -``` - -After deploying, `contrail connect https://api.example.com` creates the version-2 provider lock and makes that deployment the generated default target. Future config-source connections can refresh local API types without changing the production lock. - -### OAuth permission versus token binding - -Contrail generates one least-privilege OAuth permission containing the exact fragmented audience and one sorted `lxm` parameter per protected method: - -```text -rpc?aud=did:web:api.example.com%23contrail&lxm=events.example.getFeed&lxm=events.example.notifyOfUpdate -``` - -Current granular OAuth RPC syntax requires the absolute DID service reference; a plain DID may be silently removed from an authorization request. The `%23` is the encoded `#` delimiter. Wildcard `lxm=*` permissions are not generated because they are broader than necessary and can trigger misleading consent descriptions. - -Each call to `com.atproto.server.getServiceAuth` still passes the specific method NSID as `lxm` and the decoded `did:web:api.example.com#contrail` audience, producing a short-lived method-bound token. - -For example, a feed token uses: - -```text -lxm=events.example.getFeed -``` - -and cannot be reused for: - -```text -lxm=events.example.notifyOfUpdate -``` - -### Service DID and audience - -For this configuration, the identities are distinct: - -```text -base service DID: did:web:api.example.com -exact audience: did:web:api.example.com#contrail -public endpoint: https://api.example.com -``` - -Contrail publishes the base DID document at: - -```text -https://api.example.com/.well-known/did.json -``` - -The document `id` is the base DID and its Contrail service entry `id` is the exact fragmented audience. Startup fails if an automatically hosted `did:web` audience resolves to a different DID-document URL. A `did:plc` audience remains externally managed and does not create a local DID route. - -The exact service reference—not the base DID—is the JWT audience passed to `getServiceAuth` and verified by Contrail. The AppView does not need a signing key merely to receive and verify user-issued service tokens. - -## Profiles and internal follow projections - -Profiles can be enabled without exposing raw profile collection methods: - -```ts -profiles: ["app.bsky.actor.profile"], -collections: { - event, - profile: { - collection: "app.bsky.actor.profile", - discover: false, - methods: [], - }, -} -``` - -Likewise, a follow collection can remain an internal feed input: - -```ts -follow: { - collection: "app.bsky.graph.follow", - discover: false, - subjectField: "subject", - methods: [], -} -``` - -`discover: false` prevents network-wide relay discovery for dependent collections. `subjectField: "subject"` excludes follows whose subject is outside the known acquisition scope. Those exclusions do not create tombstones. - -Profiles can then appear through `getProfile` and `profiles=true` hydration, while follows power `getFeed` without creating a public social-graph directory. - -## Runtime validation is explicit per collection - -The deployment already ships one reviewed generated Lexicon bundle. Select which collection policies use it directly: - -```ts -collections: { - event: { - collection: "community.lexicon.calendar.event", - validate: true, - }, - legacy: { - collection: "community.example.legacy", - // Omitted or false means no runtime validation. - }, -}, -validation: { - strict: true, - verifyCid: true, -} -``` - -`createWorker(config, { lexicons })` binds the exact deployment bundle to opted-in collections; no second validation-specific array is needed. Validation applies across every acquisition source and startup fails if an opted-in record schema or transitive reference is absent. Collections without `validate: true` retain compatibility behavior. - -## CORS - -Public services allow browser requests and explicitly permit the `Authorization`, `Content-Type`, and `Atproto-Accept-Labelers` headers. Authentication failures expose `WWW-Authenticate` so browser clients can distinguish missing, expired, wrong-audience, and wrong-method tokens. - -Never place a reusable application secret in browser code. AT Protocol service tokens are short-lived and minted for the authenticated user's DID. - -## Provisioning and activation - -Local development can use the normal local backfill command: - -```bash -pnpm contrail backfill -``` - -For a substantial D1 production deployment, do not run a long bulk load through Wrangler's remote development proxy. Prefer a fresh generation: - -1. capture the ordered-source replay boundary; -2. build canonical state in native SQLite; -3. leave unavailable accounts visibly pending, retrying, or failed; -4. catch up through the ordered source; -5. import canonical tables into a fresh D1 database; -6. rebuild FTS, relation counts, and other derived projections; -7. verify record/version consistency, status, discovery, and representative queries; -8. test the candidate through a non-production Worker; and -9. activate the matching Worker and D1 binding together. - -Keep the previous D1 generation available for rollback. Do not split percentage traffic between independent databases with different serving positions. - -## Provider checklist - -Before announcing an origin: - -- run public Lexicon drift checking and TypeScript typechecking; -- verify the manifest and Lexicon digests differ and both recompute correctly; -- verify every advertised method has a matching query or procedure Lexicon; -- verify protected methods reject missing, wrong-audience, and wrong-`lxm` tokens; -- verify feed actors and notify URIs are bound to the token issuer; -- verify browser CORS preflight with `Authorization`; -- verify `/status` contains no sensitive operational detail; -- verify `getCursor` reports the committed ordered-source position; and -- connect and compile an independent consumer project. diff --git a/docs/public-services/using.md b/docs/public-services/using.md deleted file mode 100644 index 69903ef..0000000 --- a/docs/public-services/using.md +++ /dev/null @@ -1,164 +0,0 @@ -# Using a public Contrail service - -A public Contrail service is a typed, read-through API over public AT Protocol records. This guide uses `https://api.atmo.rsvp`; replace it with the provider you want to use. - -## Install and connect - -```bash -pnpm add @atcute/client @atcute/lexicons @atmo-dev/contrail -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp -``` - -`connect` validates the provider description and content-addressed Lexicon bundle, writes a version-2 `contrail.lock.json`, and generates: - -```text -lex.config.js -src/contrail/ - index.ts - lexicons/ - types/ -``` - -The generated configuration contains everything needed to identify the service and generate its types: - -```js -export default { - contrail: { - endpoint: "https://api.atmo.rsvp", - serviceDid: "did:web:api.atmo.rsvp", - serviceAudience: "did:web:api.atmo.rsvp#contrail", - scope: - "rpc?aud=did:web:api.atmo.rsvp%23contrail&lxm=rsvp.atmo.getFeed&lxm=rsvp.atmo.notifyOfUpdate", - protectedMethods: [ - "rsvp.atmo.getFeed", - "rsvp.atmo.notifyOfUpdate", - ], - collections: [ - "app.bsky.actor.profile", - "app.bsky.graph.follow", - "community.lexicon.calendar.event", - "community.lexicon.calendar.rsvp", - ], - }, - generate: { - files: ["src/contrail/lexicons/**/*.json"], - outdir: "src/contrail/types/", - }, -}; -``` - -For JavaScript output, choose a `.js` client path: - -```bash -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp \ - --client src/contrail/index.js -``` - -Commit `contrail.lock.json` and the generated files. - -When the application owns or can read the provider config, connect directly to that source before a deployment exists: - -```bash -pnpx @atmo-dev/contrail connect ../api/src/contrail.config.ts -pnpx @atmo-dev/contrail dev --config ../api/src/contrail.config.ts -``` - -The config connection compiles Lexicons and types without creating or modifying `contrail.lock.json`. `contrail dev` only runs the loopback service. The generated module exports `createLocalContrailClient()`: - -```ts -import { - contrail as productionContrail, - createLocalContrailClient, -} from "./contrail/index.js"; - -export const contrail = process.env.CONTRAIL_URL - ? createLocalContrailClient(process.env.CONTRAIL_URL) - : productionContrail; -``` - -The helper permits HTTP only for `localhost`, `127.0.0.1`, or `[::1]` and does not inherit a production service-auth audience. Local notification and configured protected methods are loopback-only operations with a null OAuth scope. A deployed service still requires HTTPS and its real service auth. - -## Query anonymous methods - -```ts -import { contrail } from "./contrail/index.js"; - -const response = await contrail.get("rsvp.atmo.event.listRecords", { - params: { - limit: 20, - sort: "startsAt", - order: "asc", - profiles: true, - }, -}); - -if (!response.ok) { - throw new Error(`Contrail query failed: ${response.status}`); -} - -for (const event of response.data.records) { - console.log(event.value.name, event.value.startsAt); -} -``` - -The generated Lexicons provide typed method names, parameters, and responses. - -## Use one authenticated client - -Add the provider's generated least-privilege scope to the application's OAuth scopes. It contains the encoded fragmented audience and only the protected methods advertised by this provider: - -```ts -import { contrail } from "./contrail/index.js"; - -export const scopes = ["atproto", contrail.scope]; -``` - -After login, combine Contrail with the existing authenticated AT Protocol client: - -```ts -const client = contrail.authenticated(authenticatedClient, { - onNotificationError(error, { uris }) { - console.warn("Contrail notification failed", uris, error); - }, -}); - -const response = await client.get("rsvp.atmo.getFeed", { - params: { - feed: "network", - actor: signedInDid, - collection: "community.lexicon.calendar.event", - profiles: true, - limit: 20, - }, -}); -``` - -The same client still handles ordinary PDS calls. Successful writes to connected collections automatically notify Contrail: - -```ts -await client.post("com.atproto.repo.createRecord", { - input: { - repo: signedInDid, - collection: "community.lexicon.calendar.event", - record: event, - }, -}); -``` - -Contrail returns the original PDS response. A notification failure is reported through `onNotificationError` but never turns a committed PDS write into a failed write. Handle-form `deleteRecord` inputs are resolved through the PDS to construct the canonical DID record URI before notification. The login session remains application-owned. - -## Update the connection - -Anonymous generated clients call the endpoint directly without fetching discovery first. Protected calls lazily discover service auth; transient discovery failures remain retryable, while endpoint, base-DID, exact-audience, scope, or protected-method mismatches fail closed. Adding anonymous provider methods does not interrupt methods already known by a generated client. Regenerate when application code wants new API surface or the protected contract changes: - -```bash -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp --update -``` - -Review changes to `contrail.lock.json`, `lex.config.js`, and `src/contrail/`, then run the application's typecheck and tests. A change from the old plain-DID/wildcard permission requires OAuth reauthorization; an existing grant cannot mint tokens for the corrected fragmented audience. `--update` cannot repoint an existing lock or abandon its provider-owned Lexicon root; remove the existing connection deliberately before switching providers or output roots. Version-1 provider locks are intentionally unsupported after the clean manifest-v2 cut and must be removed before reconnecting. A version-2 lock written before exact audiences existed is reported separately and needs the same removal, reconnection, and OAuth reauthorization. - -## Completeness - -A public Contrail service is a shared, possibly incomplete read-through cache. Reads may fetch missing public records or profiles, but an empty result does not prove that no matching record exists on the network. - -Applications still authenticate users and publish records through their PDS. Contrail service auth only authorizes the protected methods advertised by that service. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 947a24d..a4a42f6 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -293,7 +293,6 @@ The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` Connect an independent consumer with `contrail connect `. The version-2 provider lock records the deployment and exact Lexicon bundle, but generated clients do not pin the provider's complete method set at runtime. Existing anonymous methods therefore continue working when a provider adds methods. A repeated connection to the same endpoint and provider-owned output root requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. Version-1 locks must be removed and reconnected. -See [Creating a public service](../../docs/public-services/creating.md), [Using a public service](../../docs/public-services/using.md), and [Example: api.atmo.rsvp](../../docs/public-services/api-atmo-rsvp.md) for complete provider and consumer walkthroughs. ## Runtime record validation diff --git a/packages/contrail/src/core/change-bootstrap.ts b/packages/contrail/src/core/change-bootstrap.ts index 2115bfc..4385bad 100644 --- a/packages/contrail/src/core/change-bootstrap.ts +++ b/packages/contrail/src/core/change-bootstrap.ts @@ -142,7 +142,8 @@ async function release( await db .prepare( `UPDATE change_consumers - SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND lease_owner = ?`, ) .bind(timestamp, consumerId, generation, owner) @@ -224,7 +225,8 @@ export async function claimCurrentSnapshotPage( bootstrap_scan_cursor = CASE WHEN bootstrap_state = 'pending' THEN NULL ELSE bootstrap_scan_cursor END, - lease_owner = ?, lease_expires_at = ?, updated_at = ? + lease_owner = ?, lease_expires_at = ?, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND initial_mode = 'current' AND bootstrap_state IN ('pending', 'scanning') AND generation_id = ( @@ -349,7 +351,8 @@ export async function claimCurrentSnapshotPage( .prepare( `UPDATE change_consumers SET bootstrap_scan_collection = ?, bootstrap_scan_cursor = NULL, - lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_state = 'scanning' AND lease_owner = ?`, ) @@ -372,7 +375,8 @@ export async function claimCurrentSnapshotPage( SELECT head_position FROM change_log_state WHERE id = 1 ), bootstrap_scan_collection = NULL, bootstrap_scan_cursor = NULL, - lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_state = 'scanning' AND lease_owner = ?`, ) @@ -393,7 +397,8 @@ export async function acknowledgeCurrentSnapshotPage( .prepare( `UPDATE change_consumers SET bootstrap_scan_cursor = ?, lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + lease_expires_at = NULL, lease_through_position = NULL, + attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? @@ -449,6 +454,7 @@ async function failBootstrapLease( .prepare( `UPDATE change_consumers SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, attempts = attempts + 1, next_attempt_at = ?, last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? @@ -544,7 +550,8 @@ export async function claimCurrentActivation( const row = await db .prepare( `UPDATE change_consumers - SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + SET lease_owner = ?, lease_expires_at = ?, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND initial_mode = 'current' AND bootstrap_state = 'activating' AND generation_id = ( @@ -596,7 +603,8 @@ export async function completeCurrentActivation( .prepare( `UPDATE change_consumers SET bootstrap_state = 'ready', lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + lease_expires_at = NULL, lease_through_position = NULL, + attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 1220a3a..937463e 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -163,6 +163,7 @@ export function buildChangeLogSchema( bootstrap_token TEXT, lease_owner TEXT, lease_expires_at ${bigint}, + lease_through_position ${bigint}, attempts INTEGER NOT NULL DEFAULT 0, next_attempt_at ${bigint}, last_success_at ${bigint}, diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 0008639..a94038d 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -306,7 +306,8 @@ async function releaseLease( await db .prepare( `UPDATE change_consumers - SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ?`, ) @@ -427,7 +428,8 @@ async function claimChangeRange( const leased = await db .prepare( `UPDATE change_consumers - SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + SET lease_owner = ?, lease_expires_at = ?, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND bootstrap_state = '${requiredState}' AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) @@ -508,7 +510,8 @@ async function claimChangeRange( .prepare( `UPDATE change_consumers SET bootstrap_state = 'activating', lease_owner = NULL, - lease_expires_at = NULL, updated_at = ? + lease_expires_at = NULL, lease_through_position = NULL, + updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_state = 'catching-up' AND acknowledged_position = bootstrap_target_position @@ -585,6 +588,33 @@ async function claimChangeRange( } const through = String(selected.at(-1)!.position); + // Persist the exact selected upper bound before exposing the lease. Ack, + // renew, and fail all compare it so a mutated claim cannot skip delivery. + const bounded = await db + .prepare( + `UPDATE change_consumers + SET lease_through_position = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + through, + limits.now, + consumerId, + generation, + from, + owner, + limits.now, + ) + .first<{ consumer_id: string }>(); + if (!bounded) { + throw new ChangeLeaseLostError( + `Change claim for ${consumerId} expired while its range was being bound`, + ); + } + const rows = await db .prepare( `SELECT position, phase, change_count, encoded_bytes, changes_json @@ -759,12 +789,13 @@ export async function acknowledgeChanges( AND ? = bootstrap_target_position THEN 'activating' ELSE bootstrap_state END, lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? - AND lease_expires_at > ? + AND lease_expires_at > ? AND lease_through_position = ? AND ? > acknowledged_position AND ? <= ( SELECT head_position FROM change_log_state @@ -784,6 +815,7 @@ export async function acknowledgeChanges( now, claim.through, claim.through, + claim.through, claim.generation, ) .first<{ consumer_id: string }>(); @@ -814,7 +846,7 @@ export async function renewChangeClaim( SET lease_expires_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? - AND lease_expires_at > ? + AND lease_expires_at > ? AND lease_through_position = ? RETURNING consumer_id`, ) .bind( @@ -825,6 +857,7 @@ export async function renewChangeClaim( claim.from, claim.leaseOwner, now, + claim.through, ) .first<{ consumer_id: string }>(); if (!renewed) { @@ -860,10 +893,12 @@ export async function failChanges( .prepare( `UPDATE change_consumers SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, attempts = attempts + 1, next_attempt_at = ?, last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? + AND lease_through_position = ? RETURNING attempts, next_attempt_at`, ) .bind( @@ -875,6 +910,7 @@ export async function failChanges( claim.generation, claim.from, claim.leaseOwner, + claim.through, ) .first<{ attempts: number | string; next_attempt_at: number | string | null }>(); if (!failed) { @@ -1117,7 +1153,8 @@ export async function skipChangeConsumer( .prepare( `UPDATE change_consumers SET acknowledged_position = ?, lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + lease_expires_at = NULL, lease_through_position = NULL, + attempts = 0, next_attempt_at = NULL, last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? @@ -1230,25 +1267,63 @@ export async function pruneChanges( }; } + // created_at is captured before serialized position allocation, so age and + // position order may differ under overlapping projectors. Inspect only the + // bounded leading range and stop at the first age blocker; deleting every + // independently old row would create holes that retainedFloor cannot express. + const candidates = await db + .prepare( + `SELECT position, created_at FROM change_batches + WHERE generation_id = ? AND position > ? AND position <= ? + ORDER BY position LIMIT ?`, + ) + .bind(state.generation, state.retainedFloor, safeThrough, maxBatches) + .all<{ position: number | string; created_at: number | string }>(); + + let expected = BigInt(state.retainedFloor) + 1n; + let deleteThrough = state.retainedFloor; + for (const candidate of candidates.results) { + const position = BigInt(String(candidate.position)); + if (position !== expected) { + throw new ChangeHistoryGapError( + `Change log has a gap after retained floor ${state.retainedFloor}`, + ); + } + if ( + options.olderThan !== undefined && + Number(candidate.created_at) >= options.olderThan + ) { + break; + } + deleteThrough = String(candidate.position); + expected++; + } + + if (deleteThrough === state.retainedFloor) { + return { + pruned: 0, + retainedFloor: state.retainedFloor, + safeThrough, + done: true, + }; + } + const results = await db.batch([ db .prepare( `DELETE FROM change_batches - WHERE generation_id = ? AND position IN ( - SELECT position FROM change_batches - WHERE generation_id = ? AND position > ? AND position <= ? - AND (? IS NULL OR created_at < ?) - ORDER BY position LIMIT ? - )`, + WHERE generation_id = ? AND position > ? AND position <= ? + AND NOT EXISTS ( + SELECT 1 FROM change_consumers + WHERE generation_id = ? AND acknowledged_position < ? + )`, ) .bind( - state.generation, state.generation, state.retainedFloor, - safeThrough, - options.olderThan ?? null, - options.olderThan ?? null, - maxBatches, + deleteThrough, + state.generation, + deleteThrough, ), db .prepare( diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index d412723..8e89aea 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -789,6 +789,12 @@ export async function initSchema( "encoded_bytes", "INTEGER", ); + await addColumnIfNotExists( + db, + "change_consumers", + "lease_through_position", + dialect.bigintType, + ); await db .prepare( "UPDATE change_batches SET encoded_bytes = LENGTH(changes_json) WHERE encoded_bytes IS NULL", diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index c4733aa..73b2fea 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -356,4 +356,44 @@ describe("consumer-aware change pruning", () => { (await getChangesStatus(db)).consumers.find((item) => item.id === "late"), ).toMatchObject({ position: "5", backlogBatches: 0 }); }); + + it("keeps age-limited pruning to a contiguous retained prefix", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + keeper: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, resolved); + for (let position = 1; position <= 3; position++) { + await apply(db, resolved, event({ rkey: String(position), time: position })); + } + const claim = await claimChanges(db, "keeper", { now: 100 }); + await acknowledgeChanges(db, claim!, { now: 101 }); + + // Allocation order is serialized, but created_at is captured before that + // lock and can therefore be non-monotonic under overlapping projectors. + await db + .prepare( + `UPDATE change_batches SET created_at = CASE position + WHEN 1 THEN 100 WHEN 2 THEN 300 ELSE 100 END`, + ) + .run(); + + const pruned = await pruneChanges(db, { + maxBatches: 10, + olderThan: 200, + }); + expect(pruned).toEqual({ + pruned: 1, + retainedFloor: "1", + safeThrough: "3", + done: true, + }); + expect( + ( + await db + .prepare("SELECT position FROM change_batches ORDER BY position") + .all<{ position: number }>() + ).results.map((row) => row.position), + ).toEqual([2, 3]); + }); }); diff --git a/packages/contrail/tests/change-consumers.test.ts b/packages/contrail/tests/change-consumers.test.ts index f1d1528..3ec818a 100644 --- a/packages/contrail/tests/change-consumers.test.ts +++ b/packages/contrail/tests/change-consumers.test.ts @@ -343,16 +343,34 @@ describe("durable change consumers", () => { expect(claim?.changes).toHaveLength(2); }); - it("rejects forged generation cursors and never regresses a checkpoint", async () => { + it("rejects forged generation and range cursors without skipping delivery", async () => { const db = createSqliteDatabase(":memory:"); const config = readyEventConsumer(); await initSchema(db, config); - await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); - const claim = await claimChanges(db, "search", { now: 100 }); - const forged: ChangeClaim = { ...claim!, generation: crypto.randomUUID() }; + for (let position = 1; position <= 3; position++) { + await apply(db, config, [ + mutation({ rkey: String(position), sourceTime: position }), + ]); + } + const claim = await claimChanges(db, "search", { + now: 100, + maxBatches: 1, + }); + expect(claim?.through).toBe("1"); + + const wrongGeneration: ChangeClaim = { + ...claim!, + generation: crypto.randomUUID(), + }; await expect( - acknowledgeChanges(db, forged, { now: 101 }), + acknowledgeChanges(db, wrongGeneration, { now: 101 }), ).rejects.toBeInstanceOf(ChangeLeaseLostError); + + const beyondClaimedRange: ChangeClaim = { ...claim!, through: "3" }; + await expect( + acknowledgeChanges(db, beyondClaimedRange, { now: 101 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await acknowledgeChanges(db, claim!, { now: 101 }); expect((await getChangesStatus(db)).consumers[0].position).toBe("1"); }); diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index 17f3303..851a83c 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -479,7 +479,7 @@ describe("transactional projection change log", () => { ); }); - it("upgrades retained pre-byte-count change batches without resetting consumers", async () => { + it("upgrades retained change-log columns without resetting consumers", async () => { const db = createSqliteDatabase(":memory:"); const resolved = loggedConfig(); await initSchema(db, resolved); @@ -487,6 +487,11 @@ describe("transactional projection change log", () => { await db .prepare("ALTER TABLE change_batches DROP COLUMN encoded_bytes") .run(); + await db + .prepare( + "ALTER TABLE change_consumers DROP COLUMN lease_through_position", + ) + .run(); await db .prepare( "UPDATE _contrail_meta SET value = 'old-change-schema' WHERE key = 'schema_fingerprint'", @@ -500,6 +505,13 @@ describe("transactional projection change log", () => { expect(row?.encoded_bytes).toBe( new TextEncoder().encode(row!.changes_json).byteLength, ); + expect( + ( + await db + .prepare("PRAGMA table_info(change_consumers)") + .all<{ name: string }>() + ).results.map((column) => column.name), + ).toContain("lease_through_position"); expect((await getChangeLogState(db))?.head).toBe("1"); }); -- 2.51.2 From c92eaf3f1fd2b1e01d20521cf6293a42a329e4bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:00:16 +0000 Subject: [PATCH 13/34] Version Packages --- .changeset/durable-projection-log.md | 7 ------- packages/contrail/CHANGELOG.md | 8 ++++++++ packages/contrail/package.json | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 .changeset/durable-projection-log.md diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md deleted file mode 100644 index 7318ebc..0000000 --- a/.changeset/durable-projection-log.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -"@atmo-dev/contrail": minor ---- - -Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. - -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Current-state consumers require both projection phases, and candidate destination tokens are scoped strictly to bootstrap deliveries. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 3dd7217..042e5ce 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,13 @@ # @atmo-dev/contrail +## 0.20.0 + +### Minor Changes + +- bc2f4c1: Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. + + Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Current-state consumers require both projection phases, and candidate destination tokens are scoped strictly to bootstrap deliveries. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. + ## 0.19.0 ### Minor Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 2532580..170b580 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.19.0", + "version": "0.20.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 0ec7a003af98e0b079c5d18d0454052d87d41e76 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:23:21 +0200 Subject: [PATCH 14/34] fixes --- .changeset/bounded-scheduled-ingest.md | 2 +- packages/contrail/README.md | 6 +- packages/contrail/src/core/bootstrap.ts | 24 +---- packages/contrail/src/core/db/index.ts | 2 +- packages/contrail/src/core/db/records.ts | 24 +++++ packages/contrail/src/core/identity.ts | 18 +++- packages/contrail/src/core/jetstream.ts | 61 +++++++++--- packages/contrail/src/core/persistent.ts | 7 +- packages/contrail/src/worker/index.ts | 3 +- packages/contrail/tests/ingest-hang.test.ts | 86 +++++++++++++++++ .../contrail/tests/scheduled-ingest.test.ts | 96 +++++++++++++++++++ 11 files changed, 281 insertions(+), 48 deletions(-) diff --git a/.changeset/bounded-scheduled-ingest.md b/.changeset/bounded-scheduled-ingest.md index 3cb5619..c796f73 100644 --- a/.changeset/bounded-scheduled-ingest.md +++ b/.changeset/bounded-scheduled-ingest.md @@ -2,4 +2,4 @@ "@atmo-dev/contrail": patch --- -Bound scheduled Jetstream cycles by retained candidate count and serialized bytes, drop exact transport observations before admission, preserve same-timestamp observations across capped restarts, capture empty initial cursors safely, reject rollback-prone endpoint pools in scheduled mode, and emit one bounded aggregate cycle summary. +Bound scheduled Jetstream cycles by retained candidate count, distinct identity updates, and serialized bytes; batch identity writes; drop exact transport observations before admission; preserve same-timestamp observations and durable actor scope across capped restarts; capture empty initial cursors safely; reject rollback-prone endpoint pools in scheduled mode; and emit one bounded aggregate cycle summary. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index f63608e..1ad89cb 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -61,10 +61,12 @@ HTTP routes expose the same pipeline, including relationship/reference hydration ```ts await contrail.ingest(); // bounded Jetstream cycle -// Optional per-cycle overrides (defaults: 25s, 250 candidates, 4 MiB). +// Optional per-cycle overrides (defaults: 25s, 250 candidates, +// 250 distinct identity updates, and 4 MiB). await contrail.ingest({ maxDrainMs: 20_000, maxCandidates: 200, + maxIdentityUpdates: 200, maxSerializedBytes: 3 * 1024 * 1024, }); @@ -77,7 +79,7 @@ await contrail.runPersistent({ After a write to a user's PDS, `contrail.notify(uri)` can fetch the authoritative record immediately. Only an authoritative not-found response deletes local state; rate limits, server errors, timeouts, malformed responses, and network failures leave it unchanged. Authentication and abuse controls for the public HTTP operation remain under design. -Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Scheduled collection drops exact Jetstream observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity and source-filtered observations consume neither candidate nor byte limits, but their yielded cursors remain accounted. Bounded exact-observation hashes at the current microsecond cursor let restarts replay one microsecond of overlap without skipping equal-cursor siblings or recounting earlier ones. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact accounted cursor in the same transaction. Scheduled mode requires one pinned Jetstream endpoint; use `runPersistent()` for an Atcute failover pool. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. +Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Scheduled collection drops exact Jetstream observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity updates have an independent retained-count limit, coalesce by DID, and commit in database batches; excess global identity traffic is accounted but omitted so it cannot starve record progress. Source-filtered observations consume neither candidate nor byte limits, but their yielded cursors remain accounted. Durable actor scope comes from identities, relay discovery, and visible discoverable records, so a failed identity refresh cannot make a restarted isolate filter equal-cursor dependent siblings. Bounded exact-observation hashes at the current microsecond cursor let restarts replay one microsecond of overlap without skipping equal-cursor siblings or recounting earlier ones. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact accounted cursor in the same transaction. Scheduled mode requires one pinned Jetstream endpoint; use `runPersistent()` for an Atcute failover pool. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. Projection winner selection is guarded again inside the write transaction. Overlapping cron, persistent, notify, or backfill writers cannot commit a stale canonical row, derived projection, tombstone, or source checkpoint; a changed predecessor rolls the complete attempt back and retries from fresh durable state. diff --git a/packages/contrail/src/core/bootstrap.ts b/packages/contrail/src/core/bootstrap.ts index e4e5757..f42ff73 100644 --- a/packages/contrail/src/core/bootstrap.ts +++ b/packages/contrail/src/core/bootstrap.ts @@ -1,7 +1,8 @@ import type { ContrailConfig, Database, IngestEvent, Statement } from "./types"; import { recordTimeUs, createIngestEvent, ingestRecords } from "./ingest"; -import { getDependentNsids, recordsTableName } from "./types"; +import { getDependentNsids } from "./types"; import { + loadKnownActorDids, rebuildDerivedProjections, saveCursorStatement, saveServingSourcePositionStatement, @@ -511,25 +512,8 @@ export class DatabaseBootstrapTarget implements BootstrapTarget { if (getDependentNsids(this.config).length === 0) return undefined; if (this.knownDidsLoaded) return this.knownDids ?? new Set(); const known = this.knownDids ?? new Set(); - const identityRows = await this.db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - for (const row of identityRows.results ?? []) known.add(row.did); - // Relay discovery defines acquisition scope before concurrent PDS workers - // necessarily resolve every identity. Load it directly so a dependent - // collection arriving first is not mistaken for an unknown actor. - const backfillRows = await this.db - .prepare("SELECT DISTINCT did FROM backfills") - .all<{ did: string }>(); - for (const row of backfillRows.results ?? []) known.add(row.did); - for (const [shortName, collection] of Object.entries( - this.config.collections, - )) { - if (collection.discover === false) continue; - const rows = await this.db - .prepare(`SELECT DISTINCT did FROM ${recordsTableName(shortName)}`) - .all<{ did: string }>(); - for (const row of rows.results ?? []) known.add(row.did); + for (const did of await loadKnownActorDids(this.db, this.config)) { + known.add(did); } this.knownDids = known; this.knownDidsLoaded = true; diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index 770eb3c..78b38a2 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,6 +1,6 @@ export { initSchema, CONTRAIL_SCHEMA_VERSION } from "./schema"; export { getMeta, setMeta, getMetaNumber } from "./meta"; export { optimizeDatabase } from "./optimize"; -export { assertServingSourceCompatibility, getLastCursor, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; +export { assertServingSourceCompatibility, getLastCursor, loadKnownActorDids, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult, ServingSourcePosition } from "./records"; export type { RecordSource } from "../types"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index db11b3b..1567b9a 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -655,6 +655,30 @@ export async function getLastCursor(db: Database): Promise { return row ? row.time_us : null; } +/** Load durable actor-acquisition scope. Identity resolution is best-effort, so + * visible discoverable records and relay backfill rows are also authoritative + * evidence that an actor is known. This keeps dependent collection filtering + * stable across process restarts even when profile resolution failed. */ +export async function loadKnownActorDids( + db: Database, + config: ContrailConfig, +): Promise> { + const known = new Set(); + const durableRows = await db + .prepare("SELECT did FROM identities UNION SELECT did FROM backfills") + .all<{ did: string }>(); + for (const row of durableRows.results ?? []) known.add(row.did); + + for (const [shortName, collection] of Object.entries(config.collections)) { + if (collection.discover === false) continue; + const rows = await db + .prepare(`SELECT DISTINCT did FROM ${recordsTableName(shortName)}`) + .all<{ did: string }>(); + for (const row of rows.results ?? []) known.add(row.did); + } + return known; +} + export function saveCursorStatement( db: Database, timeUs: number, diff --git a/packages/contrail/src/core/identity.ts b/packages/contrail/src/core/identity.ts index a18120d..fbd46e8 100644 --- a/packages/contrail/src/core/identity.ts +++ b/packages/contrail/src/core/identity.ts @@ -1,5 +1,5 @@ import type { Did } from "@atcute/lexicons"; -import type { ContrailConfig, Database, Logger } from "./types"; +import type { ContrailConfig, Database, Logger, Statement } from "./types"; import { isDid, isHandle } from "@atcute/lexicons/syntax"; import { resolvePDS } from "./client"; @@ -134,15 +134,23 @@ export async function resolveActor( * partial rows confuse the rest of the pipeline). PDS column is left * untouched; it gets refreshed lazily via `getPDS` / next slingshot resolve. */ +export function applyIdentityEventStatement( + db: Database, + did: string, + handle: string, + updatedAt = Date.now(), +): Statement { + return db + .prepare("UPDATE identities SET handle = ?, resolved_at = ? WHERE did = ?") + .bind(handle, updatedAt, did); +} + export async function applyIdentityEvent( db: Database, did: string, handle: string ): Promise { - await db - .prepare("UPDATE identities SET handle = ?, resolved_at = ? WHERE did = ?") - .bind(handle, Date.now(), did) - .run(); + await applyIdentityEventStatement(db, did, handle).run(); } export async function refreshStaleIdentities( diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index e285c67..d0f3c07 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -11,7 +11,7 @@ import { optimizeIntervalMs, optimizeAnalysisLimit, } from "./types"; -import { initSchema, getLastCursor, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; +import { initSchema, getLastCursor, loadKnownActorDids, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; import { createIngestEvent, ingestRecords, @@ -19,7 +19,10 @@ import { type IngestDropCounts, type IngestWarningSamples, } from "./ingest"; -import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; +import { + refreshStaleIdentities, + applyIdentityEventStatement, +} from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; const BATCH_SIZE = 50; @@ -36,12 +39,14 @@ export const SCHEDULED_INGEST_METADATA_BYTES = 512; export const DEFAULT_SCHEDULED_INGEST_BUDGET = Object.freeze({ maxDrainMs: 25_000, maxCandidates: 250, + maxIdentityUpdates: 250, maxSerializedBytes: 4 * 1024 * 1024, }) satisfies ScheduledIngestBudget; export interface ScheduledIngestBudget { maxDrainMs: number; maxCandidates: number; + maxIdentityUpdates: number; maxSerializedBytes: number; } @@ -50,6 +55,8 @@ export interface ScheduledIngestOptions { maxDrainMs?: number; /** Maximum unique commit candidates retained by one drain. Default: 250. */ maxCandidates?: number; + /** Maximum distinct handle updates retained by one drain. Default: 250. */ + maxIdentityUpdates?: number; /** UTF-8 record bytes plus metadata allowances. Default: 4 MiB. */ maxSerializedBytes?: number; /** @deprecated Compatibility alias for maxDrainMs. */ @@ -69,6 +76,8 @@ export interface ScheduledIngestCollectionStats { commitObservations: number; identityObservations: number; retainedCandidates: number; + retainedIdentityUpdates: number; + identityUpdatesOmitted: number; exactDuplicatesDropped: number; cursorBoundaryDuplicatesDropped: number; resumeOverlapDropped: number; @@ -118,6 +127,11 @@ export function resolveScheduledIngestBudget( options.maxCandidates ?? DEFAULT_SCHEDULED_INGEST_BUDGET.maxCandidates, "maxCandidates", ), + maxIdentityUpdates: positiveInteger( + options.maxIdentityUpdates ?? + DEFAULT_SCHEDULED_INGEST_BUDGET.maxIdentityUpdates, + "maxIdentityUpdates", + ), maxSerializedBytes: positiveInteger( options.maxSerializedBytes ?? DEFAULT_SCHEDULED_INGEST_BUDGET.maxSerializedBytes, @@ -415,6 +429,8 @@ export async function ingestEvents( commitObservations: 0, identityObservations: 0, retainedCandidates: 0, + retainedIdentityUpdates: 0, + identityUpdatesOmitted: 0, exactDuplicatesDropped: 0, cursorBoundaryDuplicatesDropped: 0, resumeOverlapDropped: 0, @@ -603,7 +619,17 @@ export async function ingestEvents( } else if (event.kind === "identity") { stats.identityObservations++; if (await accountSourceItem(event, normalizedJson(event))) return; - identityUpdates.set(event.did, event.identity.handle); + if ( + identityUpdates.has(event.did) || + identityUpdates.size < budget.maxIdentityUpdates + ) { + identityUpdates.set(event.did, event.identity.handle); + stats.retainedIdentityUpdates = identityUpdates.size; + } else { + // Handle updates are best-effort and do not define record projection. + // Account overflow so global identity traffic cannot starve commits. + stats.identityUpdatesOmitted++; + } } else { await accountSourceItem(event, normalizedJson(event)); } @@ -768,10 +794,7 @@ export async function runIngestCycle( if (s.cachedKnownDids) { knownDids = s.cachedKnownDids; } else { - const result = await db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - knownDids = new Set((result.results ?? []).map((r) => r.did)); + knownDids = await loadKnownActorDids(db, config); s.cachedKnownDids = knownDids; } } @@ -840,17 +863,26 @@ export async function runIngestCycle( } } - // Apply handle changes from #identity events. UPDATE-only, so unknown DIDs - // are no-ops. Failures are sampled into the bounded cycle summary. + // Apply the independently capped handle changes in bounded database batches. + // UPDATE-only statements make unknown DIDs no-ops without spending one D1 + // round-trip apiece. Failures are sampled into the bounded cycle summary. let identityUpdateFailures = 0; - for (const [did, handle] of identityUpdates) { + const identityEntries = [...identityUpdates]; + for (let index = 0; index < identityEntries.length; index += BATCH_SIZE) { + const chunk = identityEntries.slice(index, index + BATCH_SIZE); try { - await applyIdentityEvent(db, did, handle); + const updatedAt = Date.now(); + await db.batch( + chunk.map(([did, handle]) => + applyIdentityEventStatement(db, did, handle, updatedAt), + ), + ); + databaseSubBatchesCommitted++; } catch (error) { - identityUpdateFailures++; + identityUpdateFailures += chunk.length; if (warningSamples.samples.length < warningSamples.maxSamples) { warningSamples.samples.push( - `identity update failed for ${did}: ${String(error)}`.slice( + `identity update batch failed (${chunk.length}, first=${chunk[0]?.[0]}): ${String(error)}`.slice( 0, MAX_DIAGNOSTIC_SAMPLE_LENGTH, ), @@ -968,6 +1000,8 @@ export async function runIngestCycle( commit_observations: stats.commitObservations, identity_observations: stats.identityObservations, retained_candidates: stats.retainedCandidates, + retained_identity_updates: stats.retainedIdentityUpdates, + identity_updates_omitted: stats.identityUpdatesOmitted, exact_duplicates_dropped: stats.exactDuplicatesDropped, cursor_boundary_duplicates_dropped: stats.cursorBoundaryDuplicatesDropped, @@ -978,6 +1012,7 @@ export async function runIngestCycle( source_inconsistencies: stats.sourceInconsistencies, serialized_candidate_bytes: stats.serializedCandidateBytes, max_candidates: budget.maxCandidates, + max_identity_updates: budget.maxIdentityUpdates, max_serialized_bytes: budget.maxSerializedBytes, max_drain_ms: budget.maxDrainMs, starting_cursor: cursor, diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index 877526b..6b19d96 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -8,7 +8,7 @@ import { jetstreamUrlOption, resolveConfig, } from "./types"; -import { initSchema, getLastCursor, saveCursorStatement, saveOrderedSourcePositionStatement } from "./db"; +import { initSchema, getLastCursor, loadKnownActorDids, saveCursorStatement, saveOrderedSourcePositionStatement } from "./db"; import { createIngestEvent, ingestRecords, recordTimeUs } from "./ingest"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; @@ -66,10 +66,7 @@ export async function runPersistent( const dependentCollections: Set = new Set(getDependentNsids(config)); let knownDids: Set | undefined; if (dependentCollections.size > 0) { - const result = await db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - knownDids = new Set((result.results ?? []).map((r) => r.did)); + knownDids = await loadKnownActorDids(db, config); state.cachedKnownDids = knownDids; log.log(`Loaded ${knownDids.size} known DIDs from database`); } diff --git a/packages/contrail/src/worker/index.ts b/packages/contrail/src/worker/index.ts index 8918a7b..8cc81af 100644 --- a/packages/contrail/src/worker/index.ts +++ b/packages/contrail/src/worker/index.ts @@ -43,7 +43,8 @@ export interface CreateWorkerOptions { lexicons?: object[]; /** Enable stable discovery and Lexicon routes for anonymous remote clients. */ publicService?: PublicServiceOptions; - /** Count, byte, and drain-time limits for each scheduled Jetstream cycle. */ + /** Record count, identity count, byte, and drain-time limits for each + * scheduled Jetstream cycle. */ scheduledIngest?: ScheduledIngestOptions; /** Bounded pending-account retry slice after each scheduled ingest. Enabled * by default; pass `false` to disable or options to tune its budget. */ diff --git a/packages/contrail/tests/ingest-hang.test.ts b/packages/contrail/tests/ingest-hang.test.ts index 2c7bebd..1dd0782 100644 --- a/packages/contrail/tests/ingest-hang.test.ts +++ b/packages/contrail/tests/ingest-hang.test.ts @@ -70,11 +70,13 @@ function commitEvent( function budget(overrides: Partial<{ maxDrainMs: number; maxCandidates: number; + maxIdentityUpdates: number; maxSerializedBytes: number; }> = {}) { return { maxDrainMs: 5_000, maxCandidates: 100, + maxIdentityUpdates: 100, maxSerializedBytes: 1024 * 1024, ...overrides, }; @@ -351,6 +353,9 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { expect(() => resolveScheduledIngestBudget({ maxCandidates: 0 })).toThrow( "maxCandidates must be a positive finite integer", ); + expect(() => + resolveScheduledIngestBudget({ maxIdentityUpdates: 0 }), + ).toThrow("maxIdentityUpdates must be a positive finite integer"); expect(() => resolveScheduledIngestBudget({ maxSerializedBytes: Number.NaN }), ).toThrow("maxSerializedBytes must be a positive finite integer"); @@ -391,6 +396,87 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { }); }); + it("caps distinct identity updates without letting global identity traffic stop record progress", async () => { + let produced = 0; + jetstream.script = async function* (self) { + for (let index = 0; index < 5; index++) { + produced++; + self.cursor = 1_500_000 + index; + yield { + kind: "identity" as const, + time_us: self.cursor, + did: `did:plc:identity-${index}`, + identity: { + did: `did:plc:identity-${index}`, + handle: `identity-${index}.test`, + seq: index, + time: "2026-04-01T10:00:00Z", + }, + }; + } + self.cursor = 1_500_005; + produced++; + yield commitEvent( + "did:plc:author", + "community.lexicon.calendar.event", + self.cursor, + "after-identities", + ); + }; + + const result = await ingestEvents( + discoverableConfig(), + 999_999, + budget({ maxIdentityUpdates: 3 }), + ); + + expect(result.identityUpdates.size).toBe(3); + expect(result.events.map((event) => event.rkey)).toEqual([ + "after-identities", + ]); + expect(produced).toBe(6); + expect(result.stats).toMatchObject({ + observedSourceItems: 6, + identityObservations: 5, + retainedIdentityUpdates: 3, + identityUpdatesOmitted: 2, + stopReason: "idle", + lastAccountedCursor: 1_500_005, + safeEndingCursor: 1_500_005, + }); + }); + + it("coalesces repeated identity updates without spending the distinct-update cap", async () => { + jetstream.script = async function* (self) { + for (let index = 0; index < 3; index++) { + self.cursor = 1_600_000 + index; + yield { + kind: "identity" as const, + time_us: self.cursor, + did: "did:plc:identity", + identity: { + did: "did:plc:identity", + handle: `identity-${index}.test`, + seq: index, + time: "2026-04-01T10:00:00Z", + }, + }; + } + }; + + const result = await ingestEvents( + discoverableConfig(), + 999_999, + budget({ maxIdentityUpdates: 2 }), + ); + + expect(result.identityUpdates).toEqual( + new Map([["did:plc:identity", "identity-2.test"]]), + ); + expect(result.stats.retainedIdentityUpdates).toBe(1); + expect(result.stats.stopReason).toBe("idle"); + }); + it("retains the byte-threshold-crossing candidate and does not split equal cursors", async () => { const firstRecord = { value: "a" }; const secondRecord = { value: "bbbb" }; diff --git a/packages/contrail/tests/scheduled-ingest.test.ts b/packages/contrail/tests/scheduled-ingest.test.ts index 12d9391..8092753 100644 --- a/packages/contrail/tests/scheduled-ingest.test.ts +++ b/packages/contrail/tests/scheduled-ingest.test.ts @@ -45,8 +45,10 @@ vi.mock("@atcute/jetstream", () => { import { getLastCursor, + initSchema, resolveConfig, runIngestCycle, + type Database, type Logger, } from "../src/index"; import { createTestDb } from "./helpers"; @@ -194,6 +196,40 @@ describe("bounded scheduled ingest cycles", () => { expect(summaries[2]?.cursor_boundary_duplicates_dropped).toBe(2); }); + it("restores actor scope from a projected record before replaying dependent siblings", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value, true); + const timeUs = 1_200_000; + source.events = [ + commit(timeUs, "discover-actor"), + { + ...commit(timeUs, "dependent-sibling"), + commit: { + ...commit(timeUs, "dependent-sibling").commit, + collection: "app.bsky.graph.follow", + }, + }, + ]; + const oneCandidate = { ...budget, maxCandidates: 1 }; + + await runIngestCycle(db, configured, oneCandidate); + expect( + await db.prepare("SELECT did FROM identities WHERE did = ?").bind("actor").first(), + ).toBeNull(); + expect( + await db.prepare("SELECT rkey FROM records_event").first<{ rkey: string }>(), + ).toEqual({ rkey: "discover-actor" }); + + // runIngestCycle receives no shared IngestState, modeling a fresh scheduled + // isolate. The exact discovery event is dropped at the cursor boundary, so + // durable projection scope must still admit its dependent sibling. + await runIngestCycle(db, configured, oneCandidate); + expect( + await db.prepare("SELECT rkey FROM records_follow").first<{ rkey: string }>(), + ).toEqual({ rkey: "dependent-sibling" }); + }); + it("persists Atcute's captured initial cursor after an empty first drain", async () => { const db = createTestDb(); const output = logger(); @@ -235,6 +271,66 @@ describe("bounded scheduled ingest cycles", () => { expect(source.requestedCursors).toHaveLength(0); }); + it("batches the independently capped identity updates", async () => { + const real = createTestDb(); + const output = logger(); + const configured = config(output.value); + await initSchema(real, configured); + await real + .prepare( + "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, NULL, NULL, 0), (?, NULL, NULL, 0)", + ) + .bind("actor-a", "actor-b") + .run(); + + let batchCalls = 0; + const db: Database = { + ...real, + batch(statements) { + batchCalls++; + return real.batch(statements); + }, + }; + source.events = [ + { + kind: "identity", + time_us: 1_000_001, + did: "actor-a", + identity: { + did: "actor-a", + handle: "a.test", + seq: 1, + time: "2026-04-01T10:00:00Z", + }, + }, + { + kind: "identity", + time_us: 1_000_002, + did: "actor-b", + identity: { + did: "actor-b", + handle: "b.test", + seq: 2, + time: "2026-04-01T10:00:00Z", + }, + }, + ] as unknown as typeof source.events; + + await runIngestCycle(db, configured, { + ...budget, + maxIdentityUpdates: 2, + }); + + expect(batchCalls).toBe(2); // one identity batch, then one cursor batch + expect( + (await real.prepare("SELECT did, handle FROM identities ORDER BY did").all()) + .results, + ).toEqual([ + { did: "actor-a", handle: "a.test" }, + { did: "actor-b", handle: "b.test" }, + ]); + }); + it("keeps admission diagnostics bounded as source volume grows", async () => { const db = createTestDb(); const output = logger(); -- 2.51.2 From beb13ca447aaf3756499193486c58451f7f14fc8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:28:31 +0000 Subject: [PATCH 15/34] Version Packages --- .changeset/bounded-scheduled-ingest.md | 5 ----- packages/contrail/CHANGELOG.md | 6 ++++++ packages/contrail/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/bounded-scheduled-ingest.md diff --git a/.changeset/bounded-scheduled-ingest.md b/.changeset/bounded-scheduled-ingest.md deleted file mode 100644 index c796f73..0000000 --- a/.changeset/bounded-scheduled-ingest.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": patch ---- - -Bound scheduled Jetstream cycles by retained candidate count, distinct identity updates, and serialized bytes; batch identity writes; drop exact transport observations before admission; preserve same-timestamp observations and durable actor scope across capped restarts; capture empty initial cursors safely; reject rollback-prone endpoint pools in scheduled mode; and emit one bounded aggregate cycle summary. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 042e5ce..f82c123 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.20.1 + +### Patch Changes + +- 92fedb0: Bound scheduled Jetstream cycles by retained candidate count, distinct identity updates, and serialized bytes; batch identity writes; drop exact transport observations before admission; preserve same-timestamp observations and durable actor scope across capped restarts; capture empty initial cursors safely; reject rollback-prone endpoint pools in scheduled mode; and emit one bounded aggregate cycle summary. + ## 0.20.0 ### Minor Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 170b580..8191324 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.20.0", + "version": "0.20.1", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 65621a507f0813a011fad44ffdebc459848e3ac3 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:29:45 +0200 Subject: [PATCH 16/34] updte docs --- README.md | 132 ------------------ ...tting-started.md => 01-getting-started.md} | 0 docs/{01-client.md => 02-client.md} | 0 ...and-query.md => 03-configure-and-query.md} | 0 ...-cloudflare.md => 04-deploy-cloudflare.md} | 0 docs/advanced/README.md | 9 -- 6 files changed, 141 deletions(-) rename docs/{00-getting-started.md => 01-getting-started.md} (100%) rename docs/{01-client.md => 02-client.md} (100%) rename docs/{02-configure-and-query.md => 03-configure-and-query.md} (100%) rename docs/{03-deploy-cloudflare.md => 04-deploy-cloudflare.md} (100%) delete mode 100644 docs/advanced/README.md diff --git a/README.md b/README.md index deaa28d..21cd54a 100644 --- a/README.md +++ b/README.md @@ -17,141 +17,9 @@ It provides: Cloudflare Workers with D1 is the primary deployment target. Node.js with SQLite or PostgreSQL is also supported. -For local development, a directory containing only `contrail.config.ts` can run `contrail dev`. It resolves source Lexicons, creates a resumable SQLite database, backfills it, follows Jetstream in bounded cycles, exposes the public XRPC service on `http://127.0.0.1:8787`, and generates the local consumer types/client automatically. Wrangler projects continue to use their local D1 automatically. - -## Install - -```bash -pnpm add @atmo-dev/contrail -``` - -## Minimal Worker - -```ts -// src/contrail.config.ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -export const config: ContrailConfig = { - namespace: "com.example", - collections: { - event: { - collection: "community.lexicon.calendar.event", - queryable: { - mode: {}, - startsAt: { type: "range" }, - }, - searchable: ["name", "description"], - }, - }, -}; -``` - -```ts -// src/worker.ts -import { createWorker } from "@atmo-dev/contrail/worker"; -import { config } from "./contrail.config"; - -export default createWorker(config); -``` - -Add a D1 binding and one-minute cron to `wrangler.jsonc`: - -```jsonc -{ - "main": "src/worker.ts", - "d1_databases": [ - { "binding": "DB", "database_name": "contrail", "database_id": "..." }, - ], - "triggers": { "crons": ["*/1 * * * *"] }, -} -``` - -Then deploy and backfill: - -```bash -pnpm wrangler d1 create contrail -pnpm wrangler deploy -pnpm contrail backfill --remote -``` - -Query the resulting AppView: - -```text -GET /xrpc/com.example.event.listRecords?startsAtMin=2026-01-01&limit=10 -GET /status -``` - -The JSON status response reports live cursor lag, indexed records, known backfill progress, and mutually exclusive pending/retrying/failed account counts. Failed PDS work is retried automatically in small scheduled slices with backoff up to 48 hours. - -## Lexicons - -Generate query Lexicons from the Contrail config and detect checked-in drift: - -```bash -pnpm contrail lexicons generate -pnpm contrail lexicons check -``` - -Use `contrail lexicons all` to generate Contrail methods, pull referenced source Lexicons, and generate TypeScript types in one pass. The `pull` and `types` actions are also available separately. Contrail updates `lex.config.js` only when the file carries its generated marker; user-owned Atcute configuration is preserved. Pass `--no-atcute-config` to skip creating or checking that generated file. Contrail owns its config-specific query generation while delegating generic pulling and TypeScript generation to [Atcute](https://github.com/mary-ext/atcute). - -## Public read-through services - -A deployment can publish a validated API description and Lexicon bundle for independent typed clients: - -```ts -export default createWorker(config, { - lexicons, - publicService: { endpoint: "https://api.example.com" }, -}); -``` - -Contrail remains a read-through cache over public AT Protocol data: anonymous reads may resolve identities, fetch missing public records, and improve profile or feed projections. Custom query handlers are public when they have matching authored query Lexicons. Anonymous discovery uses the HTTPS origin directly and does not require a service DID. The optional `notifyOfUpdate` procedure is not advertised in the anonymous read contract. - -Consumers connect and generate Atcute types with one command: - -```bash -pnpx @atmo-dev/contrail connect https://api.example.com -``` - -The generated client sends anonymous requests directly. Protected methods use a least-privilege OAuth scope with the provider's exact fragmented service audience and one `lxm` per protected method. They lazily discover service auth and fail closed on endpoint, base-DID, audience, scope, or protected-method mismatches. The content-addressed Lexicon digest remains in the version-2 provider lock; the complete provider method set is not pinned at runtime, so additive anonymous deployments do not interrupt existing calls. - -An application that owns the provider source can generate the same typed surface before deployment without creating a provider lock: - -```bash -pnpx @atmo-dev/contrail connect ../api/src/contrail.config.ts -pnpx @atmo-dev/contrail dev --config ../api/src/contrail.config.ts -``` - -The generated module exports `createLocalContrailClient()` for selecting the loopback service while retaining any existing production lock and default target. - -Public-service mode requires `orderedSource`; `getCursor` then returns the committed opaque `{ source, epoch, cursor }` position of that primary source. Compare complete positions for equality only; a source or epoch change requires a full client refetch. To avoid racing ingestion, read a position before and after a query and accept the query snapshot only when both positions match. Existing non-public deployments without `orderedSource` retain the legacy `time_us`, `date`, and `seconds_ago` response. - -## Other databases - -```ts -import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; -import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; -``` - -The runnable [`apps/sqlite`](apps/sqlite) example wires the standard backfill CLI to a local SQLite file, including the optional Alluvium base/archive path. - ## Documentation 1. [Get started locally](docs/00-getting-started.md) 2. [Add the typed client](docs/01-client.md) 3. [Configure and query](docs/02-configure-and-query.md) 4. [Deploy to Cloudflare Workers](docs/03-deploy-cloudflare.md) - -See [advanced topics](docs/advanced/README.md) for other runtimes and features. - -## Repository layout - -There is one published package and one implementation: - -```text -packages/contrail/ @atmo-dev/contrail -``` - -The previous AppView, base, community, authority, record-host, sync, and Lexicon packages have been removed. - -See [development.md](development.md) for repository commands. diff --git a/docs/00-getting-started.md b/docs/01-getting-started.md similarity index 100% rename from docs/00-getting-started.md rename to docs/01-getting-started.md diff --git a/docs/01-client.md b/docs/02-client.md similarity index 100% rename from docs/01-client.md rename to docs/02-client.md diff --git a/docs/02-configure-and-query.md b/docs/03-configure-and-query.md similarity index 100% rename from docs/02-configure-and-query.md rename to docs/03-configure-and-query.md diff --git a/docs/03-deploy-cloudflare.md b/docs/04-deploy-cloudflare.md similarity index 100% rename from docs/03-deploy-cloudflare.md rename to docs/04-deploy-cloudflare.md diff --git a/docs/advanced/README.md b/docs/advanced/README.md deleted file mode 100644 index 49e9282..0000000 --- a/docs/advanced/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Advanced topics - -Start with the four-page happy path: [local AppView](../00-getting-started.md), [typed client](../01-client.md), [configuration and queries](../02-configure-and-query.md), and [Cloudflare deployment](../03-deploy-cloudflare.md). - -Optional features: - -- [Outbox](./outbox.md) -- [Feeds](./feeds.md) -- [Labels](./labels.md) -- 2.51.2 From 2202c5eca9c6c5a3b4829087fd7df7f61d55e8f2 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:36:49 +0200 Subject: [PATCH 17/34] add init cli command --- .changeset/soft-clouds-init.md | 5 ++ README.md | 8 +-- docs/01-getting-started.md | 12 +++- docs/02-client.md | 2 +- docs/03-configure-and-query.md | 2 - packages/contrail/README.md | 6 +- packages/contrail/src/cli.ts | 2 + packages/contrail/src/cli/commands/init.ts | 47 ++++++++++++++ packages/contrail/tests/init.test.ts | 71 ++++++++++++++++++++++ 9 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 .changeset/soft-clouds-init.md create mode 100644 packages/contrail/src/cli/commands/init.ts create mode 100644 packages/contrail/tests/init.test.ts diff --git a/.changeset/soft-clouds-init.md b/.changeset/soft-clouds-init.md new file mode 100644 index 0000000..f472ad6 --- /dev/null +++ b/.changeset/soft-clouds-init.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Add `contrail init [directory]` to create a ready-to-run starter `contrail.config.ts` without overwriting an existing config. diff --git a/README.md b/README.md index 21cd54a..639f67c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Cloudflare Workers with D1 is the primary deployment target. Node.js with SQLite ## Documentation -1. [Get started locally](docs/00-getting-started.md) -2. [Add the typed client](docs/01-client.md) -3. [Configure and query](docs/02-configure-and-query.md) -4. [Deploy to Cloudflare Workers](docs/03-deploy-cloudflare.md) +1. [Get started locally](docs/01-getting-started.md) +2. [Add the typed client](docs/02-client.md) +3. [Configure and query](docs/03-configure-and-query.md) +4. [Deploy to Cloudflare Workers](docs/04-deploy-cloudflare.md) diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index 2e5cad0..fbdb23b 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -2,10 +2,16 @@ Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. -Create an empty directory with one file: +Create a project with the starter config: + +```bash +pnpx @atmo-dev/contrail init my-appview +cd my-appview +``` + +This creates `contrail.config.ts`: ```ts -// contrail.config.ts export default { namespace: "com.example", collections: { @@ -29,4 +35,4 @@ Contrail resolves the Lexicons, backfills existing records, follows new records, curl 'http://127.0.0.1:8787/xrpc/com.example.event.listRecords?limit=10' ``` -Replace the collection and fields with your own. Next: [add the typed client to your app](./01-client.md). +Replace the collection and fields with your own. Next: [add the typed client to your app](./02-client.md). diff --git a/docs/02-client.md b/docs/02-client.md index e498f93..2aa477b 100644 --- a/docs/02-client.md +++ b/docs/02-client.md @@ -31,4 +31,4 @@ for (const event of response.data.records) { The method name, parameters, and response are all typed from the Lexicons. Re-run `connect` when the AppView config changes. -Next: [configure filters, sorting, and hydration](./02-configure-and-query.md). To use an existing deployed AppView, pass its HTTPS URL to `contrail connect` instead of a config path. +Next: [configure filters, sorting, and hydration](./03-configure-and-query.md). To use an existing deployed AppView, pass its HTTPS URL to `contrail connect` instead of a config path. diff --git a/docs/03-configure-and-query.md b/docs/03-configure-and-query.md index 2b5e27f..a5d3bae 100644 --- a/docs/03-configure-and-query.md +++ b/docs/03-configure-and-query.md @@ -79,5 +79,3 @@ Every record has `uri`, `cid`, and its original record body in `value`. Hydrated Pass the returned opaque `cursor` into the same query to get the next page. `limit` defaults to 50 and may be 1–200. Full-text `search` works with D1 and PostgreSQL. The zero-config local SQLite AppView does not provide full-text search. - -Next: [deploy to Cloudflare Workers](./03-deploy-cloudflare.md). diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 1ad89cb..e30e3dd 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -166,12 +166,16 @@ Backups retain the database's change-log generation ID, positions, leases, boots ## Local development -A project containing only `contrail.config.ts` can start a complete local service: +Seed a new project with a basic `contrail.config.ts`, then start a complete local service: ```bash +contrail init my-appview +cd my-appview contrail dev ``` +Run `contrail init` without a directory to write the config in the current directory. The command refuses to replace a config in any standard auto-detected location. + Without a Wrangler config this creates or resumes `.contrail/dev.sqlite`, resolves missing configured record/ref Lexicons from the AT Protocol network without overwriting project-owned schemas, runs PDS backfill, serves the complete public Contrail discovery/XRPC/Lexicon surface at `http://127.0.0.1:8787`, and runs bounded Jetstream ingestion every minute. It never creates or changes a deployment provider lock. Add `.contrail/` to the project ignore file. Existing Wrangler projects retain the prior D1 development behavior automatically. Useful SQLite options include: ```bash diff --git a/packages/contrail/src/cli.ts b/packages/contrail/src/cli.ts index 2aa6d9a..101482b 100644 --- a/packages/contrail/src/cli.ts +++ b/packages/contrail/src/cli.ts @@ -12,9 +12,11 @@ import { registerAppendScheduled } from "./cli/commands/append-scheduled.js"; import { registerConnect } from "./cli/commands/connect.js"; import { registerLexicons } from "./cli/commands/lexicons.js"; import { registerChanges } from "./cli/commands/changes.js"; +import { registerInit } from "./cli/commands/init.js"; const cli = cac("contrail"); +registerInit(cli); registerBackfill(cli); registerDev(cli); registerAppendScheduled(cli); diff --git a/packages/contrail/src/cli/commands/init.ts b/packages/contrail/src/cli/commands/init.ts new file mode 100644 index 0000000..5d95e95 --- /dev/null +++ b/packages/contrail/src/cli/commands/init.ts @@ -0,0 +1,47 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import type { CAC } from "cac"; +import { findConfigFile } from "../../cli-config.js"; + +export const STARTER_CONFIG = `export default { + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { startsAt: { type: "range" } }, + }, + }, +}; +`; + +/** Create a starter config without replacing an existing Contrail config. */ +export async function seedConfig(directory: string): Promise { + const root = resolve(directory); + await mkdir(root, { recursive: true }); + + const existing = findConfigFile(root); + if (existing) { + throw new Error(`A Contrail config already exists at ${existing}`); + } + + const path = join(root, "contrail.config.ts"); + try { + await writeFile(path, STARTER_CONFIG, { flag: "wx" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new Error(`A Contrail config already exists at ${path}`); + } + throw error; + } + return path; +} + +export function registerInit(cli: CAC): void { + cli + .command("init [directory]", "Create a starter contrail.config.ts") + .action(async (directory: string | undefined) => { + const path = await seedConfig(directory ?? process.cwd()); + const displayPath = relative(process.cwd(), path) || "contrail.config.ts"; + console.log(`Created ${displayPath}`); + }); +} diff --git a/packages/contrail/tests/init.test.ts b/packages/contrail/tests/init.test.ts new file mode 100644 index 0000000..c387344 --- /dev/null +++ b/packages/contrail/tests/init.test.ts @@ -0,0 +1,71 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadConfig } from "../src/cli-config"; +import { seedConfig, STARTER_CONFIG } from "../src/cli/commands/init"; + +const roots: string[] = []; + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +async function temporaryRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), "contrail-init-")); + roots.push(root); + return root; +} + +describe("contrail init", () => { + it("creates a loadable starter config", async () => { + const root = await temporaryRoot(); + const path = await seedConfig(root); + + expect(path).toBe(join(root, "contrail.config.ts")); + expect(await readFile(path, "utf8")).toBe(STARTER_CONFIG); + expect(await loadConfig(path)).toEqual({ + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { startsAt: { type: "range" } }, + }, + }, + }); + }); + + it("creates a requested directory", async () => { + const root = await temporaryRoot(); + const nested = join(root, "my-appview"); + + await expect(seedConfig(nested)).resolves.toBe( + join(nested, "contrail.config.ts"), + ); + }); + + it("does not replace an existing config", async () => { + const root = await temporaryRoot(); + const path = join(root, "contrail.config.ts"); + const original = "export default { keep: true };\n"; + await writeFile(path, original); + + await expect(seedConfig(root)).rejects.toThrow( + `A Contrail config already exists at ${path}`, + ); + expect(await readFile(path, "utf8")).toBe(original); + }); + + it("does not add a second config when a discovered config exists", async () => { + const root = await temporaryRoot(); + const path = join(root, "src", "contrail.config.ts"); + await mkdir(join(root, "src")); + await writeFile(path, "export default {};\n"); + + await expect(seedConfig(root)).rejects.toThrow( + `A Contrail config already exists at ${path}`, + ); + }); +}); -- 2.51.2 From 3d433dce09ea6e7d550539d50cb6fe2ca635b440 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:39:31 +0000 Subject: [PATCH 18/34] Version Packages --- .changeset/soft-clouds-init.md | 5 ----- packages/contrail/CHANGELOG.md | 6 ++++++ packages/contrail/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/soft-clouds-init.md diff --git a/.changeset/soft-clouds-init.md b/.changeset/soft-clouds-init.md deleted file mode 100644 index f472ad6..0000000 --- a/.changeset/soft-clouds-init.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": minor ---- - -Add `contrail init [directory]` to create a ready-to-run starter `contrail.config.ts` without overwriting an existing config. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index f82c123..ab4cb8a 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.21.0 + +### Minor Changes + +- 2202c5e: Add `contrail init [directory]` to create a ready-to-run starter `contrail.config.ts` without overwriting an existing config. + ## 0.20.1 ### Patch Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 8191324..deda4ce 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.20.1", + "version": "0.21.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 28f94c92d9e0bcdd9deb690e68def980ced52b0a Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:10:49 +0200 Subject: [PATCH 19/34] allow dev without lexicons --- .changeset/calm-winds-connect.md | 5 + packages/contrail/README.md | 6 +- packages/contrail/src/cli/commands/connect.ts | 14 +- packages/contrail/src/cli/commands/dev.ts | 9 +- packages/contrail/src/cli/dev-lexicons.ts | 105 +++++++++++- packages/contrail/src/cli/shared.ts | 17 ++ packages/contrail/tests/dev-lexicons.test.ts | 153 +++++++++++++++++- 7 files changed, 297 insertions(+), 12 deletions(-) create mode 100644 .changeset/calm-winds-connect.md diff --git a/.changeset/calm-winds-connect.md b/.changeset/calm-winds-connect.md new file mode 100644 index 0000000..1d9be89 --- /dev/null +++ b/.changeset/calm-winds-connect.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": patch +--- + +Allow config-backed `contrail dev` and `contrail connect` to continue after confirmation when configured Lexicons cannot yet be resolved, omitting incomplete schema closures and generating untyped values for affected records. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index e30e3dd..859475b 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -176,7 +176,11 @@ contrail dev Run `contrail init` without a directory to write the config in the current directory. The command refuses to replace a config in any standard auto-detected location. -Without a Wrangler config this creates or resumes `.contrail/dev.sqlite`, resolves missing configured record/ref Lexicons from the AT Protocol network without overwriting project-owned schemas, runs PDS backfill, serves the complete public Contrail discovery/XRPC/Lexicon surface at `http://127.0.0.1:8787`, and runs bounded Jetstream ingestion every minute. It never creates or changes a deployment provider lock. Add `.contrail/` to the project ignore file. Existing Wrangler projects retain the prior D1 development behavior automatically. Useful SQLite options include: +Without a Wrangler config this creates or resumes `.contrail/dev.sqlite`, resolves missing configured record/ref Lexicons from the AT Protocol network without overwriting project-owned schemas, runs PDS backfill, serves the complete public Contrail discovery/XRPC/Lexicon surface at `http://127.0.0.1:8787`, and runs bounded Jetstream ingestion every minute. It never creates or changes a deployment provider lock. Add `.contrail/` to the project ignore file. Existing Wrangler projects retain the prior D1 development behavior automatically. + +When a configured Lexicon does not exist yet or cannot be resolved, interactive `contrail dev` and config-backed `contrail connect` warn and ask whether to continue with an incomplete local bundle. Unresolved documents and schemas that depend on them are omitted, and directly affected record values are generated as `unknown`; collections using `validate: true` still require their complete Lexicon closure. The prompt defaults to no, and non-interactive use fails closed unless `--yes` is passed. + +Useful SQLite options include: ```bash contrail dev --fresh # reset local SQLite first diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index 5f92304..286fb01 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -40,6 +40,7 @@ import { prepareDevLexicons, } from "../dev-lexicons.js"; import { generateLexiconTypesWithAtcute } from "../atcute.js"; +import { confirmUnresolvedLexicons } from "../shared.js"; const MAX_DISCOVERY_BYTES = 10 * 1024 * 1024; const REQUEST_TIMEOUT_MS = 15_000; @@ -63,6 +64,7 @@ interface ConnectOptions { skipClient?: boolean; update?: boolean; allowInsecureHttp?: boolean; + yes?: boolean; } export interface ProviderDefinition { @@ -482,6 +484,9 @@ export async function connectConfigSource(options: { out: string; lock?: string; endpoint?: string; + confirmUnresolvedLexicons?: ( + nsids: readonly string[], + ) => boolean | Promise; }): Promise<{ manifest: PublicServiceManifest; definition: ProviderDefinition; @@ -520,12 +525,13 @@ export async function connectConfigSource(options: { const outputRoot = resolveInsideRoot(projectRoot, options.out); const providerKey = "source"; const providerRoot = resolveInsideRoot(outputRoot, providerKey); - const lexicons = prepareDevLexicons( + const lexicons = await prepareDevLexicons( sourceConfig, projectRoot, workspaceRoot, providerRoot, configProjectRoot(configPath), + { confirmUnresolved: options.confirmUnresolvedLexicons }, ); const description = await describePublicService( sourceConfig, @@ -785,6 +791,10 @@ export function registerConnect(cli: CAC): void { "--no-generate", "Resolve the API without generating types or a client module", ) + .option( + "--yes, -y", + "Continue when configured Lexicons cannot be resolved", + ) .action(async (source: string, options: ConnectOptions) => { const endpoint = endpointSource(source); let api: ProviderDefinition; @@ -815,6 +825,8 @@ export function registerConnect(cli: CAC): void { root: options.root, out: options.out, lock: options.lock, + confirmUnresolvedLexicons: (nsids) => + confirmUnresolvedLexicons(nsids, options.yes === true), }); api = result.definition; target = result.target; diff --git a/packages/contrail/src/cli/commands/dev.ts b/packages/contrail/src/cli/commands/dev.ts index 1759c89..de43e57 100644 --- a/packages/contrail/src/cli/commands/dev.ts +++ b/packages/contrail/src/cli/commands/dev.ts @@ -20,6 +20,7 @@ import { prepareDevLexicons, } from "../dev-lexicons.js"; import { + confirmUnresolvedLexicons, promptYesNo, resolveConfig, resolveValidationLexicons, @@ -246,12 +247,16 @@ async function runSqliteDev( root, options.clientLexicons ?? defaultConsumerLexiconRoot(root), ); - const lexicons = prepareDevLexicons( + const lexicons = await prepareDevLexicons( config, root, lexiconWorkspace, clientLexiconRoot, sourceRoot, + { + confirmUnresolved: (nsids) => + confirmUnresolvedLexicons(nsids, options.yes === true), + }, ); const db = createSqliteDatabase(databasePath); const contrail = new Contrail({ ...config, db, lexicons }); @@ -398,7 +403,7 @@ export function registerDev(cli: CAC): void { .option("--concurrency ", "PDS backfill identity concurrency", { default: 100, }) - .option("--yes, -y", "Accept Wrangler backfill prompts") + .option("--yes, -y", "Accept confirmation prompts") .option("--alluvium", "Use Alluvium base + archive for SQLite bootstrap") .option("--alluvium-endpoint ", "Alluvium HTTP origin", { default: "https://alluvium-v0.atmo.tools", diff --git a/packages/contrail/src/cli/dev-lexicons.ts b/packages/contrail/src/cli/dev-lexicons.ts index 2af0f27..3e7d951 100644 --- a/packages/contrail/src/cli/dev-lexicons.ts +++ b/packages/contrail/src/cli/dev-lexicons.ts @@ -1,5 +1,11 @@ -import { existsSync, writeFileSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { + existsSync, + mkdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { isNsid } from "@atcute/lexicons/syntax"; import type { ContrailConfig } from "../core/types.js"; import { generateLexicons } from "../lexicons/generate.js"; import { pullLexiconsWithAtcute } from "./atcute.js"; @@ -10,9 +16,83 @@ export function defaultConsumerLexiconRoot(projectRoot: string): string { : "src/contrail/lexicons"; } +function referencedNsids(document: object): string[] { + const references = new Set(); + const visit = (value: unknown, key?: string) => { + if (typeof value === "string" && (key === "ref" || key === "refs")) { + const nsid = value.split("#", 1)[0]; + if (nsid?.includes(".")) references.add(nsid); + return; + } + if (Array.isArray(value)) { + for (const child of value) { + visit(child, key === "refs" ? "refs" : undefined); + } + } else if (value && typeof value === "object") { + for (const [childKey, child] of Object.entries(value)) { + visit(child, childKey); + } + } + }; + visit(document); + return [...references]; +} + +/** Stage only source documents whose complete external reference closure is + * available. Generated methods are rebuilt against this directory so values + * backed by an incomplete source schema become `unknown` rather than retaining + * a dangling reference that breaks downstream type generation. */ +function stageCompleteSourceLexicons( + workspaceRoot: string, + result: ReturnType, +): string { + const generatedIds = new Set(Object.keys(result.generated)); + const sources = new Map(); + for (const document of result.lexicons) { + const id = (document as { id?: unknown }).id; + if (typeof id === "string" && !generatedIds.has(id)) { + sources.set(id, document); + } + } + + const available = new Set([...generatedIds, ...sources.keys()]); + let changed = true; + while (changed) { + changed = false; + for (const [id, document] of sources) { + if (referencedNsids(document).some((nsid) => !available.has(nsid))) { + sources.delete(id); + available.delete(id); + changed = true; + } + } + } + + const directory = join(workspaceRoot, "dev-complete"); + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { recursive: true }); + for (const [id, document] of sources) { + if (!isNsid(id)) throw new Error(`Invalid Lexicon NSID: ${id}`); + const path = join(directory, ...id.split(".")) + ".json"; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(document, null, 2)}\n`); + } + return directory; +} + /** Resolve the exact public Lexicon bundle used by a config-backed local * service. Project-owned documents win over remotely pulled dependencies. */ -export function prepareDevLexicons( +export interface PrepareDevLexiconsOptions { + /** Ask whether preparation may continue with documents that could not be + * resolved. Omitted means fail closed. */ + confirmUnresolved?: ( + nsids: readonly string[], + ) => boolean | Promise; + /** @internal Test seam for network resolution. */ + pullLexicons?: typeof pullLexiconsWithAtcute; +} + +export async function prepareDevLexicons( config: ContrailConfig, projectRoot: string, workspaceRoot: string, @@ -21,7 +101,8 @@ export function prepareDevLexicons( defaultConsumerLexiconRoot(projectRoot), ), configRoot = projectRoot, -): object[] { + options: PrepareDevLexiconsOptions = {}, +): Promise { const pulled = join(workspaceRoot, "dev-pulled"); const roots = [...new Set([projectRoot, configRoot])]; const localSourceDirs = [ @@ -58,8 +139,20 @@ export function prepareDevLexicons( const current = JSON.stringify(missing); if (current === attempted) { + if (await options.confirmUnresolved?.(missing)) { + const complete = stageCompleteSourceLexicons(workspaceRoot, result); + return generateLexicons({ + config, + rootDir: workspaceRoot, + outputDir: output, + sourceDirs: [complete], + surface: "public", + writeAtcuteConfig: false, + quiet: true, + }).lexicons; + } throw new Error( - `Could not resolve required development Lexicons: ${missing.join(", ")}`, + `Could not resolve configured development Lexicons: ${missing.join(", ")}`, ); } attempted = current; @@ -84,7 +177,7 @@ export function prepareDevLexicons( )};\n`, ); console.log(`lexicons: resolving ${missing.join(", ")}`); - pullLexiconsWithAtcute(workspaceRoot, pullConfig); + (options.pullLexicons ?? pullLexiconsWithAtcute)(workspaceRoot, pullConfig); } throw new Error("Development Lexicon reference discovery did not converge"); } diff --git a/packages/contrail/src/cli/shared.ts b/packages/contrail/src/cli/shared.ts index 4ca8b12..3ca39ca 100644 --- a/packages/contrail/src/cli/shared.ts +++ b/packages/contrail/src/cli/shared.ts @@ -81,3 +81,20 @@ export async function promptYesNo( rl.close(); } } + +/** Warn before omitting unresolved documents from a temporary source bundle. */ +export async function confirmUnresolvedLexicons( + nsids: readonly string[], + autoYes: boolean, +): Promise { + console.warn("\nwarning: could not resolve these configured Lexicons:"); + for (const nsid of nsids) console.warn(` - ${nsid}`); + console.warn( + "Continuing omits them and schemas that depend on them from the local " + + "bundle. Directly affected record values will be generated as `unknown`.", + ); + console.warn( + "Collections with `validate: true` still cannot start without their complete Lexicon closure.", + ); + return promptYesNo("continue without these Lexicons?", false, autoYes); +} diff --git a/packages/contrail/tests/dev-lexicons.test.ts b/packages/contrail/tests/dev-lexicons.test.ts index 3f1c32b..0558a0e 100644 --- a/packages/contrail/tests/dev-lexicons.test.ts +++ b/packages/contrail/tests/dev-lexicons.test.ts @@ -32,7 +32,7 @@ function writeLexicon(root: string, nsid: string, document: object): void { } describe("contrail dev Lexicon preparation", () => { - it("reuses an unpublished source Lexicon from the stable Svelte consumer root", () => { + it("reuses an unpublished source Lexicon from the stable Svelte consumer root", async () => { const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); roots.push(root); const workspace = join(root, ".contrail"); @@ -62,7 +62,7 @@ describe("contrail dev Lexicon preparation", () => { review: { collection }, }, }; - const lexicons = prepareDevLexicons(config, root, workspace); + const lexicons = await prepareDevLexicons(config, root, workspace); const ids = lexicons.map( (document) => (document as { id?: string }).id, ); @@ -71,4 +71,153 @@ describe("contrail dev Lexicon preparation", () => { expect(ids).toContain("test.svelte.review.getRecord"); expect(ids).toContain("test.svelte.review.listRecords"); }); + + it("can continue with an unresolved collection Lexicon and generate unknown values", async () => { + const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); + roots.push(root); + const workspace = join(root, ".contrail"); + mkdirSync(workspace, { recursive: true }); + + const collection = "social.example.future.record"; + const config: ContrailConfig = { + namespace: "test.future", + profiles: [], + collections: { + future: { collection }, + }, + }; + const confirmed: string[][] = []; + const lexicons = await prepareDevLexicons( + config, + root, + workspace, + undefined, + undefined, + { + pullLexicons: () => {}, + confirmUnresolved: (nsids) => { + confirmed.push([...nsids]); + return true; + }, + }, + ); + + expect(confirmed).toEqual([[collection]]); + expect( + lexicons.some( + (document) => (document as { id?: string }).id === collection, + ), + ).toBe(false); + const list = lexicons.find( + (document) => + (document as { id?: string }).id === "test.future.future.listRecords", + ) as any; + expect(list.defs.record.properties.value).toEqual({ type: "unknown" }); + }); + + it("drops schemas with unresolved dependencies before generating the incomplete bundle", async () => { + const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); + roots.push(root); + const workspace = join(root, ".contrail"); + mkdirSync(workspace, { recursive: true }); + + const collection = "social.example.future.record"; + const dependency = "social.example.future.defs"; + const missing = "social.example.future.missing"; + writeLexicon(root, collection, { + lexicon: 1, + id: collection, + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + properties: { + subject: { type: "ref", ref: `${dependency}#main` }, + }, + }, + }, + }, + }); + writeLexicon(root, dependency, { + lexicon: 1, + id: dependency, + defs: { + main: { + type: "object", + properties: { + nested: { type: "ref", ref: `${missing}#main` }, + }, + }, + }, + }); + const confirmed: string[][] = []; + const lexicons = await prepareDevLexicons( + { + namespace: "test.future", + profiles: [], + collections: { future: { collection } }, + }, + root, + workspace, + undefined, + undefined, + { + pullLexicons: () => {}, + confirmUnresolved: (nsids) => { + confirmed.push([...nsids]); + return true; + }, + }, + ); + + expect(confirmed).toEqual([[missing]]); + expect( + lexicons.some((document) => { + const id = (document as { id?: string }).id; + return id === collection || id === dependency; + }), + ).toBe(false); + const list = lexicons.find( + (document) => + (document as { id?: string }).id === "test.future.future.listRecords", + ) as any; + expect(list.defs.record.properties.value).toEqual({ type: "unknown" }); + + const available = new Set( + lexicons.map((document) => (document as { id?: string }).id), + ); + expect( + JSON.stringify(lexicons).includes(`\"ref\":\"${missing}#main\"`), + ).toBe(false); + expect(available.has(missing)).toBe(false); + }); + + it("still fails closed when continuation is declined", async () => { + const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); + roots.push(root); + const workspace = join(root, ".contrail"); + mkdirSync(workspace, { recursive: true }); + + await expect( + prepareDevLexicons( + { + namespace: "test.future", + profiles: [], + collections: { + future: { collection: "social.example.future.record" }, + }, + }, + root, + workspace, + undefined, + undefined, + { + pullLexicons: () => {}, + confirmUnresolved: () => false, + }, + ), + ).rejects.toThrow("Could not resolve configured development Lexicons"); + }); }); -- 2.51.2 From de3f48adae3a84cb94061285e8e2015f325bde76 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:34:17 +0200 Subject: [PATCH 20/34] patch deps --- .changeset/calm-lexicons-run.md | 5 +++++ packages/contrail/package.json | 1 + pnpm-lock.yaml | 3 +++ 3 files changed, 9 insertions(+) create mode 100644 .changeset/calm-lexicons-run.md diff --git a/.changeset/calm-lexicons-run.md b/.changeset/calm-lexicons-run.md new file mode 100644 index 0000000..be20b80 --- /dev/null +++ b/.changeset/calm-lexicons-run.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": patch +--- + +Declare the byte-array helper used by Atcute's Lexicon CLI so `contrail dev` can resolve Lexicons in isolated `pnpx` installs. diff --git a/packages/contrail/package.json b/packages/contrail/package.json index deda4ce..01c3932 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -89,6 +89,7 @@ "@atcute/lexicons": "^2.0.3", "@atcute/oauth-types": "^1.0.1", "@atcute/tid": "1.1.4", + "@atcute/uint8array": "1.1.5", "@atcute/xrpc-server": "^2.0.2", "cac": "^7.0.0", "hono": "^4.13.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a97826f..a049815 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -275,6 +275,9 @@ importers: '@atcute/tid': specifier: 1.1.4 version: 1.1.4 + '@atcute/uint8array': + specifier: 1.1.5 + version: 1.1.5 '@atcute/xrpc-server': specifier: ^2.0.2 version: 2.0.2(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.3)(typescript@6.0.3) -- 2.51.2 From 43eaf21497e64db0937732e1e7d9514340e2004b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:36:26 +0000 Subject: [PATCH 21/34] Version Packages --- .changeset/calm-lexicons-run.md | 5 ----- packages/contrail/CHANGELOG.md | 6 ++++++ packages/contrail/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/calm-lexicons-run.md diff --git a/.changeset/calm-lexicons-run.md b/.changeset/calm-lexicons-run.md deleted file mode 100644 index be20b80..0000000 --- a/.changeset/calm-lexicons-run.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": patch ---- - -Declare the byte-array helper used by Atcute's Lexicon CLI so `contrail dev` can resolve Lexicons in isolated `pnpx` installs. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index ab4cb8a..1d91632 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.21.1 + +### Patch Changes + +- de3f48a: Declare the byte-array helper used by Atcute's Lexicon CLI so `contrail dev` can resolve Lexicons in isolated `pnpx` installs. + ## 0.21.0 ### Minor Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 01c3932..cb61300 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.21.0", + "version": "0.21.1", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 60e7b3a15ae8d6e81ea1d527da967e32d8e0c31c Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:23:29 +0200 Subject: [PATCH 22/34] fix again ^^ --- .changeset/tidy-lexicon-links.md | 5 +++ packages/contrail/src/cli/atcute.ts | 41 +++++++++++++++++++- packages/contrail/tests/atcute-cli.test.ts | 44 ++++++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 .changeset/tidy-lexicon-links.md create mode 100644 packages/contrail/tests/atcute-cli.test.ts diff --git a/.changeset/tidy-lexicon-links.md b/.changeset/tidy-lexicon-links.md new file mode 100644 index 0000000..741ec48 --- /dev/null +++ b/.changeset/tidy-lexicon-links.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": patch +--- + +Run Atcute's Lexicon CLI with an explicit module redirect for its byte-array helper so `contrail dev` works with pnpm's strictly isolated global package links. diff --git a/packages/contrail/src/cli/atcute.ts b/packages/contrail/src/cli/atcute.ts index a02539e..93fcbf5 100644 --- a/packages/contrail/src/cli/atcute.ts +++ b/packages/contrail/src/cli/atcute.ts @@ -1,9 +1,35 @@ import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; const require = createRequire(import.meta.url); +function dataModule(source: string): string { + return `data:text/javascript,${encodeURIComponent(source)}`; +} + +/** Create a preload module that redirects one bare import before starting a + * child CLI. This keeps undeclared transitive imports working in package + * managers whose content-addressed links fully isolate dependency trees. */ +export function createModuleRedirectRegistration( + specifier: string, + targetUrl: string, +): string { + const loader = dataModule( + `export async function resolve(specifier, context, nextResolve) { + if (specifier === ${JSON.stringify(specifier)}) { + return { url: ${JSON.stringify(targetUrl)}, shortCircuit: true }; + } + return nextResolve(specifier, context); +}`, + ); + return dataModule( + `import { register } from "node:module"; +register(${JSON.stringify(loader)}, import.meta.url);`, + ); +} + function runAtcute( action: "pull" | "generate", root: string, @@ -11,9 +37,22 @@ function runAtcute( ): void { const entry = require.resolve("@atcute/lex-cli"); const cli = join(dirname(entry), "..", "cli.mjs"); + // @atcute/lex-cli 3.3.0 imports this helper but declares it only as a dev + // dependency. Resolve Contrail's declared copy explicitly until upstream's + // package metadata includes it as a runtime dependency. + const registration = createModuleRedirectRegistration( + "@atcute/uint8array", + pathToFileURL(require.resolve("@atcute/uint8array")).href, + ); const result = spawnSync( process.execPath, - [cli, action, ...(configPath ? ["--config", configPath] : [])], + [ + "--import", + registration, + cli, + action, + ...(configPath ? ["--config", configPath] : []), + ], { cwd: root, stdio: "inherit", diff --git a/packages/contrail/tests/atcute-cli.test.ts b/packages/contrail/tests/atcute-cli.test.ts new file mode 100644 index 0000000..0d7cc08 --- /dev/null +++ b/packages/contrail/tests/atcute-cli.test.ts @@ -0,0 +1,44 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { createModuleRedirectRegistration } from "../src/cli/atcute"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("Atcute CLI module resolution", () => { + it("redirects an undeclared bare import in an isolated child process", () => { + const root = mkdtempSync(join(tmpdir(), "contrail-atcute-loader-")); + roots.push(root); + const target = join(root, "target.mjs"); + const entry = join(root, "entry.mjs"); + writeFileSync(target, `export const value = "resolved";\n`); + writeFileSync( + entry, + `import { value } from "@missing/runtime-helper"; +process.stdout.write(value); +`, + ); + + const registration = createModuleRedirectRegistration( + "@missing/runtime-helper", + pathToFileURL(target).href, + ); + const result = spawnSync( + process.execPath, + ["--import", registration, entry], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("resolved"); + }); +}); -- 2.51.2 From bf0bf4120b36f255075aabd91d1a37f077580215 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:24:50 +0000 Subject: [PATCH 23/34] Version Packages --- .changeset/tidy-lexicon-links.md | 5 ----- packages/contrail/CHANGELOG.md | 6 ++++++ packages/contrail/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/tidy-lexicon-links.md diff --git a/.changeset/tidy-lexicon-links.md b/.changeset/tidy-lexicon-links.md deleted file mode 100644 index 741ec48..0000000 --- a/.changeset/tidy-lexicon-links.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": patch ---- - -Run Atcute's Lexicon CLI with an explicit module redirect for its byte-array helper so `contrail dev` works with pnpm's strictly isolated global package links. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 1d91632..41d6824 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.21.2 + +### Patch Changes + +- 60e7b3a: Run Atcute's Lexicon CLI with an explicit module redirect for its byte-array helper so `contrail dev` works with pnpm's strictly isolated global package links. + ## 0.21.1 ### Patch Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index cb61300..097a403 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.21.1", + "version": "0.21.2", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From e050b0b28b0dc7274e456ba76098ced8081b9d67 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sat, 29 Aug 2026 18:50:14 +0200 Subject: [PATCH 24/34] Add verified Lexicon prefix initialization --- .changeset/lexicon-prefix-init.md | 5 + docs/01-getting-started.md | 45 +- packages/contrail/README.md | 27 +- packages/contrail/src/cli/commands/init.ts | 184 +++++- packages/contrail/src/cli/dev-lexicons.ts | 1 + packages/contrail/src/cli/init/analyze.ts | 435 +++++++++++++ packages/contrail/src/cli/init/references.ts | 170 +++++ packages/contrail/src/cli/init/registry.ts | 590 ++++++++++++++++++ packages/contrail/src/cli/init/render.ts | 227 +++++++ packages/contrail/src/cli/init/write.ts | 117 ++++ packages/contrail/src/lexicons/generate.ts | 15 +- packages/contrail/tests/init-analysis.test.ts | 217 +++++++ packages/contrail/tests/init-registry.test.ts | 218 +++++++ packages/contrail/tests/init.test.ts | 109 +++- .../contrail/tests/lexicon-generation.test.ts | 36 ++ 15 files changed, 2369 insertions(+), 27 deletions(-) create mode 100644 .changeset/lexicon-prefix-init.md create mode 100644 packages/contrail/src/cli/init/analyze.ts create mode 100644 packages/contrail/src/cli/init/references.ts create mode 100644 packages/contrail/src/cli/init/registry.ts create mode 100644 packages/contrail/src/cli/init/render.ts create mode 100644 packages/contrail/src/cli/init/write.ts create mode 100644 packages/contrail/tests/init-analysis.test.ts create mode 100644 packages/contrail/tests/init-registry.test.ts diff --git a/.changeset/lexicon-prefix-init.md b/.changeset/lexicon-prefix-init.md new file mode 100644 index 0000000..2e7014f --- /dev/null +++ b/.changeset/lexicon-prefix-init.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Add verified Lexicon-prefix initialization with CID-pinned dependency bundles, automatic safe query fields, and optional reference/relation prompts. diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index fbdb23b..c7e64ea 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -2,25 +2,37 @@ Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. -Create a project with the starter config: +Create a project from a verified Lexicon namespace: ```bash -pnpx @atmo-dev/contrail init my-appview +pnpx @atmo-dev/contrail init my-appview \ + --prefix community.lexicon.calendar. \ + --namespace com.example.calendar cd my-appview ``` -This creates `contrail.config.ts`: - -```ts -export default { - namespace: "com.example", - collections: { - event: { - collection: "community.lexicon.calendar.event", - queryable: { startsAt: { type: "range" } }, - }, - }, -}; +Contrail queries `https://lex.atmo.tools`, selects record Lexicons under the +prefix, pins their complete dependency graph under `lexicons/pinned/`, and +creates `contrail.config.ts`. String fields become equality filters and sort +options; datetime fields become range filters and chronological sort options. +Arrays, unions, numeric fields, and other unsupported shapes are reported rather +than configured incorrectly. + +When a strongRef or AT URI could point at another imported collection, an +interactive terminal asks whether to add forward hydration and an inverse +relation. The schema does not encode its target, so non-interactive runs leave +ambiguous references unconfigured. Use `--no-interactive` to request that +behavior explicitly. + +The import requires complete verification and catalog indexing. Use +`--allow-partial` only when you deliberately accept that some prefix documents +may be absent; required dependencies must always resolve. Every imported schema +is recorded with its CID and authority in `lexicons/pinned.lock`. + +For an offline example config with no registry request, omit `--prefix`: + +```bash +pnpx @atmo-dev/contrail init my-appview ``` Then run: @@ -32,7 +44,8 @@ pnpx @atmo-dev/contrail dev Contrail resolves the Lexicons, backfills existing records, follows new records, and serves a resumable SQLite AppView at `http://127.0.0.1:8787`. ```bash -curl 'http://127.0.0.1:8787/xrpc/com.example.event.listRecords?limit=10' +curl 'http://127.0.0.1:8787/xrpc/com.example.calendar.event.listRecords?limit=10' ``` -Replace the collection and fields with your own. Next: [add the typed client to your app](./02-client.md). +If you used the offline starter instead, its method is +`com.example.event.listRecords`. Next: [add the typed client to your app](./02-client.md). diff --git a/packages/contrail/README.md b/packages/contrail/README.md index e30e3dd..b58d992 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -166,15 +166,36 @@ Backups retain the database's change-log generation ID, positions, leases, boots ## Local development -Seed a new project with a basic `contrail.config.ts`, then start a complete local service: +Seed a project from proof-verified record Lexicons, then start a complete local service: ```bash -contrail init my-appview +contrail init my-appview \ + --prefix community.lexicon.calendar. \ + --namespace com.example.calendar cd my-appview contrail dev ``` -Run `contrail init` without a directory to write the config in the current directory. The command refuses to replace a config in any standard auto-detected location. +Prefix initialization uses `https://lex.atmo.tools` by default. It requires a +stable, fully verified and indexed catalog snapshot, pins each root and +transitive dependency by CID under `lexicons/pinned/`, writes provenance to +`lexicons/pinned.lock`, and generates equality/range fields which are safe on +all supported databases. Use `--lexicon-api` for a different registry, +`--timeout` to change the 60-second readiness budget, or `--allow-partial` to +explicitly accept incomplete prefix discovery. Missing required dependencies +always fail. + +In an interactive terminal, reference-shaped fields can be connected to an +imported target collection and optionally exposed as inverse relations. These +choices are never guessed from descriptions. `--no-interactive` leaves all +ambiguous references unconfigured. strongRef hydration resolves the current +indexed record by URI; it does not retrieve the historical version named by the +embedded CID. + +Run `contrail init my-appview` without `--prefix` to create the original offline +example config without a network request. Run either form without a directory to +write into the current directory. The command refuses to replace a config or +pinned import in any standard owned location. Without a Wrangler config this creates or resumes `.contrail/dev.sqlite`, resolves missing configured record/ref Lexicons from the AT Protocol network without overwriting project-owned schemas, runs PDS backfill, serves the complete public Contrail discovery/XRPC/Lexicon surface at `http://127.0.0.1:8787`, and runs bounded Jetstream ingestion every minute. It never creates or changes a deployment provider lock. Add `.contrail/` to the project ignore file. Existing Wrangler projects retain the prior D1 development behavior automatically. Useful SQLite options include: diff --git a/packages/contrail/src/cli/commands/init.ts b/packages/contrail/src/cli/commands/init.ts index 5d95e95..44182d4 100644 --- a/packages/contrail/src/cli/commands/init.ts +++ b/packages/contrail/src/cli/commands/init.ts @@ -2,6 +2,24 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join, relative, resolve } from "node:path"; import type { CAC } from "cac"; import { findConfigFile } from "../../cli-config.js"; +import { + analyzeLexiconGraph, + buildLexiconGraph, + type PrefixAnalysis, +} from "../init/analyze.js"; +import { + DEFAULT_LEXICON_API, + fetchExactLexicon, + importLexiconPrefix, + type RegistryClientOptions, +} from "../init/registry.js"; +import { + promptReferenceChoices, + type ReferenceChoice, + type SemanticChoicePrompt, +} from "../init/references.js"; +import { createInitArtifacts } from "../init/render.js"; +import { writeInitArtifacts } from "../init/write.js"; export const STARTER_CONFIG = `export default { namespace: "com.example", @@ -36,12 +54,168 @@ export async function seedConfig(directory: string): Promise { return path; } +export interface PrefixInitOptions extends RegistryClientOptions { + prefix: string; + namespace?: string; + interactive?: boolean; + prompt?: SemanticChoicePrompt; +} + +export interface PrefixInitResult { + configPath: string; + lockPath: string; + analysis: PrefixAnalysis; + choices: ReferenceChoice[]; +} + +/** Registry-backed initializer, kept injectable for deterministic tests. */ +export async function initializeFromPrefix( + directory: string, + options: PrefixInitOptions, +): Promise { + const root = resolve(directory); + const existing = findConfigFile(root); + if (existing) { + throw new Error(`A Contrail config already exists at ${existing}`); + } + + const registryOptions: RegistryClientOptions = { + api: options.api, + timeoutMs: options.timeoutMs, + allowPartial: options.allowPartial, + fetcher: options.fetcher, + now: options.now, + sleep: options.sleep, + }; + const prefix = await importLexiconPrefix(options.prefix, registryOptions); + const graph = await buildLexiconGraph(prefix, (nsid) => + fetchExactLexicon(nsid, registryOptions), + ); + const analysis = analyzeLexiconGraph(graph); + const choices = + options.interactive === false + ? [] + : await (options.prompt ?? promptReferenceChoices)(analysis); + const artifacts = createInitArtifacts( + options.namespace ?? "com.example", + analysis, + choices, + ); + const written = await writeInitArtifacts(root, artifacts); + return { + configPath: written.configPath, + lockPath: written.lockPath, + analysis, + choices, + }; +} + +interface InitCliOptions { + prefix?: string; + namespace: string; + lexiconApi: string; + timeout: number; + allowPartial?: boolean; + interactive?: boolean; +} + +function positiveSeconds(value: number): number { + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds <= 0) { + throw new TypeError("--timeout must be a positive number of seconds"); + } + return seconds; +} + +function printPrefixSummary(result: PrefixInitResult): void { + const fields = result.analysis.collections.reduce( + (total, collection) => total + Object.keys(collection.queryable).length, + 0, + ); + const skipped = result.analysis.collections.reduce( + (total, collection) => total + collection.skipped.length, + 0, + ); + const candidates = result.analysis.collections.reduce( + (total, collection) => total + collection.references.length, + 0, + ); + const dependencies = result.analysis.graph.documents.filter( + (document) => document.role === "dependency", + ).length; + const unresolvedLinks = Math.max(0, candidates - result.choices.length); + console.log( + `Imported ${result.analysis.graph.prefix.prefix} from ${result.analysis.graph.prefix.api}`, + ); + console.log(` ${result.analysis.collections.length} record collections`); + console.log(` ${dependencies} pinned dependency documents`); + console.log(` ${fields} queryable fields / expression indexes`); + if (skipped > 0) console.log(` ${skipped} unsupported fields skipped`); + if (result.choices.length > 0) { + const inverse = result.choices.filter((choice) => choice.inverse).length; + console.log( + ` ${result.choices.length} references, ${inverse} inverse relations configured`, + ); + } + if (unresolvedLinks > 0) { + console.log(` ${unresolvedLinks} reference candidates left unconfigured`); + } + const safeBsky = result.analysis.collections.filter((collection) => + collection.nsid.startsWith("app.bsky."), + ); + if (safeBsky.length > 0) { + console.warn( + ` warning: ${safeBsky.length} app.bsky.* collections retain discover:false safety defaults`, + ); + } + if (result.analysis.graph.prefix.partial) { + console.warn(" warning: the registry import is partial"); + } + console.log(`Created ${relative(process.cwd(), result.configPath) || "contrail.config.ts"}`); + console.log(`Created ${relative(process.cwd(), result.lockPath)}`); + console.log("\nRun: contrail dev"); +} + export function registerInit(cli: CAC): void { cli - .command("init [directory]", "Create a starter contrail.config.ts") - .action(async (directory: string | undefined) => { - const path = await seedConfig(directory ?? process.cwd()); - const displayPath = relative(process.cwd(), path) || "contrail.config.ts"; - console.log(`Created ${displayPath}`); + .command( + "init [directory]", + "Create a starter config or import verified Lexicons by prefix", + ) + .option("--prefix ", "Verified Lexicon prefix to import") + .option("--namespace ", "Generated AppView XRPC namespace", { + default: "com.example", + }) + .option("--lexicon-api ", "atmo Lexicons API origin", { + default: DEFAULT_LEXICON_API, + }) + .option("--timeout ", "Registry readiness timeout", { + default: 60, + }) + .option( + "--allow-partial", + "Accept incomplete prefix verification or catalog indexing", + ) + .option( + "--no-interactive", + "Skip ambiguous reference and inverse-relation questions", + ) + .action(async (directory: string | undefined, options: InitCliOptions) => { + const root = directory ?? process.cwd(); + if (!options.prefix) { + const path = await seedConfig(root); + const displayPath = relative(process.cwd(), path) || "contrail.config.ts"; + console.log(`Created ${displayPath}`); + return; + } + const result = await initializeFromPrefix(root, { + prefix: options.prefix, + namespace: options.namespace, + api: options.lexiconApi, + timeoutMs: positiveSeconds(options.timeout) * 1_000, + allowPartial: options.allowPartial, + interactive: options.interactive, + }); + printPrefixSummary(result); }); } diff --git a/packages/contrail/src/cli/dev-lexicons.ts b/packages/contrail/src/cli/dev-lexicons.ts index 2af0f27..11a2d04 100644 --- a/packages/contrail/src/cli/dev-lexicons.ts +++ b/packages/contrail/src/cli/dev-lexicons.ts @@ -27,6 +27,7 @@ export function prepareDevLexicons( const localSourceDirs = [ ...roots.flatMap((root) => [ join(root, "lexicons", "custom"), + join(root, "lexicons", "pinned"), join(root, "lexicons", "pulled"), ]), // A prior source connection places the complete local bundle here. Reuse diff --git a/packages/contrail/src/cli/init/analyze.ts b/packages/contrail/src/cli/init/analyze.ts new file mode 100644 index 0000000..164f3c7 --- /dev/null +++ b/packages/contrail/src/cli/init/analyze.ts @@ -0,0 +1,435 @@ +import { createHash } from "node:crypto"; +import { + findExternalReferences, + lexiconDoc, + parseLexiconRef, + refineLexiconDoc, + type LexiconDoc, +} from "@atcute/lexicon-doc"; +import { parse as parseValibot } from "valibot"; +import { validateFieldName, type QueryableField } from "../../core/types.js"; +import type { + PrefixImport, + VerifiedLexiconDocument, +} from "./registry.js"; + +const STRONG_REF = "com.atproto.repo.strongRef"; + +export interface PinnedLexicon extends VerifiedLexiconDocument { + role: "root" | "dependency"; +} + +export interface LexiconGraph { + prefix: PrefixImport; + roots: string[]; + documents: PinnedLexicon[]; + parsed: Map; +} + +export interface SkippedField { + path: string; + reason: + | "array" + | "blob" + | "boolean" + | "bytes" + | "cid-link" + | "constant" + | "cycle" + | "integer" + | "object" + | "token" + | "union" + | "unknown" + | "unsafe-path" + | "unsupported"; +} + +export interface ReferenceCandidate { + path: string; + kind: "at-uri" | "strongRef"; + cidPath?: string; +} + +export interface GroupFieldCandidate { + path: string; + values: string[]; +} + +export interface CollectionAnalysis { + nsid: string; + alias: string; + queryable: Record; + skipped: SkippedField[]; + references: ReferenceCandidate[]; + groupFields: GroupFieldCandidate[]; +} + +export interface PrefixAnalysis { + graph: LexiconGraph; + collections: CollectionAnalysis[]; +} + +function parseDocument(document: VerifiedLexiconDocument): LexiconDoc { + let parsed: LexiconDoc; + try { + parsed = parseValibot(lexiconDoc, document.value); + } catch { + throw new Error(`verified Lexicon ${document.id} is not a valid document`); + } + const issues = refineLexiconDoc(parsed, true); + if (issues.length > 0) { + const issue = issues[0]!; + throw new Error( + `verified Lexicon ${document.id} is invalid at ${issue.path.join(".") || "document"}: ${issue.message}`, + ); + } + if (parsed.id !== document.id) { + throw new Error(`verified Lexicon ID mismatch for ${document.id}`); + } + return parsed; +} + +function primaryType(document: VerifiedLexiconDocument): unknown { + return (document.value.defs as Record | undefined)?.main?.type; +} + +/** Select record roots and close their complete external Lexicon dependency graph. */ +export async function buildLexiconGraph( + prefix: PrefixImport, + resolveExact: (nsid: string) => Promise, +): Promise { + const prefixDocuments = new Map( + prefix.documents.map((document) => [document.id, document]), + ); + const roots = prefix.documents + .filter((document) => primaryType(document) === "record") + .map((document) => document.id) + .sort(); + if (roots.length === 0) { + const detail = + prefix.verification.candidates === 0 + ? "the prefix has no observed candidates" + : `the prefix returned ${prefix.documents.length} verified documents but no records`; + throw new Error(`No record Lexicons found for ${prefix.prefix}: ${detail}`); + } + + const documents = new Map(); + const parsed = new Map(); + const pending = [...roots]; + while (pending.length > 0) { + const nsid = pending.shift()!; + if (documents.has(nsid)) continue; + const document = prefixDocuments.get(nsid) ?? (await resolveExact(nsid)); + const value = parseDocument(document); + documents.set(nsid, document); + parsed.set(nsid, value); + + for (const reference of findExternalReferences(value)) { + const target = parseLexiconRef(reference, value.id).nsid; + if (!documents.has(target) && !pending.includes(target)) { + pending.push(target); + } + } + pending.sort(); + } + + // Verify both external and local targets after the closure is present. + for (const [nsid, document] of parsed) { + for (const reference of findExternalReferences(document)) { + const target = parseLexiconRef(reference, document.id); + const dependency = parsed.get(target.nsid); + if (!dependency) { + throw new Error(`missing dependency ${target.nsid} referenced by ${nsid}`); + } + if (!dependency.defs[target.defId]) { + throw new Error( + `missing definition ${target.nsid}#${target.defId} referenced by ${nsid}`, + ); + } + } + } + + const rootSet = new Set(roots); + return { + prefix, + roots, + parsed, + documents: [...documents.values()] + .sort((left, right) => left.id.localeCompare(right.id)) + .map((document) => ({ + ...document, + role: rootSet.has(document.id) ? "root" : "dependency", + })), + }; +} + +function normalizeSegment(segment: string): string { + const words = segment.split(/[^a-zA-Z0-9]+/).filter(Boolean); + if (words.length === 0) return "collection"; + const value = words + .map((word, index) => + index === 0 + ? word.charAt(0).toLowerCase() + word.slice(1) + : word.charAt(0).toUpperCase() + word.slice(1), + ) + .join(""); + return /^[a-zA-Z]/.test(value) ? value : `collection${value}`; +} + +function aliasAtDepth(nsid: string, depth: number): string { + const parts = nsid.split("."); + const selected = parts.slice(Math.max(0, parts.length - depth)); + return selected + .map((part, index) => { + const normalized = normalizeSegment(part); + return index === 0 + ? normalized + : normalized.charAt(0).toUpperCase() + normalized.slice(1); + }) + .join(""); +} + +/** Derive readable aliases while expanding every side of a collision. */ +export function deriveCollectionAliases(nsids: string[]): Map { + const sorted = [...new Set(nsids)].sort(); + const depths = new Map(sorted.map((nsid) => [nsid, 1])); + + for (let pass = 0; pass < 32; pass++) { + const groups = new Map(); + for (const nsid of sorted) { + const candidate = aliasAtDepth(nsid, depths.get(nsid)!); + const group = groups.get(candidate) ?? []; + group.push(nsid); + groups.set(candidate, group); + } + const collisions = [...groups.values()].filter((group) => group.length > 1); + if (collisions.length === 0) { + return new Map( + sorted.map((nsid) => [nsid, aliasAtDepth(nsid, depths.get(nsid)!)]), + ); + } + let expanded = false; + for (const group of collisions) { + for (const nsid of group) { + const max = nsid.split(".").length; + const depth = depths.get(nsid)!; + if (depth < max) { + depths.set(nsid, depth + 1); + expanded = true; + } + } + } + if (!expanded) break; + } + + const used = new Set(); + return new Map( + sorted.map((nsid) => { + const base = aliasAtDepth(nsid, nsid.split(".").length); + let alias = base; + if (used.has(alias)) { + alias = `${base}${createHash("sha256").update(nsid).digest("hex").slice(0, 8)}`; + } + used.add(alias); + return [nsid, alias]; + }), + ); +} + +function safePath(path: string): boolean { + try { + validateFieldName(path); + return true; + } catch { + return false; + } +} + +interface WalkState { + documents: Map; + queryable: Map; + skipped: Map; + references: Map; + groupFields: Map; +} + +function skip(state: WalkState, path: string, reason: SkippedField["reason"]) { + if (!state.skipped.has(path)) state.skipped.set(path, { path, reason }); +} + +function addReference( + state: WalkState, + candidate: ReferenceCandidate, +): void { + const existing = state.references.get(candidate.path); + if (!existing || candidate.kind === "strongRef") { + state.references.set(candidate.path, candidate); + } +} + +function walkDefinition( + state: WalkState, + definition: any, + path: string, + contextNsid: string, + seen: ReadonlySet, +): void { + if (!definition || typeof definition !== "object") { + skip(state, path, "unsupported"); + return; + } + + switch (definition.type) { + case "string": { + if (!safePath(path)) { + skip(state, path, "unsafe-path"); + return; + } + if (definition.const !== undefined) { + skip(state, path, "constant"); + return; + } + state.queryable.set( + path, + definition.format === "datetime" ? { type: "range" } : {}, + ); + if (definition.format === "at-uri") { + addReference(state, { path, kind: "at-uri" }); + } + const values: unknown[] = Array.isArray(definition.enum) + ? definition.enum + : Array.isArray(definition.knownValues) + ? definition.knownValues + : []; + const strings = values.filter( + (value: unknown): value is string => typeof value === "string", + ); + if (strings.length > 0 && strings.length <= 20) { + state.groupFields.set(path, { + path, + values: [...new Set(strings)].sort(), + }); + } + return; + } + case "ref": { + const target = parseLexiconRef( + definition.ref, + contextNsid as LexiconDoc["id"], + ); + const key = `${target.nsid}#${target.defId}`; + if (seen.has(key)) { + skip(state, path, "cycle"); + return; + } + const document = state.documents.get(target.nsid); + const resolved = document?.defs[target.defId]; + if (!resolved) { + skip(state, path, "unsupported"); + return; + } + if (target.nsid === STRONG_REF && target.defId === "main") { + addReference(state, { + path: `${path}.uri`, + kind: "strongRef", + cidPath: `${path}.cid`, + }); + } + walkDefinition( + state, + resolved, + path, + target.nsid, + new Set([...seen, key]), + ); + return; + } + case "object": + if (!definition.properties || Object.keys(definition.properties).length === 0) { + skip(state, path, "object"); + return; + } + for (const [name, property] of Object.entries(definition.properties)) { + walkDefinition( + state, + property, + path ? `${path}.${name}` : name, + contextNsid, + seen, + ); + } + return; + case "record": + walkDefinition(state, definition.record, path, contextNsid, seen); + return; + case "array": + skip(state, path, "array"); + return; + case "union": + skip(state, path, "union"); + return; + case "integer": + skip(state, path, "integer"); + return; + case "boolean": + skip(state, path, "boolean"); + return; + case "blob": + skip(state, path, "blob"); + return; + case "bytes": + skip(state, path, "bytes"); + return; + case "cid-link": + skip(state, path, "cid-link"); + return; + case "unknown": + skip(state, path, "unknown"); + return; + case "token": + skip(state, path, "token"); + return; + default: + skip(state, path, "unsupported"); + } +} + +export function analyzeLexiconGraph(graph: LexiconGraph): PrefixAnalysis { + const aliases = deriveCollectionAliases(graph.roots); + const collections = graph.roots.map((nsid) => { + const document = graph.parsed.get(nsid)!; + const main = document.defs.main as any; + const state: WalkState = { + documents: graph.parsed, + queryable: new Map(), + skipped: new Map(), + references: new Map(), + groupFields: new Map(), + }; + for (const [name, property] of Object.entries( + main.record?.properties ?? {}, + )) { + walkDefinition(state, property, name, nsid, new Set()); + } + return { + nsid, + alias: aliases.get(nsid)!, + queryable: Object.fromEntries( + [...state.queryable.entries()].sort(([left], [right]) => + left.localeCompare(right), + ), + ), + skipped: [...state.skipped.values()].sort((left, right) => + left.path.localeCompare(right.path), + ), + references: [...state.references.values()].sort((left, right) => + left.path.localeCompare(right.path), + ), + groupFields: [...state.groupFields.values()].sort((left, right) => + left.path.localeCompare(right.path), + ), + } satisfies CollectionAnalysis; + }); + return { graph, collections }; +} diff --git a/packages/contrail/src/cli/init/references.ts b/packages/contrail/src/cli/init/references.ts new file mode 100644 index 0000000..a1fad95 --- /dev/null +++ b/packages/contrail/src/cli/init/references.ts @@ -0,0 +1,170 @@ +import { createInterface } from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; +import type { PrefixAnalysis, ReferenceCandidate } from "./analyze.js"; + +export interface InverseRelationChoice { + name: string; + groupBy?: string; +} + +export interface ReferenceChoice { + source: string; + path: string; + target: string; + name: string; + inverse?: InverseRelationChoice; +} + +export type SemanticChoicePrompt = ( + analysis: PrefixAnalysis, +) => Promise; + +function pluralize(value: string): string { + if (value.endsWith("s")) return `${value}es`; + if (value.endsWith("y") && !/[aeiou]y$/i.test(value)) { + return `${value.slice(0, -1)}ies`; + } + return `${value}s`; +} + +function pathName(path: string): string { + const parts = path.split("."); + return parts[parts.length - 1] ?? "reference"; +} + +function uniqueName(base: string, used: Set): string { + let value = base; + let suffix = 2; + while (used.has(value)) value = `${base}${suffix++}`; + used.add(value); + return value; +} + +async function chooseIndex( + question: string, + labels: string[], + defaultIndex: number, + ask: (question: string) => Promise, +): Promise { + output.write(`${question}\n`); + labels.forEach((label, index) => output.write(` ${index}) ${label}\n`)); + const answer = (await ask(`Selection [${defaultIndex}]: `)).trim(); + if (answer === "") return defaultIndex; + const selected = Number(answer); + return Number.isInteger(selected) && selected >= 0 && selected < labels.length + ? selected + : defaultIndex; +} + +async function askYesNo( + question: string, + defaultYes: boolean, + ask: (question: string) => Promise, +): Promise { + const answer = ( + await ask(`${question} ${defaultYes ? "[Y/n]" : "[y/N]"} `) + ) + .trim() + .toLowerCase(); + if (answer === "") return defaultYes; + return answer === "y" || answer === "yes"; +} + +/** Ask only for semantics the Lexicon does not encode. */ +export async function promptReferenceChoices( + analysis: PrefixAnalysis, +): Promise { + if (!input.isTTY || !output.isTTY) return []; + const rl = createInterface({ input, output }); + const ask = (question: string) => rl.question(question); + const choices: ReferenceChoice[] = []; + const usedReferenceNames = new Map>(); + const usedRelationNames = new Map>(); + + try { + for (const source of analysis.collections) { + for (const candidate of source.references) { + const targetOptions = [ + "skip", + ...analysis.collections.map( + (collection) => `${collection.alias} (${collection.nsid})`, + ), + ]; + const selected = await chooseIndex( + `\n${source.alias}.${candidate.path} is ${candidateLabel(candidate)}. Target collection?`, + targetOptions, + 0, + ask, + ); + if (selected === 0) continue; + const target = analysis.collections[selected - 1]!; + const used = usedReferenceNames.get(source.alias) ?? new Set(); + usedReferenceNames.set(source.alias, used); + const defaultName = uniqueName( + target.alias === source.alias ? pathName(candidate.path) : target.alias, + used, + ); + const enteredName = ( + await ask(`Forward reference name [${defaultName}]: `) + ).trim(); + const name = enteredName || defaultName; + used.add(name); + + let inverse: InverseRelationChoice | undefined; + if ( + await askYesNo( + `Add inverse relation on ${target.alias}?`, + true, + ask, + ) + ) { + const relationUsed = + usedRelationNames.get(target.alias) ?? new Set(); + usedRelationNames.set(target.alias, relationUsed); + const relationDefault = uniqueName(pluralize(source.alias), relationUsed); + const entered = ( + await ask(`Inverse relation name [${relationDefault}]: `) + ).trim(); + const relationName = entered || relationDefault; + relationUsed.add(relationName); + + const groupOptions = [ + "none", + ...source.groupFields.map( + (field) => `${field.path} (${field.values.join(", ")})`, + ), + ]; + const group = await chooseIndex( + `Group ${target.alias}.${relationName} counts by a field?`, + groupOptions, + 0, + ask, + ); + inverse = { + name: relationName, + ...(group > 0 + ? { groupBy: source.groupFields[group - 1]!.path } + : {}), + }; + } + + choices.push({ + source: source.alias, + path: candidate.path, + target: target.alias, + name, + ...(inverse ? { inverse } : {}), + }); + } + } + } finally { + rl.close(); + } + return choices; +} + +function candidateLabel(candidate: ReferenceCandidate): string { + return candidate.kind === "strongRef" + ? `a strongRef at ${candidate.path} (hydration resolves the current record by URI, not the pinned CID)` + : `an AT URI at ${candidate.path}`; +} diff --git a/packages/contrail/src/cli/init/registry.ts b/packages/contrail/src/cli/init/registry.ts new file mode 100644 index 0000000..778201f --- /dev/null +++ b/packages/contrail/src/cli/init/registry.ts @@ -0,0 +1,590 @@ +import { + isDid, + isNsid, + isResourceUri, +} from "@atcute/lexicons/syntax"; + +export const DEFAULT_LEXICON_API = "https://lex.atmo.tools"; +const DEFAULT_LIMIT = 200; +const DEFAULT_TIMEOUT_MS = 60_000; +const MAX_RESPONSE_BYTES = 25 * 1024 * 1024; +const MAX_IMPORT_BYTES = 50 * 1024 * 1024; +const MAX_IMPORT_DOCUMENTS = 2_000; +const MAX_SNAPSHOT_RESTARTS = 5; + +export interface VerificationSummary { + candidates: number; + verified: number; + pending: number; + stale: number; + temporaryFailure: number; + unresolved: number; + invalid: number; + settled: boolean; + allVerified: boolean; +} + +export interface IndexingSummary { + available: boolean; + relayDiscoveryComplete: boolean; + backfillsPending: number; + backfillsExhausted: number; + settled: boolean; + complete: boolean; +} + +export interface VerifiedLexiconDocument { + id: string; + authorityDid: string; + uri: string; + cid: string; + value: Record; + verifiedAt: string; + verifiedUntil: string; +} + +export interface PrefixImport { + api: string; + prefix: string; + snapshot: string; + partial: boolean; + verification: VerificationSummary; + indexing: IndexingSummary; + documents: VerifiedLexiconDocument[]; +} + +export interface RegistryClientOptions { + api?: string; + timeoutMs?: number; + allowPartial?: boolean; + fetcher?: typeof fetch; + now?: () => number; + sleep?: (milliseconds: number) => Promise; +} + +interface PrefixPage { + prefix: string; + verified: true; + snapshot: string; + lexicons: VerifiedLexiconDocument[]; + cursor?: string; + verification: VerificationSummary; + indexing: IndexingSummary; +} + +interface ErrorBody { + error?: string; + id?: string; + status?: string; + retryAt?: string; +} + +export class LexiconRegistryError extends Error { + constructor( + message: string, + readonly code: string, + readonly status?: number, + ) { + super(message); + this.name = "LexiconRegistryError"; + } +} + +function numberField(value: unknown, name: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new LexiconRegistryError( + `Lexicon API returned an invalid ${name}`, + "InvalidResponse", + ); + } + return value; +} + +function booleanField(value: unknown, name: string): boolean { + if (typeof value !== "boolean") { + throw new LexiconRegistryError( + `Lexicon API returned an invalid ${name}`, + "InvalidResponse", + ); + } + return value; +} + +function objectField(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new LexiconRegistryError( + `Lexicon API returned an invalid ${name}`, + "InvalidResponse", + ); + } + return value as Record; +} + +function stringField(value: unknown, name: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new LexiconRegistryError( + `Lexicon API returned an invalid ${name}`, + "InvalidResponse", + ); + } + return value; +} + +function verificationSummary(value: unknown): VerificationSummary { + const input = objectField(value, "verification summary"); + return { + candidates: numberField(input.candidates, "verification.candidates"), + verified: numberField(input.verified, "verification.verified"), + pending: numberField(input.pending, "verification.pending"), + stale: numberField(input.stale, "verification.stale"), + temporaryFailure: numberField( + input.temporaryFailure, + "verification.temporaryFailure", + ), + unresolved: numberField(input.unresolved, "verification.unresolved"), + invalid: numberField(input.invalid, "verification.invalid"), + settled: booleanField(input.settled, "verification.settled"), + allVerified: booleanField( + input.allVerified, + "verification.allVerified", + ), + }; +} + +function indexingSummary(value: unknown): IndexingSummary { + const input = objectField(value, "indexing summary"); + return { + available: booleanField(input.available, "indexing.available"), + relayDiscoveryComplete: booleanField( + input.relayDiscoveryComplete, + "indexing.relayDiscoveryComplete", + ), + backfillsPending: numberField( + input.backfillsPending, + "indexing.backfillsPending", + ), + backfillsExhausted: numberField( + input.backfillsExhausted, + "indexing.backfillsExhausted", + ), + settled: booleanField(input.settled, "indexing.settled"), + complete: booleanField(input.complete, "indexing.complete"), + }; +} + +export function parseVerifiedDocument( + value: unknown, + now = Date.now(), +): VerifiedLexiconDocument { + const input = objectField(value, "verified Lexicon"); + const id = stringField(input.id, "Lexicon id"); + if (!isNsid(id)) { + throw new LexiconRegistryError( + `Lexicon API returned an invalid NSID: ${id}`, + "InvalidResponse", + ); + } + const document = objectField(input.value, `Lexicon ${id} value`); + if (document.id !== id) { + throw new LexiconRegistryError( + `Lexicon API document ID mismatch for ${id}`, + "InvalidResponse", + ); + } + const verifiedAt = stringField(input.verifiedAt, `${id}.verifiedAt`); + const verifiedUntil = stringField(input.verifiedUntil, `${id}.verifiedUntil`); + if (!Number.isFinite(Date.parse(verifiedAt))) { + throw new LexiconRegistryError( + `Lexicon API returned an invalid verifiedAt for ${id}`, + "InvalidResponse", + ); + } + const expiry = Date.parse(verifiedUntil); + if (!Number.isFinite(expiry) || expiry <= now) { + throw new LexiconRegistryError( + `Lexicon API returned an expired verification for ${id}`, + "ExpiredVerification", + ); + } + const authorityDid = stringField(input.authorityDid, `${id}.authorityDid`); + const uri = stringField(input.uri, `${id}.uri`); + if (!isDid(authorityDid) || !isResourceUri(uri)) { + throw new LexiconRegistryError( + `Lexicon API returned invalid provenance for ${id}`, + "InvalidResponse", + ); + } + return { + id, + authorityDid, + uri, + cid: stringField(input.cid, `${id}.cid`), + value: document, + verifiedAt, + verifiedUntil, + }; +} + +function parsePrefixPage( + value: unknown, + expectedPrefix: string, + now: number, +): PrefixPage { + const input = objectField(value, "prefix response"); + if (input.verified !== true || input.prefix !== expectedPrefix) { + throw new LexiconRegistryError( + "Lexicon API returned the wrong prefix or an unverified response", + "InvalidResponse", + ); + } + if (!Array.isArray(input.lexicons)) { + throw new LexiconRegistryError( + "Lexicon API response must contain a lexicons array", + "InvalidResponse", + ); + } + return { + prefix: expectedPrefix, + verified: true, + snapshot: stringField(input.snapshot, "snapshot"), + lexicons: input.lexicons.map((document) => + parseVerifiedDocument(document, now), + ), + ...(input.cursor === undefined + ? {} + : { cursor: stringField(input.cursor, "cursor") }), + verification: verificationSummary(input.verification), + indexing: indexingSummary(input.indexing), + }; +} + +function normalizeApi(input: string): URL { + let url: URL; + try { + url = new URL(input); + } catch { + throw new LexiconRegistryError( + `Invalid Lexicon API URL: ${input}`, + "InvalidEndpoint", + ); + } + if ( + url.username || + url.password || + url.search || + url.hash || + (url.pathname !== "/" && url.pathname !== "") + ) { + throw new LexiconRegistryError( + "Lexicon API URL must be an origin without credentials, path, query, or hash", + "InvalidEndpoint", + ); + } + const loopback = + url.hostname === "localhost" || + url.hostname === "127.0.0.1" || + url.hostname === "[::1]"; + if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) { + throw new LexiconRegistryError( + "Lexicon API must use HTTPS (HTTP is allowed only on loopback)", + "InvalidEndpoint", + ); + } + url.pathname = url.pathname.replace(/\/$/, ""); + return url; +} + +async function defaultSleep(milliseconds: number): Promise { + await new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +interface RequestContext { + fetcher: typeof fetch; + api: URL; + deadline: number; + now: () => number; + sleep: (milliseconds: number) => Promise; + bytes: number; +} + +async function request( + context: RequestContext, + path: string, +): Promise<{ response: Response; value: unknown }> { + const current = context.now(); + if (current >= context.deadline) { + throw new LexiconRegistryError( + "Timed out waiting for the Lexicon API", + "Timeout", + ); + } + const url = new URL(path, `${context.api.toString()}/`); + const response = await context.fetcher(url, { + redirect: "manual", + signal: AbortSignal.timeout(Math.max(1, context.deadline - current)), + }); + if (response.url && new URL(response.url).origin !== url.origin) { + throw new LexiconRegistryError( + "Lexicon API redirected to another origin", + "UnexpectedRedirect", + response.status, + ); + } + if (response.status >= 300 && response.status < 400) { + throw new LexiconRegistryError( + `Lexicon API refused redirect (${response.status})`, + "UnexpectedRedirect", + response.status, + ); + } + const declared = Number(response.headers.get("content-length") ?? 0); + if (declared > MAX_RESPONSE_BYTES) { + throw new LexiconRegistryError( + "Lexicon API response is too large", + "ResponseTooLarge", + response.status, + ); + } + const text = await response.text(); + const bytes = new TextEncoder().encode(text).byteLength; + if (bytes > MAX_RESPONSE_BYTES) { + throw new LexiconRegistryError( + "Lexicon API response is too large", + "ResponseTooLarge", + response.status, + ); + } + context.bytes += bytes; + if (context.bytes > MAX_IMPORT_BYTES) { + throw new LexiconRegistryError( + "Lexicon import exceeds the total response size limit", + "ResponseTooLarge", + ); + } + if (text.length === 0) return { response, value: null }; + try { + return { response, value: JSON.parse(text) as unknown }; + } catch { + throw new LexiconRegistryError( + "Lexicon API returned invalid JSON", + "InvalidResponse", + response.status, + ); + } +} + +function createContext(options: RegistryClientOptions): RequestContext { + const now = options.now ?? Date.now; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new TypeError("Lexicon API timeout must be positive"); + } + return { + fetcher: options.fetcher ?? fetch, + api: normalizeApi(options.api ?? DEFAULT_LEXICON_API), + deadline: now() + timeoutMs, + now, + sleep: options.sleep ?? defaultSleep, + bytes: 0, + }; +} + +function readinessError(page: PrefixPage): LexiconRegistryError | null { + if (page.verification.settled && !page.verification.allVerified) { + return new LexiconRegistryError( + `Lexicon prefix verification failed: ${page.verification.unresolved} unresolved, ${page.verification.invalid} invalid`, + "IncompleteVerification", + ); + } + if (page.indexing.settled && !page.indexing.complete) { + return new LexiconRegistryError( + `Lexicon catalog indexing is incomplete: ${page.indexing.backfillsExhausted} backfills exhausted`, + "IncompleteIndexing", + ); + } + return null; +} + +async function waitBeforeRetry( + context: RequestContext, + milliseconds = 500, +): Promise { + const remaining = context.deadline - context.now(); + if (remaining <= 0) { + throw new LexiconRegistryError( + "Timed out waiting for the Lexicon API", + "Timeout", + ); + } + await context.sleep(Math.min(milliseconds, remaining)); +} + +async function getPrefixPage( + context: RequestContext, + prefix: string, + cursor?: string, +): Promise { + const params = new URLSearchParams({ + prefix, + limit: String(DEFAULT_LIMIT), + refresh: "true", + }); + if (cursor) params.set("cursor", cursor); + const { response, value } = await request( + context, + `/lexicons?${params.toString()}`, + ); + if (response.status === 409) { + const body = value as ErrorBody | null; + if (body?.error === "SnapshotChanged") return "snapshot-changed"; + } + if (!response.ok) { + const body = value as ErrorBody | null; + throw new LexiconRegistryError( + `Lexicon prefix request failed: ${body?.error ?? response.status}`, + body?.error ?? "RequestFailed", + response.status, + ); + } + return parsePrefixPage(value, prefix, context.now()); +} + +export async function importLexiconPrefix( + prefix: string, + options: RegistryClientOptions = {}, +): Promise { + if ( + !prefix || + prefix.length > 317 || + !/^[A-Za-z0-9.-]+$/.test(prefix) || + prefix.startsWith(".") || + prefix.includes("..") + ) { + throw new TypeError("prefix must be a non-empty NSID prefix"); + } + const context = createContext(options); + const allowPartial = options.allowPartial === true; + + for (let restart = 0; restart < MAX_SNAPSHOT_RESTARTS; restart++) { + let first: PrefixPage; + while (true) { + const result = await getPrefixPage(context, prefix); + if (result === "snapshot-changed") { + await waitBeforeRetry(context, 100); + continue; + } + first = result; + if (allowPartial) break; + const failure = readinessError(first); + if (failure) throw failure; + if (first.verification.allVerified && first.indexing.complete) break; + await waitBeforeRetry(context); + } + + const documents = new Map(); + const addPage = (page: PrefixPage) => { + if (page.snapshot !== first.snapshot) { + throw new LexiconRegistryError( + "Lexicon API changed snapshots without rejecting the cursor", + "SnapshotChanged", + ); + } + for (const document of page.lexicons) { + const existing = documents.get(document.id); + if (existing && existing.cid !== document.cid) { + throw new LexiconRegistryError( + `Lexicon ${document.id} changed inside one snapshot`, + "SnapshotChanged", + ); + } + documents.set(document.id, document); + } + if (documents.size > MAX_IMPORT_DOCUMENTS) { + throw new LexiconRegistryError( + `Lexicon prefix exceeds ${MAX_IMPORT_DOCUMENTS} documents`, + "TooManyDocuments", + ); + } + }; + addPage(first); + + let cursor = first.cursor; + let changed = false; + while (cursor) { + const result = await getPrefixPage(context, prefix, cursor); + if (result === "snapshot-changed") { + changed = true; + break; + } + addPage(result); + cursor = result.cursor; + } + if (changed) continue; + + return { + api: context.api.origin, + prefix, + snapshot: first.snapshot, + partial: + !first.verification.allVerified || !first.indexing.complete, + verification: first.verification, + indexing: first.indexing, + documents: [...documents.values()].sort((left, right) => + left.id.localeCompare(right.id), + ), + }; + } + + throw new LexiconRegistryError( + "Lexicon prefix changed too often while it was being imported", + "SnapshotChanged", + ); +} + +function retryDelay( + response: Response, + body: ErrorBody, + now: number, +): number { + const retryAfter = Number(response.headers.get("retry-after")); + if (Number.isFinite(retryAfter) && retryAfter > 0) { + return Math.min(retryAfter * 1_000, 5_000); + } + const retryAt = body.retryAt ? Date.parse(body.retryAt) : Number.NaN; + if (Number.isFinite(retryAt) && retryAt > now) { + return Math.min(retryAt - now, 5_000); + } + return 500; +} + +export async function fetchExactLexicon( + nsid: string, + options: RegistryClientOptions = {}, +): Promise { + if (!isNsid(nsid)) throw new TypeError(`invalid Lexicon NSID: ${nsid}`); + const context = createContext(options); + + while (true) { + const { response, value } = await request( + context, + `/lexicons/${encodeURIComponent(nsid)}?refresh=true`, + ); + if (response.status === 200) { + return parseVerifiedDocument(value, context.now()); + } + const body = (value ?? {}) as ErrorBody; + if (response.status === 202 || response.status === 503) { + await waitBeforeRetry( + context, + retryDelay(response, body, context.now()), + ); + continue; + } + const code = body.error ?? "RequestFailed"; + throw new LexiconRegistryError( + `Could not resolve Lexicon ${nsid}: ${code}`, + code, + response.status, + ); + } +} diff --git a/packages/contrail/src/cli/init/render.ts b/packages/contrail/src/cli/init/render.ts new file mode 100644 index 0000000..0633116 --- /dev/null +++ b/packages/contrail/src/cli/init/render.ts @@ -0,0 +1,227 @@ +import { isNsid } from "@atcute/lexicons/syntax"; +import type { + CollectionConfig, + ContrailConfig, + QueryableField, +} from "../../core/types.js"; +import { validateConfig } from "../../core/types.js"; +import type { PrefixAnalysis } from "./analyze.js"; +import type { ReferenceChoice } from "./references.js"; + +export interface LexiconImportLock { + format: "contrail.lexicon-import"; + version: 1; + api: string; + prefix: string; + snapshot: string; + partial: boolean; + verification: PrefixAnalysis["graph"]["prefix"]["verification"]; + indexing: PrefixAnalysis["graph"]["prefix"]["indexing"]; + roots: string[]; + documents: Array<{ + id: string; + cid: string; + authorityDid: string; + uri: string; + role: "root" | "dependency"; + verifiedAt: string; + verifiedUntil: string; + }>; +} + +export interface InitArtifacts { + config: ContrailConfig; + configSource: string; + lock: LexiconImportLock; + analysis: PrefixAnalysis; + choices: ReferenceChoice[]; +} + +function groupName(value: string, used: Set): string { + const fragment = value.includes("#") ? value.slice(value.lastIndexOf("#") + 1) : value; + const words = fragment.split(/[^a-zA-Z0-9]+/).filter(Boolean); + let base = words + .map((word, index) => + index === 0 + ? word.charAt(0).toLowerCase() + word.slice(1) + : word.charAt(0).toUpperCase() + word.slice(1), + ) + .join(""); + if (!base) base = "value"; + if (!/^[a-zA-Z]/.test(base)) base = `value${base}`; + let name = base; + let suffix = 2; + while (used.has(name)) name = `${base}${suffix++}`; + used.add(name); + return name; +} + +function formatKey(key: string): string { + return /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(key) + ? key + : JSON.stringify(key); +} + +function renderValue(value: unknown, indentation = 0): string { + if (value === null || typeof value !== "object") { + return JSON.stringify(value); + } + if (Array.isArray(value)) { + if (value.length === 0) return "[]"; + const indent = " ".repeat(indentation); + const child = " ".repeat(indentation + 2); + return `[\n${value + .map((item) => `${child}${renderValue(item, indentation + 2)},`) + .join("\n")}\n${indent}]`; + } + const entries = Object.entries(value as Record).filter( + ([, item]) => item !== undefined, + ); + if (entries.length === 0) return "{}"; + const indent = " ".repeat(indentation); + const child = " ".repeat(indentation + 2); + return `{\n${entries + .map( + ([key, item]) => + `${child}${formatKey(key)}: ${renderValue(item, indentation + 2)},`, + ) + .join("\n")}\n${indent}}`; +} + +function cloneQueryable( + queryable: Record, +): Record { + return Object.fromEntries( + Object.entries(queryable).map(([field, config]) => [field, { ...config }]), + ); +} + +export function createInitArtifacts( + namespace: string, + analysis: PrefixAnalysis, + choices: ReferenceChoice[] = [], +): InitArtifacts { + const byAlias = new Map( + analysis.collections.map((collection) => [collection.alias, collection]), + ); + const collections: Record = {}; + for (const collection of analysis.collections) { + collections[collection.alias] = { + collection: collection.nsid, + validate: true, + queryable: cloneQueryable(collection.queryable), + }; + } + + for (const choice of choices) { + const sourceAnalysis = byAlias.get(choice.source); + const targetAnalysis = byAlias.get(choice.target); + const source = collections[choice.source]; + const target = collections[choice.target]; + if (!sourceAnalysis || !targetAnalysis || !source || !target) { + throw new Error(`Reference choice names an unknown collection`); + } + const candidate = sourceAnalysis.references.find( + (reference) => reference.path === choice.path, + ); + if (!candidate) { + throw new Error( + `Reference choice names unknown path ${choice.source}.${choice.path}`, + ); + } + if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(choice.name)) { + throw new Error(`Invalid reference name ${choice.source}.${choice.name}`); + } + source.references ??= {}; + if (source.references[choice.name]) { + throw new Error( + `Duplicate reference name ${choice.source}.${choice.name}`, + ); + } + source.references[choice.name] = { + collection: choice.target, + field: choice.path, + }; + + if (choice.inverse) { + if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(choice.inverse.name)) { + throw new Error( + `Invalid relation name ${choice.target}.${choice.inverse.name}`, + ); + } + target.relations ??= {}; + if (target.relations[choice.inverse.name]) { + throw new Error( + `Duplicate relation name ${choice.target}.${choice.inverse.name}`, + ); + } + const group = choice.inverse.groupBy + ? sourceAnalysis.groupFields.find( + (field) => field.path === choice.inverse!.groupBy, + ) + : undefined; + if (choice.inverse.groupBy && !group) { + throw new Error( + `Unknown relation group field ${choice.source}.${choice.inverse.groupBy}`, + ); + } + const used = new Set(); + target.relations[choice.inverse.name] = { + collection: choice.source, + field: choice.path, + ...(group + ? { + groupBy: group.path, + groups: Object.fromEntries( + group.values.map((value) => [groupName(value, used), value]), + ), + } + : {}), + }; + } + } + + const config: ContrailConfig = { + namespace, + profiles: [], + collections, + }; + validateConfig(config); + for (const alias of Object.keys(collections)) { + // A namespace is an XRPC prefix, not itself a complete NSID. + const method = `${namespace}.${alias}.listRecords`; + if (!isNsid(method)) { + throw new Error(`Invalid generated XRPC namespace: ${namespace}`); + } + } + + const prefix = analysis.graph.prefix; + const lock: LexiconImportLock = { + format: "contrail.lexicon-import", + version: 1, + api: prefix.api, + prefix: prefix.prefix, + snapshot: prefix.snapshot, + partial: prefix.partial, + verification: prefix.verification, + indexing: prefix.indexing, + roots: [...analysis.graph.roots], + documents: analysis.graph.documents.map((document) => ({ + id: document.id, + cid: document.cid, + authorityDid: document.authorityDid, + uri: document.uri, + role: document.role, + verifiedAt: document.verifiedAt, + verifiedUntil: document.verifiedUntil, + })), + }; + + return { + config, + configSource: `export default ${renderValue(config)};\n`, + lock, + analysis, + choices, + }; +} diff --git a/packages/contrail/src/cli/init/write.ts b/packages/contrail/src/cli/init/write.ts new file mode 100644 index 0000000..8baa43f --- /dev/null +++ b/packages/contrail/src/cli/init/write.ts @@ -0,0 +1,117 @@ +import { constants } from "node:fs"; +import { + access, + mkdir, + mkdtemp, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import { findConfigFile, loadConfig } from "../../cli-config.js"; +import { validateConfig, type ContrailConfig } from "../../core/types.js"; +import { generateLexicons } from "../../lexicons/generate.js"; +import type { InitArtifacts } from "./render.js"; + +export interface WrittenInitArtifacts { + configPath: string; + pinnedRoot: string; + lockPath: string; +} + +async function exists(path: string): Promise { + try { + await access(path, constants.F_OK); + return true; + } catch { + return false; + } +} + +function lexiconPath(root: string, nsid: string): string { + return join(root, ...nsid.split(".")) + ".json"; +} + +/** Stage, validate, and commit all prefix-init owned files without overwrites. */ +export async function writeInitArtifacts( + directory: string, + artifacts: InitArtifacts, +): Promise { + const root = resolve(directory); + const existingConfig = findConfigFile(root); + if (existingConfig) { + throw new Error(`A Contrail config already exists at ${existingConfig}`); + } + const configPath = join(root, "contrail.config.ts"); + const pinnedRoot = join(root, "lexicons", "pinned"); + const lockPath = join(root, "lexicons", "pinned.lock"); + for (const path of [configPath, pinnedRoot, lockPath]) { + if (await exists(path)) { + throw new Error(`Contrail init refuses to overwrite ${path}`); + } + } + + const parent = dirname(root); + await mkdir(parent, { recursive: true }); + const stage = await mkdtemp(join(parent, `.${basename(root)}-contrail-init-`)); + const stageConfig = join(stage, "contrail.config.ts"); + const stagePinned = join(stage, "lexicons", "pinned"); + const stageLock = join(stage, "lexicons", "pinned.lock"); + try { + await mkdir(stagePinned, { recursive: true }); + await writeFile(stageConfig, artifacts.configSource); + for (const document of artifacts.analysis.graph.documents) { + const path = lexiconPath(stagePinned, document.id); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(document.value, null, 2)}\n`); + } + await writeFile(stageLock, `${JSON.stringify(artifacts.lock, null, 2)}\n`); + + const loaded = await loadConfig(stageConfig); + validateConfig(loaded); + const validationOutput = join(stage, ".validation-generated"); + generateLexicons({ + config: loaded, + rootDir: stage, + outputDir: validationOutput, + writeAtcuteConfig: false, + quiet: true, + }); + await rm(validationOutput, { recursive: true, force: true }); + + // Recheck after network/staging work, then commit the config last. A caller + // never observes a config whose pinned inputs were not installed. + const racedConfig = findConfigFile(root); + if (racedConfig) { + throw new Error(`A Contrail config already exists at ${racedConfig}`); + } + for (const path of [pinnedRoot, lockPath]) { + if (await exists(path)) { + throw new Error(`Contrail init refuses to overwrite ${path}`); + } + } + + await mkdir(join(root, "lexicons"), { recursive: true }); + let movedPinned = false; + let wroteLock = false; + let wroteConfig = false; + try { + await rename(stagePinned, pinnedRoot); + movedPinned = true; + await writeFile(lockPath, await readFile(stageLock), { flag: "wx" }); + wroteLock = true; + await writeFile(configPath, artifacts.configSource, { flag: "wx" }); + wroteConfig = true; + } catch (error) { + if (wroteConfig) await rm(configPath, { force: true }); + if (wroteLock) await rm(lockPath, { force: true }); + if (movedPinned) await rm(pinnedRoot, { recursive: true, force: true }); + throw error; + } + + return { configPath, pinnedRoot, lockPath }; + } finally { + await rm(stage, { recursive: true, force: true }); + } +} diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index e525263..c6a9203 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -469,14 +469,24 @@ function calculatePullNsids( for (const nsid of collectReferencedNsids(path)) values.add(nsid); } } - return [...values].sort(); + + // Custom and registry-pinned documents are deliberately immutable inputs. + // Keep mutable `pulled` documents in Atcute's refresh list, but do not fetch + // another copy of a source the project owns or has pinned by CID. + const fixedSourceDirs = sourceDirs.filter((directory) => { + const name = directory.replaceAll("\\", "/").split("/").pop(); + return name === "custom" || name === "pinned"; + }); + return [...values] + .filter((nsid) => !findLexicon(fixedSourceDirs, nsid)) + .sort(); } const GENERATED_ATCUTE_CONFIG_HEADER = "// Generated by `contrail lexicons generate`. Re-run the command to update; do not edit.\n"; function atcuteConfiguration(pullNsids: string[]): string { - return `${GENERATED_ATCUTE_CONFIG_HEADER}import { defineLexiconConfig } from "@atcute/lex-cli";\n\nexport default defineLexiconConfig({\n generate: {\n files: [\n "lexicons/custom/**/*.json",\n "lexicons/pulled/**/*.json",\n "lexicons/generated/**/*.json",\n ],\n outdir: "src/lexicon-types/",\n },\n pull: {\n outdir: "lexicons/pulled/",\n clean: true,\n sources: [\n {\n type: "atproto",\n mode: "nsids",\n nsids: ${JSON.stringify(pullNsids, null, 2).replace(/^/gm, " ").trim()},\n },\n ],\n },\n});\n`; + return `${GENERATED_ATCUTE_CONFIG_HEADER}import { defineLexiconConfig } from "@atcute/lex-cli";\n\nexport default defineLexiconConfig({\n generate: {\n files: [\n "lexicons/custom/**/*.json",\n "lexicons/pinned/**/*.json",\n "lexicons/pulled/**/*.json",\n "lexicons/generated/**/*.json",\n ],\n outdir: "src/lexicon-types/",\n },\n pull: {\n outdir: "lexicons/pulled/",\n clean: true,\n sources: [\n {\n type: "atproto",\n mode: "nsids",\n nsids: ${JSON.stringify(pullNsids, null, 2).replace(/^/gm, " ").trim()},\n },\n ],\n },\n});\n`; } function writeAtcuteConfiguration(rootDir: string, pullNsids: string[]): boolean { @@ -773,6 +783,7 @@ export function generateLexicons( } const sourceDirs = options.sourceDirs ?? [ join(rootDir, "lexicons", "custom"), + join(rootDir, "lexicons", "pinned"), join(rootDir, "lexicons", "pulled"), ]; const log = options.quiet ? () => {} : console.log; diff --git a/packages/contrail/tests/init-analysis.test.ts b/packages/contrail/tests/init-analysis.test.ts new file mode 100644 index 0000000..e6490a9 --- /dev/null +++ b/packages/contrail/tests/init-analysis.test.ts @@ -0,0 +1,217 @@ +import { readFileSync } from "node:fs"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadConfig } from "../src/cli-config"; +import { + analyzeLexiconGraph, + buildLexiconGraph, + deriveCollectionAliases, +} from "../src/cli/init/analyze"; +import type { + PrefixImport, + VerifiedLexiconDocument, +} from "../src/cli/init/registry"; +import { createInitArtifacts } from "../src/cli/init/render"; +import { writeInitArtifacts } from "../src/cli/init/write"; + +const fixtureRoot = resolve("../../apps/atmo-rsvp/lexicons/pulled"); +const temporaryRoots: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map((root) => + rm(root, { recursive: true, force: true }), + ), + ); +}); + +function value(nsid: string): Record { + return JSON.parse( + readFileSync(join(fixtureRoot, ...nsid.split(".")) + ".json", "utf8"), + ) as Record; +} + +function verified(nsid: string): VerifiedLexiconDocument { + return { + id: nsid, + authorityDid: "did:plc:authority", + uri: `at://did:plc:authority/com.atproto.lexicon.schema/${nsid}`, + cid: `cid-${nsid}`, + value: value(nsid), + verifiedAt: "2026-08-29T00:00:00.000Z", + verifiedUntil: "2099-08-29T00:00:00.000Z", + }; +} + +function prefixImport(): PrefixImport { + return { + api: "https://lex.atmo.tools", + prefix: "community.lexicon.calendar.", + snapshot: "snapshot", + partial: false, + verification: { + candidates: 2, + verified: 2, + pending: 0, + stale: 0, + temporaryFailure: 0, + unresolved: 0, + invalid: 0, + settled: true, + allVerified: true, + }, + indexing: { + available: true, + relayDiscoveryComplete: true, + backfillsPending: 0, + backfillsExhausted: 0, + settled: true, + complete: true, + }, + documents: [ + verified("community.lexicon.calendar.event"), + verified("community.lexicon.calendar.rsvp"), + ], + }; +} + +describe("Lexicon prefix analysis", () => { + it("closes dependencies and derives safe calendar query fields", async () => { + const graph = await buildLexiconGraph(prefixImport(), async (nsid) => + verified(nsid), + ); + const analysis = analyzeLexiconGraph(graph); + + expect(graph.roots).toEqual([ + "community.lexicon.calendar.event", + "community.lexicon.calendar.rsvp", + ]); + expect(graph.documents.map((document) => document.id)).toContain( + "com.atproto.repo.strongRef", + ); + const event = analysis.collections.find( + (collection) => collection.alias === "event", + )!; + const rsvp = analysis.collections.find( + (collection) => collection.alias === "rsvp", + )!; + expect(event.queryable).toEqual({ + createdAt: { type: "range" }, + description: {}, + endsAt: { type: "range" }, + mode: {}, + name: {}, + startsAt: { type: "range" }, + status: {}, + }); + expect(event.skipped).toEqual( + expect.arrayContaining([ + { path: "locations", reason: "array" }, + { path: "uris", reason: "array" }, + ]), + ); + expect(rsvp.queryable).toEqual({ + status: {}, + "subject.cid": {}, + "subject.uri": {}, + }); + expect(rsvp.references).toContainEqual({ + path: "subject.uri", + kind: "strongRef", + cidPath: "subject.cid", + }); + expect(rsvp.groupFields).toContainEqual({ + path: "status", + values: [ + "community.lexicon.calendar.rsvp#going", + "community.lexicon.calendar.rsvp#interested", + "community.lexicon.calendar.rsvp#notgoing", + ], + }); + }); + + it("expands colliding final segments deterministically", () => { + expect( + Object.fromEntries( + deriveCollectionAliases([ + "com.example.feed.post", + "com.example.chat.post", + "com.example.actor.profile", + ]), + ), + ).toEqual({ + "com.example.actor.profile": "profile", + "com.example.chat.post": "chatPost", + "com.example.feed.post": "feedPost", + }); + }); + + it("renders explicit references and writes a validated pinned project", async () => { + const graph = await buildLexiconGraph(prefixImport(), async (nsid) => + verified(nsid), + ); + const analysis = analyzeLexiconGraph(graph); + const artifacts = createInitArtifacts("api.example.calendar", analysis, [ + { + source: "rsvp", + path: "subject.uri", + target: "event", + name: "event", + inverse: { name: "rsvps", groupBy: "status" }, + }, + ]); + const root = await mkdtemp(join(tmpdir(), "contrail-prefix-init-")); + temporaryRoots.push(root); + const written = await writeInitArtifacts(root, artifacts); + const config = await loadConfig(written.configPath); + + expect(config).toMatchObject({ + namespace: "api.example.calendar", + profiles: [], + collections: { + event: { + validate: true, + relations: { + rsvps: { + collection: "rsvp", + field: "subject.uri", + groupBy: "status", + groups: { + going: "community.lexicon.calendar.rsvp#going", + interested: + "community.lexicon.calendar.rsvp#interested", + notgoing: "community.lexicon.calendar.rsvp#notgoing", + }, + }, + }, + }, + rsvp: { + references: { + event: { collection: "event", field: "subject.uri" }, + }, + }, + }, + }); + expect( + JSON.parse(await readFile(written.lockPath, "utf8")), + ).toMatchObject({ + format: "contrail.lexicon-import", + version: 1, + roots: graph.roots, + }); + expect( + await readFile( + join( + written.pinnedRoot, + "com", + "atproto", + "repo", + "strongRef.json", + ), + "utf8", + ), + ).toContain('"com.atproto.repo.strongRef"'); + }); +}); diff --git a/packages/contrail/tests/init-registry.test.ts b/packages/contrail/tests/init-registry.test.ts new file mode 100644 index 0000000..de2123f --- /dev/null +++ b/packages/contrail/tests/init-registry.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it } from "vitest"; +import { + fetchExactLexicon, + importLexiconPrefix, + LexiconRegistryError, + type VerifiedLexiconDocument, +} from "../src/cli/init/registry"; + +const prefix = "com.example."; + +function document(id: string): VerifiedLexiconDocument { + return { + id, + authorityDid: "did:plc:authority", + uri: `at://did:plc:authority/com.atproto.lexicon.schema/${id}`, + cid: `cid-${id}`, + value: { + lexicon: 1, + id, + defs: { + main: { + type: "record", + key: "tid", + record: { type: "object", properties: {} }, + }, + }, + }, + verifiedAt: "2026-08-29T00:00:00.000Z", + verifiedUntil: "2099-08-29T00:00:00.000Z", + }; +} + +function page(options: { + documents?: VerifiedLexiconDocument[]; + cursor?: string; + snapshot?: string; + allVerified?: boolean; + settled?: boolean; + indexingComplete?: boolean; +}) { + const documents = options.documents ?? []; + const allVerified = options.allVerified ?? true; + const settled = options.settled ?? true; + return { + prefix, + verified: true, + snapshot: options.snapshot ?? "stable", + lexicons: documents, + ...(options.cursor ? { cursor: options.cursor } : {}), + verification: { + candidates: allVerified ? documents.length : documents.length + 1, + verified: documents.length, + pending: settled ? 0 : 1, + stale: 0, + temporaryFailure: 0, + unresolved: settled && !allVerified ? 1 : 0, + invalid: 0, + settled, + allVerified, + }, + indexing: { + available: true, + relayDiscoveryComplete: true, + backfillsPending: 0, + backfillsExhausted: 0, + settled: options.indexingComplete ?? true, + complete: options.indexingComplete ?? true, + }, + }; +} + +function json(value: unknown, status = 200, headers?: HeadersInit): Response { + return Response.json(value, { status, headers }); +} + +describe("atmo Lexicon registry client", () => { + it("paginates one verified snapshot", async () => { + const requests: URL[] = []; + const fetcher: typeof fetch = async (input) => { + const url = new URL(String(input)); + requests.push(url); + return url.searchParams.has("cursor") + ? json(page({ documents: [document("com.example.two")] })) + : json( + page({ + documents: [document("com.example.one")], + cursor: "next", + }), + ); + }; + + const result = await importLexiconPrefix(prefix, { + fetcher, + timeoutMs: 5_000, + }); + expect(result.documents.map((item) => item.id)).toEqual([ + "com.example.one", + "com.example.two", + ]); + expect(requests).toHaveLength(2); + expect(requests[0]!.origin).toBe("https://lex.atmo.tools"); + expect(requests[0]!.searchParams.get("refresh")).toBe("true"); + }); + + it("discards pages and restarts after SnapshotChanged", async () => { + let requests = 0; + const fetcher: typeof fetch = async (input) => { + requests++; + const url = new URL(String(input)); + if (requests === 1) { + return json( + page({ + documents: [document("com.example.old")], + cursor: "old-next", + snapshot: "old", + }), + ); + } + if (requests === 2) return json({ error: "SnapshotChanged" }, 409); + return json( + page({ + documents: [document("com.example.new")], + snapshot: "new", + }), + ); + }; + + const result = await importLexiconPrefix(prefix, { + fetcher, + timeoutMs: 5_000, + sleep: async () => {}, + }); + expect(result.documents.map((item) => item.id)).toEqual([ + "com.example.new", + ]); + expect(requests).toBe(3); + }); + + it("fails a settled but incomplete verification unless partial is allowed", async () => { + const fetcher: typeof fetch = async () => + json( + page({ + documents: [document("com.example.one")], + allVerified: false, + settled: true, + }), + ); + + await expect( + importLexiconPrefix(prefix, { fetcher, timeoutMs: 5_000 }), + ).rejects.toMatchObject>({ + code: "IncompleteVerification", + }); + await expect( + importLexiconPrefix(prefix, { + fetcher, + timeoutMs: 5_000, + allowPartial: true, + }), + ).resolves.toMatchObject({ partial: true }); + }); + + it("fails settled exhausted catalog indexing without waiting", async () => { + const fetcher: typeof fetch = async () => { + const response = page({ + documents: [document("com.example.one")], + indexingComplete: false, + }); + response.indexing.settled = true; + response.indexing.backfillsExhausted = 1; + return json(response); + }; + await expect( + importLexiconPrefix(prefix, { fetcher, timeoutMs: 5_000 }), + ).rejects.toMatchObject>({ + code: "IncompleteIndexing", + }); + }); + + it("resolves an exact unobserved dependency after a pending response", async () => { + const nsid = "com.example.dependency"; + let requests = 0; + const fetcher: typeof fetch = async () => { + requests++; + return requests === 1 + ? json( + { + error: "VerificationPending", + id: nsid, + status: "pending", + retryAt: "2026-08-29T00:00:00.000Z", + }, + 202, + { "retry-after": "1" }, + ) + : json(document(nsid)); + }; + + await expect( + fetchExactLexicon(nsid, { + fetcher, + timeoutMs: 5_000, + sleep: async () => {}, + }), + ).resolves.toMatchObject({ id: nsid }); + expect(requests).toBe(2); + }); + + it("rejects raw or malformed registry data", async () => { + const fetcher: typeof fetch = async () => + json({ ...page({ documents: [] }), verified: false }); + await expect( + importLexiconPrefix(prefix, { fetcher, timeoutMs: 5_000 }), + ).rejects.toMatchObject>({ + code: "InvalidResponse", + }); + }); +}); diff --git a/packages/contrail/tests/init.test.ts b/packages/contrail/tests/init.test.ts index c387344..f5aaca6 100644 --- a/packages/contrail/tests/init.test.ts +++ b/packages/contrail/tests/init.test.ts @@ -3,7 +3,11 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { loadConfig } from "../src/cli-config"; -import { seedConfig, STARTER_CONFIG } from "../src/cli/commands/init"; +import { + initializeFromPrefix, + seedConfig, + STARTER_CONFIG, +} from "../src/cli/commands/init"; const roots: string[] = []; @@ -68,4 +72,107 @@ describe("contrail init", () => { `A Contrail config already exists at ${path}`, ); }); + + it("rejects prefix initialization before making a network request when a config exists", async () => { + const root = await temporaryRoot(); + const path = join(root, "contrail.config.ts"); + await writeFile(path, "export default { keep: true };\n"); + let requested = false; + + await expect( + initializeFromPrefix(root, { + prefix: "com.example.", + fetcher: async () => { + requested = true; + throw new Error("must not fetch"); + }, + interactive: false, + }), + ).rejects.toThrow(`A Contrail config already exists at ${path}`); + expect(requested).toBe(false); + }); + + it("initializes a complete non-interactive project from a verified prefix", async () => { + const root = await temporaryRoot(); + const project = join(root, "prefix-project"); + const id = "com.example.note"; + const verified = { + id, + authorityDid: "did:plc:authority", + uri: `at://did:plc:authority/com.atproto.lexicon.schema/${id}`, + cid: "cid-note", + value: { + lexicon: 1, + id, + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + properties: { + createdAt: { type: "string", format: "datetime" }, + text: { type: "string" }, + rating: { type: "integer" }, + }, + }, + }, + }, + }, + verifiedAt: "2026-08-29T00:00:00.000Z", + verifiedUntil: "2099-08-29T00:00:00.000Z", + }; + const fetcher: typeof fetch = async () => + Response.json({ + prefix: "com.example.", + verified: true, + snapshot: "stable", + lexicons: [verified], + verification: { + candidates: 1, + verified: 1, + pending: 0, + stale: 0, + temporaryFailure: 0, + unresolved: 0, + invalid: 0, + settled: true, + allVerified: true, + }, + indexing: { + available: true, + relayDiscoveryComplete: true, + backfillsPending: 0, + backfillsExhausted: 0, + settled: true, + complete: true, + }, + }); + + const result = await initializeFromPrefix(project, { + prefix: "com.example.", + namespace: "api.example.notes", + fetcher, + interactive: false, + timeoutMs: 5_000, + }); + expect(await loadConfig(result.configPath)).toEqual({ + namespace: "api.example.notes", + profiles: [], + collections: { + note: { + collection: id, + validate: true, + queryable: { + createdAt: { type: "range" }, + text: {}, + }, + }, + }, + }); + expect(result.analysis.collections[0]!.skipped).toContainEqual({ + path: "rating", + reason: "integer", + }); + }); }); diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts index 2203c59..751d526 100644 --- a/packages/contrail/tests/lexicon-generation.test.ts +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, mkdirSync, readFileSync, + renameSync, rmSync, writeFileSync, } from "node:fs"; @@ -286,6 +287,41 @@ describe("Contrail Lexicon generation", () => { ).toThrow("lex.config.js"); }); + it("bundles pinned sources without scheduling mutable pulls for them", () => { + const { root, config } = fixture(); + const source = join( + root, + "lexicons", + "pulled", + "community", + "example", + "event.json", + ); + const target = join( + root, + "lexicons", + "pinned", + "community", + "example", + "event.json", + ); + mkdirSync(join(target, ".."), { recursive: true }); + renameSync(source, target); + + generateLexicons({ config, rootDir: root, quiet: true }); + const bundle = readFileSync( + join(root, "lexicons", "generated", "index.ts"), + "utf8", + ); + expect(bundle).toContain( + 'import _0 from "../pinned/community/example/event.json";', + ); + const atcute = readFileSync(join(root, "lex.config.js"), "utf8"); + expect(atcute).toContain('"lexicons/pinned/**/*.json"'); + expect(atcute).not.toContain('"community.example.event"'); + expect(atcute).toContain('"community.example.rsvp"'); + }); + it("preserves user-owned Atcute configuration and supports opting out", () => { const { root, config } = fixture(); const path = join(root, "lex.config.js"); -- 2.51.2 From b8515837b82422b02451f6563b25e4accbbb2a99 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 30 Aug 2026 11:56:57 +0200 Subject: [PATCH 25/34] jetstream v2 --- .changeset/jetstream-v2-live.md | 5 + .github/workflows/benchmark.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- apps/atmo-rsvp/src/contrail.config.ts | 4 +- .../generated/com/example/getCursor.json | 10 +- apps/cloudflare-workers/package.json | 3 +- .../generated/statusphere/app/getCursor.json | 10 +- .../sveltekit-cloudflare-workers/package.json | 5 +- .../types/statusphere/app/getCursor.ts | 4 +- .../src/routes/+page.svelte | 42 +- docs/01-getting-started.md | 2 +- docs/04-deploy-cloudflare.md | 2 +- package.json | 3 + packages/contrail/README.md | 17 +- packages/contrail/package.json | 5 + .../contrail/src/cli/commands/backfill.ts | 8 + packages/contrail/src/cli/commands/connect.ts | 2 +- packages/contrail/src/cli/commands/dev.ts | 14 +- packages/contrail/src/core/backfill.ts | 7 +- packages/contrail/src/core/db/index.ts | 2 +- packages/contrail/src/core/db/records.ts | 137 ++++- packages/contrail/src/core/jetstream-live.ts | 330 +++++++++++ .../contrail/src/core/jetstream-source.ts | 6 +- packages/contrail/src/core/jetstream.ts | 259 +++----- packages/contrail/src/core/persistent.ts | 118 +++- .../contrail/src/core/router/diagnostics.ts | 14 +- packages/contrail/src/core/types.ts | 52 +- packages/contrail/src/index.ts | 4 + packages/contrail/src/lexicons/generate.ts | 10 +- packages/contrail/src/workers/backfill.ts | 18 +- packages/contrail/tests/ingest-hang.test.ts | 135 +++-- .../tests/jetstream-single-url.test.ts | 49 +- .../contrail/tests/jetstream-v2-live.test.ts | 212 +++++++ .../contrail/tests/lexicon-generation.test.ts | 6 +- .../tests/network-overrides-appview.test.ts | 7 +- packages/contrail/tests/persistent.test.ts | 122 +++- .../contrail/tests/scheduled-ingest.test.ts | 163 ++++-- packages/contrail/tests/types.test.ts | 4 +- packages/contrail/tests/worker.test.ts | 7 +- pnpm-lock.yaml | 553 +++++++++++++++++- 41 files changed, 1892 insertions(+), 465 deletions(-) create mode 100644 .changeset/jetstream-v2-live.md create mode 100644 packages/contrail/src/core/jetstream-live.ts create mode 100644 packages/contrail/tests/jetstream-v2-live.test.ts diff --git a/.changeset/jetstream-v2-live.md b/.changeset/jetstream-v2-live.md new file mode 100644 index 0000000..fa487ae --- /dev/null +++ b/.changeset/jetstream-v2-live.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Switch scheduled and persistent live ingestion to Jetstream v2 using the official `@bsky/jetstream` client. Live cursors now use instance-local sequence numbers, one normalized v2 service is durably pinned to each generation, legacy timestamp cursors transition atomically into the seq domain, and existing PDS/Alluvium historical acquisition remains unchanged. Alluvium bootstrap now has a separate `sourceUrl`/`--alluvium-source-url` for its legacy v1 manifest identity. Unscoped `getCursor` responses now return `{ cursor }`; ordered-source responses retain their opaque `{ source, epoch, cursor }` position. The minimum supported Node.js version is now 22.15. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4ebb3f4..322cbd4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22.14 + node-version: 22.15 cache: 'pnpm' - name: Install dependencies diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d026ac1..8735e58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22.14 + node-version: 22.15 cache: 'pnpm' - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c99c117..9ae1749 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22.14 + node-version: 22.15 cache: 'pnpm' registry-url: 'https://registry.npmjs.org' diff --git a/apps/atmo-rsvp/src/contrail.config.ts b/apps/atmo-rsvp/src/contrail.config.ts index 37d894e..9e57275 100644 --- a/apps/atmo-rsvp/src/contrail.config.ts +++ b/apps/atmo-rsvp/src/contrail.config.ts @@ -3,10 +3,10 @@ import type { ContrailConfig } from "@atmo-dev/contrail"; export const config: ContrailConfig = { namespace: "rsvp.atmo", profiles: ["app.bsky.actor.profile"], - jetstreams: ["wss://jetstream1.us-east.bsky.network"], + jetstreams: ["https://jetstream.us-east.bsky.network"], orderedSource: { source: "jetstream", - epoch: "api-atmo-rsvp-primary-2026-08", + epoch: "api-atmo-rsvp-primary-v2-2026-08", }, notify: true, serviceAuth: { diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json index 93bb5dd..345596c 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json @@ -4,19 +4,13 @@ "defs": { "main": { "type": "query", - "description": "Get the current ingestion observation time", + "description": "Get the current Jetstream live cursor", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "time_us": { - "type": "integer" - }, - "date": { - "type": "string" - }, - "seconds_ago": { + "cursor": { "type": "integer" } } diff --git a/apps/cloudflare-workers/package.json b/apps/cloudflare-workers/package.json index 96bf1e5..8b6926d 100644 --- a/apps/cloudflare-workers/package.json +++ b/apps/cloudflare-workers/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "dev": "contrail dev", - "deploy": "wrangler deploy" + "deploy": "wrangler deploy", + "typecheck": "contrail lexicons check --no-atcute-config && tsc --noEmit" }, "dependencies": { "@atmo-dev/contrail": "workspace:*" diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json index 32cd295..a7e6ce7 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json @@ -4,19 +4,13 @@ "defs": { "main": { "type": "query", - "description": "Get the current ingestion observation time", + "description": "Get the current Jetstream live cursor", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "time_us": { - "type": "integer" - }, - "date": { - "type": "string" - }, - "seconds_ago": { + "cursor": { "type": "integer" } } diff --git a/apps/sveltekit-cloudflare-workers/package.json b/apps/sveltekit-cloudflare-workers/package.json index 9023e69..dbcacdb 100644 --- a/apps/sveltekit-cloudflare-workers/package.json +++ b/apps/sveltekit-cloudflare-workers/package.json @@ -12,7 +12,7 @@ "preview": "vite preview", "prepare": "svelte-kit sync || echo ''", "check": "pnpm generate:types && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "typecheck": "pnpm check", + "typecheck": "contrail lexicons check --no-atcute-config && pnpm check", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "format": "prettier --write .", "lint": "prettier --check . && eslint .", @@ -58,8 +58,9 @@ }, "license": "MIT", "dependencies": { - "@atcute/jetstream": "^2.0.2", "@atmo-dev/contrail": "workspace:*", + "@atproto/lex": "^0.3.8", + "@bsky/jetstream": "^1.0.0", "@foxui/core": "^0.9.1", "@foxui/social": "^0.8.10", "@foxui/time": "^0.8.5", diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts index 05a42fc..4906ad0 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts @@ -9,9 +9,7 @@ const _mainSchema = /*#__PURE__*/ v.query( "output": { "type": "lex", "schema": /*#__PURE__*/ v.object({ - "date": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), - "seconds_ago": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), - "time_us": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + "cursor": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), }), } } diff --git a/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte b/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte index 422fb87..ed85282 100644 --- a/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte +++ b/apps/sveltekit-cloudflare-workers/src/routes/+page.svelte @@ -7,7 +7,7 @@ import { Button } from '@foxui/core'; import { GithubCorner, PopoverEmojiPicker } from '@foxui/social'; import { RelativeTime } from '@foxui/time'; - import { JetstreamSubscription } from '@atcute/jetstream'; + import { Jetstream } from '@bsky/jetstream'; import { createTID } from '$lib/atproto/methods'; import { putRecord } from '$lib/atproto/server/repo.remote'; @@ -51,34 +51,26 @@ let allStatuses = $derived([...localStatuses, ...liveStatuses, ...data.statuses]); - // Jetstream subscription + // Jetstream v2 live subscription onMount(() => { - const subscription = new JetstreamSubscription({ - url: 'wss://jetstream1.us-east.bsky.network', - wantedCollections: ['xyz.statusphere.status'] - }); - - const iterator = subscription[Symbol.asyncIterator](); + const controller = new AbortController(); + const jetstream = new Jetstream('https://jetstream.us-east.bsky.network'); (async () => { try { - while (true) { - const { value: event, done } = await iterator.next(); - if (done) break; - - if (event.kind !== 'commit') continue; - if (event.commit.operation !== 'create') continue; + for await (const event of jetstream.live({ + raw: true, + kinds: ['commit'], + collections: ['xyz.statusphere.status'], + signal: controller.signal + })) { + if (event.kind !== 'commit' || event.commit.operation !== 'create') continue; const { did } = event; const { rkey, record } = event.commit; - if ( - record === null || - typeof record !== 'object' || - typeof record.status !== 'string' || - typeof record.createdAt !== 'string' - ) { - continue; - } + if (record === null || typeof record !== 'object' || Array.isArray(record)) continue; + const value = record as Record; + if (typeof value.status !== 'string' || typeof value.createdAt !== 'string') continue; const key = `${did}-${rkey}`; if (seenKeys.has(key)) continue; @@ -87,7 +79,7 @@ await fetchProfile(did); liveStatuses = [ - { did, rkey, status: record.status, createdAt: record.createdAt }, + { did, rkey, status: value.status, createdAt: value.createdAt }, ...liveStatuses ]; } @@ -96,9 +88,7 @@ } })(); - return () => { - iterator.return!(); - }; + return () => controller.abort(); }); diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index fbdb23b..5d90863 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -1,6 +1,6 @@ # Get started locally -Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. +Run a local AppView for a public AT Protocol collection with Node.js 22.15 or newer. Create a project with the starter config: diff --git a/docs/04-deploy-cloudflare.md b/docs/04-deploy-cloudflare.md index 5252755..496e039 100644 --- a/docs/04-deploy-cloudflare.md +++ b/docs/04-deploy-cloudflare.md @@ -22,7 +22,7 @@ const config: ContrailConfig = { namespace: "com.example", orderedSource: { source: "jetstream", - epoch: "my-appview-v1", + epoch: "my-appview-jetstream-v2", }, collections: { // ...your existing collections diff --git a/package.json b/package.json index 0252b84..3feede4 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,9 @@ "private": true, "version": "0.0.0", "type": "module", + "engines": { + "node": ">=22.15.0" + }, "scripts": { "build": "turbo run build --filter=./packages/*", "build:apps": "turbo run build", diff --git a/packages/contrail/README.md b/packages/contrail/README.md index e30e3dd..7f7278b 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -79,7 +79,9 @@ await contrail.runPersistent({ After a write to a user's PDS, `contrail.notify(uri)` can fetch the authoritative record immediately. Only an authoritative not-found response deletes local state; rate limits, server errors, timeouts, malformed responses, and network failures leave it unchanged. Authentication and abuse controls for the public HTTP operation remain under design. -Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Scheduled collection drops exact Jetstream observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity updates have an independent retained-count limit, coalesce by DID, and commit in database batches; excess global identity traffic is accounted but omitted so it cannot starve record progress. Source-filtered observations consume neither candidate nor byte limits, but their yielded cursors remain accounted. Durable actor scope comes from identities, relay discovery, and visible discoverable records, so a failed identity refresh cannot make a restarted isolate filter equal-cursor dependent siblings. Bounded exact-observation hashes at the current microsecond cursor let restarts replay one microsecond of overlap without skipping equal-cursor siblings or recounting earlier ones. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact accounted cursor in the same transaction. Scheduled mode requires one pinned Jetstream endpoint; use `runPersistent()` for an Atcute failover pool. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. +Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Live ingestion speaks Jetstream v2 and checkpoints its unique monotonic `seq`; the RFC 3339 event time remains display/source metadata and is never used as the resume coordinate. Scheduled collection drops exact observations before admission, counts UTF-8 record bodies plus a fixed 512-byte metadata allowance, and retains the threshold-crossing candidate before stopping. Identity updates have an independent retained-count limit, coalesce by DID, and commit in database batches; excess global identity traffic is accounted but omitted so it cannot starve record progress. Source-filtered observations consume neither candidate nor byte limits, but their yielded seqs remain accounted. Durable actor scope comes from identities, relay discovery, and visible discoverable records. Existing v1 timestamp cursors cross a bounded overlap bridge and atomically transition once to the v2 seq domain. If either a timestamp or seq cursor has already fallen outside the hosted live-retention window, ingestion fails rather than accepting the server's timestamp clamp and silently skipping a gap; rebuild/backfill from a fresh boundary until archive replay is added. Durable tombstones reject stale resurrection, and live projection commits its exact accounted seq in the same transaction. Both scheduled and persistent modes require one pinned Jetstream v2 service because seqs are instance-local. Persistent ingestion retains its streaming batch lifecycle and does not inherit the scheduled count/byte defaults. Historical acquisition still uses the existing PDS or optional Alluvium paths; Jetstream v2 archive replay is not used yet. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. + +When upgrading an existing database, point `jetstreams` at one v2 service and change `orderedSource.epoch` to identify the new continuity history. The normalized service origin is durably bound before the timestamp-to-seq transition because seqs are instance-local; changing it later is a hard error. Initialization permits the ordered-source epoch change only while the durable cursor is still in the legacy timestamp domain; the first fully handled v2 event then installs the seq domain one-way. Once a database is in the seq domain, an epoch mismatch remains a hard error. An already-seq-domain database with no durable service binding is also rejected because its originating instance cannot be inferred safely. Projection winner selection is guarded again inside the write transaction. Overlapping cron, persistent, notify, or backfill writers cannot commit a stale canonical row, derived projection, tombstone, or source checkpoint; a changed predecessor rolls the complete attempt back and retries from fresh durable state. @@ -208,10 +210,11 @@ An optional experimental Alluvium adapter consumes existing version-1 collection ```bash contrail backfill --sqlite ./data/contrail.sqlite --alluvium \ --alluvium-epoch my-source-continuity-epoch \ + --alluvium-source-url wss://jetstream1.us-east.bsky.network \ --allow-partial ``` -The epoch is deliberately required because Alluvium protocol v1 does not publish one. `--allow-partial` is needed only when selected manifests explicitly report historical omissions. The command rejects a populated database unless it is resuming durable Alluvium bootstrap state. The equivalent programmatic API is: +The epoch is deliberately required because Alluvium protocol v1 does not publish one. `--alluvium-source-url` names the exact legacy v1 source advertised by the manifests and defaults to the hosted east source shown above; it is intentionally separate from `config.jetstreams`, which selects the v2 live service used after bootstrap. `--allow-partial` is needed only when selected manifests explicitly report historical omissions. The command rejects a populated database unless it is resuming durable Alluvium bootstrap state. The equivalent programmatic API is: ```ts import { DatabaseBootstrapTarget } from "@atmo-dev/contrail"; @@ -236,7 +239,7 @@ const source = createAlluviumBootstrapSources(config, { }); const target = new DatabaseBootstrapTarget(db, config, { deferDerivedProjections: true, - // Commit the archived time_us as the ordinary cron ingestion cursor. + // Commit the archived v1 time_us as the one-time v2 live bridge cursor. liveCursor: createAlluviumLiveCursor(sourceIdentity), }); @@ -247,7 +250,7 @@ await bootstrapFreshProjection({ }); ``` -The offline call loads only the base plus published archive tail. It does not replay direct Jetstream to a newly marked head. The target atomically persists Alluvium's final archived `time_us` as the normal live cursor, so subsequent bounded cron invocations resume there and catch up over as many scheduled windows as necessary. +The offline call loads only the base plus published archive tail. It does not replay direct Jetstream to a newly marked head. The target atomically persists Alluvium's final archived v1 `time_us`; the next v2 live observation transitions that bridge coordinate to the service's seq domain, after which bounded cron invocations resume by seq. The configured live v2 service and continuity epoch remain operator-owned because Alluvium protocol v1 cannot publish a v2 handoff coordinate. This first adapter is generation-only and deliberately strict: selected manifests must expose one shared base and archive boundary, source ID/URL must match configuration, known capture gaps fail, and partial historical coverage requires `allowPartial: true`. It buffers each bounded compressed object so checksum and byte length are verified before any rows are projected. Because version-1 manifests lack global multi-collection delivery steps, it also restores source order with a bounded 100,000-mutation tail buffer; larger tails require the planned bundle protocol. It does not silently fall back to PDS acquisition; omitted-account repair and dependent profile/follow enrichment remain separately accounted PDS work. @@ -290,7 +293,7 @@ const config = { The service description advertises the base service DID, exact fragmented audience, canonical OAuth scope, and each protected query/procedure separately from anonymous methods. `contrail connect` generates `src/contrail/index.ts`; anonymous calls go directly to the configured endpoint, while protected calls lazily discover and validate all service-auth fields. Its exported `contrail.scope` contains one sorted `lxm` parameter per protected method, such as `rpc?aud=did:web:api.example.com%23contrail&lxm=.getFeed&lxm=.notifyOfUpdate`. `contrail.authenticated(authenticatedClient)` returns one Atcute client: advertised provider methods route to Contrail, ordinary methods route to the PDS, and successful tracked `createRecord`, `putRecord`, and `deleteRecord` calls automatically notify Contrail. Handle-form deletes are resolved to canonical DID URIs. Protected methods use cached exact method-bound tokens, while transient discovery failures remain retryable. Feed actors must resolve to the token issuer, and every notified AT URI must belong to the issuer. When the base DID resolves to the public endpoint's `/.well-known/did.json`, the Worker publishes a DID document whose service entry ID is the exact audience. -Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Existing non-public deployments without one retain the legacy ingestion-time cursor response: +Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Configure that continuity identity explicitly; non-public deployments without it expose only the unscoped numeric live cursor as `{ cursor }`: ```ts const config = { @@ -302,7 +305,7 @@ const config = { }; ``` -The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` value for equality; never order cursors from different epochs. Consumers can read the position before and after a query, retry if it changed, then poll it as a refetch/invalidation signal. +The ordered-source cursor is opaque. Compare the complete `{ source, epoch, cursor }` value for equality; never order cursors from different epochs. Consumers can read the position before and after a query, retry if it changed, then poll it as a refetch/invalidation signal. Connect an independent consumer with `contrail connect `. The version-2 provider lock records the deployment and exact Lexicon bundle, but generated clients do not pin the provider's complete method set at runtime. Existing anonymous methods therefore continue working when a provider adds methods. A repeated connection to the same endpoint and provider-owned output root requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. Version-1 locks must be removed and reconnected. @@ -365,4 +368,4 @@ import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; ``` -The SQLite adapter uses the built-in `node:sqlite` module and therefore requires Node.js 22.13 or newer. D1 implements Contrail's database interface directly. +Contrail requires Node.js 22.15 or newer. The SQLite adapter uses the built-in `node:sqlite` module; D1 implements Contrail's database interface directly. diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 097a403..b02a1d6 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -4,6 +4,9 @@ "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, + "engines": { + "node": ">=22.15.0" + }, "files": [ "dist" ], @@ -91,6 +94,8 @@ "@atcute/tid": "1.1.4", "@atcute/uint8array": "1.1.5", "@atcute/xrpc-server": "^2.0.2", + "@atproto/lex": "^0.3.8", + "@bsky/jetstream": "^1.0.0", "cac": "^7.0.0", "hono": "^4.13.0", "jiti": "^2.7.0", diff --git a/packages/contrail/src/cli/commands/backfill.ts b/packages/contrail/src/cli/commands/backfill.ts index 24fac2a..1414b59 100644 --- a/packages/contrail/src/cli/commands/backfill.ts +++ b/packages/contrail/src/cli/commands/backfill.ts @@ -4,6 +4,7 @@ import { backfillAll, bootstrapAlluvium, bootstrapAlluviumDatabase, + DEFAULT_ALLUVIUM_SOURCE_URL, labelsBackfillAll, labelsBackfillDatabase, } from "../../workers/backfill.js"; @@ -26,6 +27,7 @@ interface BackfillOpts { alluvium?: boolean; alluviumEndpoint: string; alluviumSourceId: string; + alluviumSourceUrl: string; alluviumEpoch?: string; alluviumRetentionHours: number; allowPartial?: boolean; @@ -87,6 +89,11 @@ export function registerBackfill(cli: CAC): void { "Logical source ID expected in Alluvium manifests", { default: "jetstream-us-east" }, ) + .option( + "--alluvium-source-url ", + "Exact legacy Jetstream v1 URL advertised by Alluvium manifests", + { default: DEFAULT_ALLUVIUM_SOURCE_URL }, + ) .option( "--alluvium-epoch ", "Operator-owned continuity epoch (required with --alluvium)", @@ -142,6 +149,7 @@ export function registerBackfill(cli: CAC): void { const alluvium = { endpoint: options.alluviumEndpoint, sourceId: options.alluviumSourceId, + sourceUrl: options.alluviumSourceUrl, sourceEpoch: options.alluviumEpoch, allowPartial: options.allowPartial === true, retentionUs: retentionHours * 60 * 60 * 1_000_000, diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index 5f92304..2b93b20 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -498,7 +498,7 @@ export async function connectConfigSource(options: { notify: config.notify ?? true, orderedSource: config.orderedSource ?? { source: "jetstream", - epoch: "contrail-local-jetstream-v1", + epoch: "contrail-local-jetstream-v2", }, }; const endpoint = normalizePublicServiceEndpoint( diff --git a/packages/contrail/src/cli/commands/dev.ts b/packages/contrail/src/cli/commands/dev.ts index 1759c89..e312f31 100644 --- a/packages/contrail/src/cli/commands/dev.ts +++ b/packages/contrail/src/cli/commands/dev.ts @@ -14,7 +14,10 @@ import { getBackfillStatus } from "../../core/status.js"; import type { ContrailConfig, Database } from "../../core/types.js"; import { configProjectRoot } from "../../cli-config.js"; import { createHandler } from "../../server.js"; -import { bootstrapAlluviumDatabase } from "../../workers/backfill.js"; +import { + bootstrapAlluviumDatabase, + DEFAULT_ALLUVIUM_SOURCE_URL, +} from "../../workers/backfill.js"; import { defaultConsumerLexiconRoot, prepareDevLexicons, @@ -43,6 +46,7 @@ interface DevOpts { alluvium?: boolean; alluviumEndpoint: string; alluviumSourceId: string; + alluviumSourceUrl: string; alluviumEpoch?: string; alluviumRetentionHours: number; allowPartial?: boolean; @@ -79,7 +83,7 @@ function devConfig(config: ContrailConfig, options: DevOpts): ContrailConfig { source: options.alluviumSourceId, epoch: options.alluviumEpoch ?? "contrail-local-alluvium-v1", } - : { source: "jetstream", epoch: "contrail-local-jetstream-v1" }), + : { source: "jetstream", epoch: "contrail-local-jetstream-v2" }), }; } @@ -278,6 +282,7 @@ async function runSqliteDev( lexicons, endpoint: options.alluviumEndpoint, sourceId: ordered.source, + sourceUrl: options.alluviumSourceUrl, sourceEpoch: ordered.epoch, retentionUs: retentionHours * 60 * 60 * 1_000_000, allowPartial: options.allowPartial === true, @@ -406,6 +411,11 @@ export function registerDev(cli: CAC): void { .option("--alluvium-source-id ", "Alluvium manifest source ID", { default: "jetstream-us-east", }) + .option( + "--alluvium-source-url ", + "Exact legacy Jetstream v1 URL advertised by Alluvium manifests", + { default: DEFAULT_ALLUVIUM_SOURCE_URL }, + ) .option("--alluvium-epoch ", "Operator-owned continuity epoch") .option( "--alluvium-retention-hours ", diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index e84ad58..1c33bc4 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -35,10 +35,9 @@ const BACKFILL_RETRY_BASE_MS = 15 * 60_000; const BACKFILL_RETRY_MAX_MS = 48 * 60 * 60_000; const DEFAULT_SCHEDULED_MAX_ATTEMPTS = 10; const DERIVED_PROJECTIONS_DIRTY_KEY = "backfill_derived_projections_dirty"; -/** Legacy Jetstream v1 uses wall-clock microseconds as its replay coordinate. - * Keep the same overlap Atcute uses for multi-endpoint clock skew. The future - * source adapter replaces this compatibility marker with a source-owned epoch - * and cursor. */ +/** Historical acquisition still captures a wall-clock boundary. Jetstream v2 + * accepts it through its timestamp compatibility domain; the first delivered + * live event atomically replaces it with the service's seq cursor. */ const INITIAL_CAPTURE_OVERLAP_US = 10_000_000; async function ensureInitialReplayBoundary( diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index 78b38a2..71a85cf 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,6 +1,6 @@ export { initSchema, CONTRAIL_SCHEMA_VERSION } from "./schema"; export { getMeta, setMeta, getMetaNumber } from "./meta"; export { optimizeDatabase } from "./optimize"; -export { assertServingSourceCompatibility, getLastCursor, loadKnownActorDids, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; +export { assertJetstreamServiceCompatibility, assertServingSourceCompatibility, getLastCursor, loadKnownActorDids, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, saveJetstreamCursor, saveJetstreamCursorStatement, getCursorObservations, saveCursorObservationStatements, saveJetstreamCursorObservationStatements, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult, ServingSourcePosition } from "./records"; export type { RecordSource } from "../types"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index 1567b9a..f0a348c 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -1,3 +1,7 @@ +import { + JETSTREAM_V2_SEQ_THRESHOLD, + isJetstreamTimestampCursor, +} from "../jetstream-live"; import type { ContrailConfig, ResolvedContrailConfig, @@ -580,6 +584,48 @@ export function saveServingSourcePositionStatement( .bind(position.source, position.epoch, position.cursor, updatedAt); } +export const JETSTREAM_V2_SERVICE_META_KEY = "jetstream_v2_service"; + +/** Bind instance-local v2 seqs to one normalized service origin. A legacy + * timestamp bridge may acquire its binding before transition; an unbound seq + * cursor is rejected because its originating instance cannot be recovered. */ +export async function assertJetstreamServiceCompatibility( + db: Database, + service: string, +): Promise { + let existing = await db + .prepare("SELECT value FROM _contrail_meta WHERE key = ?") + .bind(JETSTREAM_V2_SERVICE_META_KEY) + .first<{ value: string }>(); + + if (!existing) { + const cursor = await getLastCursor(db); + if (cursor !== null && !isJetstreamTimestampCursor(cursor)) { + throw new Error( + `durable Jetstream v2 seq cursor ${cursor} has no pinned service identity; ` + + "rebuild or explicitly migrate the generation", + ); + } + await db + .prepare( + "INSERT INTO _contrail_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING", + ) + .bind(JETSTREAM_V2_SERVICE_META_KEY, service) + .run(); + existing = await db + .prepare("SELECT value FROM _contrail_meta WHERE key = ?") + .bind(JETSTREAM_V2_SERVICE_META_KEY) + .first<{ value: string }>(); + } + + if (existing?.value !== service) { + throw new Error( + `configured Jetstream v2 service ${service} does not match durable service ` + + `${existing?.value ?? ""}; seq cursors are instance-local`, + ); + } +} + export async function assertServingSourceCompatibility( db: Database, orderedSource?: OrderedSourceConfig, @@ -602,6 +648,15 @@ export async function assertServingSourceCompatibility( existing.position.source !== orderedSource.source || existing.position.epoch !== orderedSource.epoch ) { + const sourcePositionCursor = Number(existing.position.cursor); + const liveCursor = await getLastCursor(db); + const pendingV2Transition = + existing.position.source === orderedSource.source && + Number.isSafeInteger(sourcePositionCursor) && + isJetstreamTimestampCursor(sourcePositionCursor) && + liveCursor !== null && + isJetstreamTimestampCursor(liveCursor); + if (pendingV2Transition) return; throw new Error( `configured ordered source ${orderedSource.source}/${orderedSource.epoch} ` + `does not match durable source position ` + @@ -679,20 +734,54 @@ export async function loadKnownActorDids( return known; } +/** Save a legacy timestamp cursor monotonically without allowing an old + * writer to move a database that has transitioned to the v2 seq domain back + * into the timestamp domain. */ export function saveCursorStatement( db: Database, timeUs: number, ): Statement { return db .prepare( - "INSERT INTO cursor (id, time_us) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET time_us = CASE WHEN excluded.time_us > cursor.time_us THEN excluded.time_us ELSE cursor.time_us END", + `INSERT INTO cursor (id, time_us) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET time_us = CASE + WHEN cursor.time_us < ${JETSTREAM_V2_SEQ_THRESHOLD} + AND excluded.time_us >= ${JETSTREAM_V2_SEQ_THRESHOLD} + THEN cursor.time_us + WHEN excluded.time_us > cursor.time_us THEN excluded.time_us + ELSE cursor.time_us + END`, ) .bind(timeUs); } -/** Exact source observations already accounted for at the current coarse - * cursor. Scheduled Jetstream resumes one microsecond earlier and uses these - * hashes to avoid both skipping same-cursor siblings and recounting prior ones. */ +/** Save a Jetstream v2 cursor monotonically while permitting the one-way + * legacy timestamp -> v2 seq transition. A stale timestamp-domain writer can + * never move an already-transitioned seq cursor back into the old domain. */ +export function saveJetstreamCursorStatement( + db: Database, + cursor: number, +): Statement { + return db + .prepare( + `INSERT INTO cursor (id, time_us) VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET time_us = CASE + WHEN cursor.time_us >= ${JETSTREAM_V2_SEQ_THRESHOLD} + AND excluded.time_us < ${JETSTREAM_V2_SEQ_THRESHOLD} + THEN excluded.time_us + WHEN cursor.time_us < ${JETSTREAM_V2_SEQ_THRESHOLD} + AND excluded.time_us >= ${JETSTREAM_V2_SEQ_THRESHOLD} + THEN cursor.time_us + WHEN excluded.time_us > cursor.time_us THEN excluded.time_us + ELSE cursor.time_us + END`, + ) + .bind(cursor); +} + +/** Legacy timestamp-domain observation hashes. V2 seqs are unique and live + * ingestion clears these rows when it next commits; the reader remains for + * schema/API compatibility during the transition. */ export async function getCursorObservations( db: Database, timeUs: number, @@ -729,6 +818,28 @@ export function saveCursorObservationStatements( ]; } +/** V2 may move once from a large timestamp cursor to a smaller seq. Retain + * observations only at the exact resulting cursor so timestamp-domain rows do + * not survive that numeric decrease indefinitely. */ +export function saveJetstreamCursorObservationStatements( + db: Database, + cursor: number, + observations: Iterable, +): Statement[] { + return [ + db.prepare( + "DELETE FROM cursor_observations WHERE time_us <> (SELECT time_us FROM cursor WHERE id = 1)", + ), + ...[...new Set(observations)].map((observation) => + db + .prepare( + "INSERT INTO cursor_observations (time_us, observation) SELECT ?, ? WHERE (SELECT time_us FROM cursor WHERE id = 1) = ? ON CONFLICT(time_us, observation) DO NOTHING", + ) + .bind(cursor, observation, cursor), + ), + ]; +} + export async function saveCursor( db: Database, timeUs: number, @@ -745,6 +856,24 @@ export async function saveCursor( await db.batch(statements); } +export async function saveJetstreamCursor( + db: Database, + cursor: number, + orderedSource?: OrderedSourceConfig, + observations: Iterable = [], +): Promise { + const statements = [saveJetstreamCursorStatement(db, cursor)]; + if (orderedSource) { + statements.push( + saveOrderedSourcePositionStatement(db, orderedSource, cursor), + ); + } + statements.push( + ...saveJetstreamCursorObservationStatements(db, cursor, observations), + ); + await db.batch(statements); +} + // --- Existing record lookup --- export interface ExistingRecordInfo { diff --git a/packages/contrail/src/core/jetstream-live.ts b/packages/contrail/src/core/jetstream-live.ts new file mode 100644 index 0000000..13ae39e --- /dev/null +++ b/packages/contrail/src/core/jetstream-live.ts @@ -0,0 +1,330 @@ +import { + Jetstream, + websocketTransport, + type CollectionFilter, + type LiveOpts, + type LiveTransport, + type RawEvent, + type RawRecordJson, +} from "@bsky/jetstream"; + +/** Jetstream v2 reserves values at and above this boundary for legacy + * unix-microsecond timestamp cursors. Real v2 seq cursors stay below it. */ +export const JETSTREAM_V2_SEQ_THRESHOLD = 1_000_000_000_000_000; + +/** A timestamp bridge intentionally starts slightly before the prior v1 + * position. V1 and v2 are separate instances with separate witness clocks; + * ordinary source ordering makes the overlap idempotent. */ +export const JETSTREAM_V1_BRIDGE_OVERLAP_US = 10_000_000; + +export class JetstreamLiveHistoryExpiredError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "JetstreamLiveHistoryExpiredError"; + } +} + +export function isJetstreamTimestampCursor(cursor: number): boolean { + return cursor >= JETSTREAM_V2_SEQ_THRESHOLD; +} + +export function advanceJetstreamCursor( + current: number | null, + observedSeq: number, +): number { + if ( + !Number.isSafeInteger(observedSeq) || + observedSeq < 0 || + observedSeq >= JETSTREAM_V2_SEQ_THRESHOLD + ) { + throw new TypeError("Jetstream seq must be in the v2 sequence domain"); + } + if (current === null || isJetstreamTimestampCursor(current)) { + return observedSeq; + } + return Math.max(current, observedSeq); +} + +/** Preserve all six microsecond digits from Jetstream's RFC 3339 display time; + * Date.parse alone truncates the final three digits. */ +export function jetstreamDatetimeToMicroseconds(value: string): number { + const match = /^(.*?)(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(value); + if (!match) throw new TypeError(`Invalid Jetstream datetime: ${value}`); + const wholeMilliseconds = Date.parse(`${match[1]}${match[3]}`); + if (!Number.isFinite(wholeMilliseconds)) { + throw new TypeError(`Invalid Jetstream datetime: ${value}`); + } + const fractionUs = Number((match[2] ?? "").slice(0, 6).padEnd(6, "0")); + const result = wholeMilliseconds * 1_000 + fractionUs; + if (!Number.isSafeInteger(result)) { + throw new TypeError(`Jetstream datetime is outside the safe range: ${value}`); + } + return result; +} + +export type JetstreamLiveEvent = + | { + kind: "commit"; + did: string; + seq: number; + time_us: number; + commit: + | { + operation: "create" | "update"; + rev: string; + collection: string; + rkey: string; + cid: string; + record: unknown; + } + | { + operation: "delete"; + rev: string; + collection: string; + rkey: string; + }; + } + | { + kind: "identity"; + did: string; + seq: number; + time_us: number; + identity: { did: string; handle?: string; time?: string }; + } + | { + kind: "account"; + did: string; + seq: number; + time_us: number; + account: { + did: string; + active: boolean; + status?: string; + time?: string; + }; + } + | { + kind: "sync"; + did: string; + seq: number; + time_us: number; + sync: { did: string; rev: string; time?: string }; + }; + +const SUBSCRIBE_EVENTS_NSID = "network.bsky.jetstream.subscribeEvents"; + +/** A normal HTTP request reaches the same pre-upgrade validation as a + * WebSocket dial. Hosted v2 services return 426 for an accepted cursor and a + * structured 400 CursorTooOld before checking Upgrade. Unlike browser + * WebSockets, fetch exposes that status and body in Workers and browsers. */ +async function assertSeqCursorAvailable(options: { + service: string; + cursor: number; + fetchImpl: typeof fetch; + signal?: AbortSignal; +}): Promise { + const service = new URL(options.service); + if (service.protocol === "ws:") service.protocol = "http:"; + if (service.protocol === "wss:") service.protocol = "https:"; + const url = new URL(`/xrpc/${SUBSCRIBE_EVENTS_NSID}`, service.origin); + url.searchParams.set("cursor", String(options.cursor)); + // Keep this probe independent of a deployment's collection count. Cursor + // validation is source-global; one valid kind is enough to reach it. + url.searchParams.set("kinds", "commit"); + + const response = await options.fetchImpl(url, { + cache: "no-store", + redirect: "error", + signal: options.signal, + }); + const body = (await response.text()).slice(0, 2_000); + let errorName: unknown; + let message: unknown; + try { + const parsed = JSON.parse(body) as { error?: unknown; message?: unknown }; + errorName = parsed.error; + message = parsed.message; + } catch { + // Valid cursors return a plain-text 426 because this probe is deliberately + // not a WebSocket upgrade. + } + + if ( + response.status === 400 && + (errorName === "CursorTooOld" || /cursor too old/i.test(body)) + ) { + throw new JetstreamLiveHistoryExpiredError( + `Jetstream seq cursor is older than the live retention window${typeof message === "string" ? `: ${message}` : ""}`, + ); + } + if (response.status !== 426) { + throw new Error( + `Jetstream live cursor preflight failed (${response.status})${body ? `: ${body}` : ""}`, + ); + } +} + +function normalizeLiveEvent( + event: RawEvent, +): JetstreamLiveEvent { + const base = { + did: event.did, + seq: event.seq, + time_us: jetstreamDatetimeToMicroseconds(event.time), + }; + if (event.kind === "commit") { + const commit = event.commit; + return commit.operation === "delete" + ? { + ...base, + kind: "commit", + commit: { + operation: "delete", + rev: commit.rev, + collection: commit.collection, + rkey: commit.rkey, + }, + } + : { + ...base, + kind: "commit", + commit: { + operation: commit.operation, + rev: commit.rev, + collection: commit.collection, + rkey: commit.rkey, + cid: commit.cid, + record: commit.record, + }, + }; + } + if (event.kind === "identity") { + return { ...base, kind: "identity", identity: event.identity }; + } + if (event.kind === "account") { + return { ...base, kind: "account", account: event.account }; + } + return { ...base, kind: "sync", sync: event.sync }; +} + +export interface JetstreamLiveSubscriptionOptions { + /** One pinned Jetstream v2 service. Seq cursors are instance-local. */ + url: string; + cursor?: number; + wantedCollections?: string[]; + signal?: AbortSignal; + /** Override fetch or the live transport for tests/custom runtimes. */ + fetchImpl?: typeof fetch; + liveTransport?: LiveTransport; + onConnectionOpen?: () => void; + onConnectionClose?: (event: { code?: number; reason?: string }) => void; + onConnectionError?: (event: { error: unknown }) => void; + onInfo?: (info: { name: string; message?: string }) => void; +} + +/** Thin live-only adapter over the official v2 client. It deliberately does not + * use snapshot/replay yet; Contrail's existing PDS/Alluvium paths still own + * historical acquisition. */ +export class JetstreamLiveSubscription + implements AsyncIterable +{ + readonly #options: JetstreamLiveSubscriptionOptions; + readonly #wireCursor: number; + cursor: number; + + constructor(options: JetstreamLiveSubscriptionOptions) { + this.#options = options; + const initial = + options.cursor ?? + Math.max(0, Date.now() * 1_000 - JETSTREAM_V1_BRIDGE_OVERLAP_US); + if (!Number.isSafeInteger(initial) || initial < 0) { + throw new TypeError("Jetstream cursor must be a non-negative safe integer"); + } + this.cursor = initial; + this.#wireCursor = isJetstreamTimestampCursor(initial) + ? Math.max(0, initial - JETSTREAM_V1_BRIDGE_OVERLAP_US) + : initial; + } + + async *[Symbol.asyncIterator](): AsyncIterator { + const options = this.#options; + const client = new Jetstream(options.url); + const cursor = { load: async () => this.#wireCursor, async save() {} }; + const transport = + options.liveTransport ?? + websocketTransport({ + // Browser WebSockets hide HTTP handshake status. Let the enclosing + // ingestion loop reconnect from its durable cursor instead of allowing + // ws-client to retry an unclassifiable stale cursor forever. + shouldReconnect: false, + onConnect: () => options.onConnectionOpen?.(), + onClose: (detail) => + options.onConnectionClose?.({ + code: detail.code, + reason: detail.reason, + }), + onError: (error) => options.onConnectionError?.({ error }), + }); + + const liveOptions: LiveOpts & { raw: true } = { + raw: true, + cursor, + collections: options.wantedCollections as CollectionFilter[] | undefined, + // Preserve the legacy live projection surface for now. Account and sync + // semantics remain a separate Contrail feature rather than being silently + // claimed by this transport migration. + kinds: ["commit", "identity"], + signal: options.signal, + liveTransport: transport, + onError: (error) => options.onConnectionError?.({ error }), + onInfo: (info) => { + if (info.name === "OutdatedCursor") { + throw new JetstreamLiveHistoryExpiredError( + `Jetstream timestamp cursor is older than the live retention window${info.message ? `: ${info.message}` : ""}`, + ); + } + options.onInfo?.(info); + }, + }; + let preflightComplete = false; + try { + if (!isJetstreamTimestampCursor(this.#wireCursor)) { + await assertSeqCursorAvailable({ + service: options.url, + cursor: this.#wireCursor, + fetchImpl: options.fetchImpl ?? fetch, + signal: options.signal, + }); + } + preflightComplete = true; + for await (const event of client.live(liveOptions)) { + this.cursor = advanceJetstreamCursor(this.cursor, event.seq); + yield normalizeLiveEvent(event); + } + } catch (error) { + if (error instanceof JetstreamLiveHistoryExpiredError) throw error; + // Close the race where a cursor crosses the retention floor between the + // initial HTTP probe and a browser WebSocket handshake. A second probe + // can promote the otherwise opaque SocketError to the fatal typed error. + if ( + preflightComplete && + !isJetstreamTimestampCursor(this.cursor) && + !options.signal?.aborted + ) { + try { + await assertSeqCursorAvailable({ + service: options.url, + cursor: this.cursor, + fetchImpl: options.fetchImpl ?? fetch, + signal: options.signal, + }); + } catch (probeError) { + if (probeError instanceof JetstreamLiveHistoryExpiredError) { + throw probeError; + } + } + } + throw error; + } + } +} diff --git a/packages/contrail/src/core/jetstream-source.ts b/packages/contrail/src/core/jetstream-source.ts index 3fbadda..30ff0e0 100644 --- a/packages/contrail/src/core/jetstream-source.ts +++ b/packages/contrail/src/core/jetstream-source.ts @@ -141,9 +141,11 @@ function mutationFromEvent( }; } -/** Ordered replay adapter for Jetstream's microsecond cursor protocol. +/** Legacy timestamp-coordinate adapter retained for existing PDS/Alluvium + * bootstrap orchestration. Runtime scheduled and persistent ingestion use the + * v2 seq adapter in `jetstream-live.ts`; archive-backed v2 bootstrap is deferred. * - * Jetstream has no separate head endpoint. Marks therefore wait for a real + * The legacy stream has no separate head endpoint. Marks therefore wait for a real * event from the same filtered stream, and replay waits for an event strictly * beyond that mark. Quiet application collections can add a busy watermark * collection without projecting its records. This avoids claiming catch-up diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index d0f3c07..6b22083 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -1,17 +1,21 @@ -import { JetstreamSubscription } from "@atcute/jetstream"; +import { + JetstreamLiveSubscription, + advanceJetstreamCursor, + isJetstreamTimestampCursor, +} from "./jetstream-live"; import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; import { DEFAULT_JETSTREAMS, getCollectionNsids, getDependentNsids, - jetstreamUrlOption, + jetstreamService, buildFeedTargetCaps, getFeedMutatingNsids, optimizeEnabled, optimizeIntervalMs, optimizeAnalysisLimit, } from "./types"; -import { initSchema, getLastCursor, loadKnownActorDids, saveCursor, saveCursorStatement, getCursorObservations, saveCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; +import { assertJetstreamServiceCompatibility, assertServingSourceCompatibility, initSchema, getLastCursor, loadKnownActorDids, saveJetstreamCursor, saveJetstreamCursorStatement, saveJetstreamCursorObservationStatements, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; import { createIngestEvent, ingestRecords, @@ -292,8 +296,8 @@ function nextWithDeadline( ): Promise | typeof INGEST_TIMEOUT> { let timer: ReturnType; const next = iterator.next(); - // The transport currently has no structural cancellation seam. If the timer - // wins, swallow a later rejection while the iterator is closed best-effort. + // If the timer wins, cancellation is triggered by ingestEvents' cleanup. + // Swallow that later abort rejection because this race has already settled. next.catch(() => {}); const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve(INGEST_TIMEOUT), ms); @@ -339,26 +343,6 @@ function normalizedJson(value: unknown): string { return `{${fields.join(",")}}`; } -async function observationHash(value: string): Promise { - const subtle = ( - globalThis as typeof globalThis & { - crypto?: { - subtle?: { - digest( - algorithm: string, - data: Uint8Array, - ): Promise; - }; - }; - } - ).crypto?.subtle; - if (!subtle) throw new Error("Web Crypto SHA-256 is required for ingestion"); - const digest = await subtle.digest("SHA-256", utf8.encode(value)); - return [...new Uint8Array(digest)] - .map((byte) => byte.toString(16).padStart(2, "0")) - .join(""); -} - interface JetstreamCommitObservation { /** One logical Jetstream source slot, before source ID/epoch prefixing. */ key: string; @@ -372,6 +356,7 @@ interface JetstreamCommitObservation { function jetstreamCommitObservation( event: { did: string; + seq: number; time_us: number; commit: { rev?: string; @@ -386,8 +371,10 @@ function jetstreamCommitObservation( const { commit } = event; const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; return { - key: JSON.stringify([event.time_us, uri, commit.rev ?? null]), + key: String(event.seq), fingerprint: JSON.stringify([ + uri, + commit.rev ?? null, commit.operation, commit.cid ?? null, commit.operation === "delete" ? null : normalizedJson(commit.record), @@ -402,7 +389,6 @@ export async function ingestEvents( budgetInput: ScheduledIngestOptions | ScheduledIngestBudget | number = DEFAULT_SCHEDULED_INGEST_BUDGET, knownDids?: Set, - startingCursorObservations: ReadonlySet = new Set(), ): Promise<{ events: IngestEvent[]; lastCursor: number | null; @@ -411,7 +397,6 @@ export async function ingestEvents( stats: ScheduledIngestCollectionStats; }> { const budget = resolveScheduledIngestBudget(budgetInput); - const startTimeUs = Date.now() * 1000; const deadline = Date.now() + budget.maxDrainMs; const collected: IngestEvent[] = []; @@ -448,19 +433,13 @@ export async function ingestEvents( diagnosticSamplesOmitted: 0, }; - // A durable timestamp cursor is resumed one microsecond earlier. This works - // with inclusive and exclusive timestamp APIs: all observations at the - // coarse cursor are replayed, while persisted hashes suppress only the exact - // items already accounted for there. It also protects a captured empty-start - // cursor from a later item that happens to share its microsecond. - const requestedCursor = - cursor === null ? null : Math.max(0, cursor - 1); - const subscription = new JetstreamSubscription({ - // A single-instance config is handed over as a string so @atcute skips its - // array-only first-connect cursor rollback (see jetstreamUrlOption). - url: jetstreamUrlOption(urls), + const requestedCursor = cursor; + const subscriptionAbort = new AbortController(); + const subscription = new JetstreamLiveSubscription({ + url: jetstreamService(urls), wantedCollections: collections, ...(requestedCursor !== null ? { cursor: requestedCursor } : {}), + signal: subscriptionAbort.signal, onConnectionOpen() { stats.connections++; }, @@ -473,65 +452,38 @@ export async function ingestEvents( }, }); - // Capture Atcute's constructor cursor before iteration can buffer frames and - // move it ahead. With no durable cursor this is the subscription's effective - // lower bound and must be persisted even when the first drain stays empty. + // Capture the adapter's effective lower bound before iteration can buffer + // frames and move it ahead. With no durable cursor this is a timestamp-domain + // bridge and must be persisted even when the first drain stays empty. const effectiveStartCursor = cursor ?? subscription.cursor ?? null; stats.safeEndingCursor = effectiveStartCursor; - let boundaryCursor = effectiveStartCursor; - let boundaryObservations = new Set(startingCursorObservations); - // Only hashes newly accounted for this cycle are returned for insertion. The - // database unions them with existing boundary rows, keeping each DB batch - // bounded even if many cycles share one coarse cursor. - let cursorObservations = new Set(); - const singleEndpoint = urls.length === 1; + // V2 seqs identify individual events, so there are no equal-cursor siblings + // to retain across cycles. Returning an empty set also retires any legacy + // timestamp-domain observation hashes when the cursor is next committed. + const cursorObservations = new Set(); const iterator = subscription[Symbol.asyncIterator](); type Ev = typeof subscription extends AsyncIterable ? V : never; - const accountSourceItem = async ( - event: Ev, - identity: string, - ): Promise => { - // The scheduled path requires one pinned endpoint. Its one-microsecond - // boundary overlap is known-complete below the durable cursor and must not - // consume candidate/byte budgets on every restart. + const accountSourceItem = (event: Ev): boolean => { + // The v2 wire is inclusive. The official client normally removes this + // overlap itself; keep the boundary guard so custom/test transports cannot + // make the already-committed seq consume a scheduled work budget. if ( - singleEndpoint && effectiveStartCursor !== null && - event.time_us < effectiveStartCursor + !isJetstreamTimestampCursor(effectiveStartCursor) && + event.seq <= effectiveStartCursor ) { stats.resumeOverlapDropped++; return true; } - const hash = await observationHash( - JSON.stringify([sourceId, sourceEpoch, identity]), - ); - if (boundaryCursor === null || event.time_us > boundaryCursor) { - boundaryCursor = event.time_us; - boundaryObservations = new Set([hash]); - cursorObservations = new Set([hash]); - } else if (event.time_us === boundaryCursor) { - const alreadyAccounted = boundaryObservations.has(hash); - boundaryObservations.add(hash); - if (!alreadyAccounted) cursorObservations.add(hash); - } - - if ( - effectiveStartCursor !== null && - event.time_us === effectiveStartCursor && - startingCursorObservations.has(hash) - ) { - stats.cursorBoundaryDuplicatesDropped++; - return true; - } return false; }; - // Fully account for one yielded item before the loop considers any stop - // threshold. Exact boundary observations are suppressed before evolving - // source-scope policy; ordinary cheap filters still precede cycle dedupe. + // Fully account for one yielded seq before the loop considers any stop + // threshold. The resume overlap is suppressed before evolving source-scope + // policy; ordinary cheap filters still precede cycle-local dedupe. const handleEvent = async (event: Ev): Promise => { if (event.kind === "commit") { const { commit } = event; @@ -542,16 +494,7 @@ export async function ingestEvents( sourceEpoch, observation.key, ]); - if ( - await accountSourceItem( - event, - JSON.stringify([ - "commit", - observation.key, - observation.fingerprint, - ]), - ) - ) { + if (accountSourceItem(event)) { return; } @@ -580,7 +523,7 @@ export async function ingestEvents( stats.sourceInconsistencies++; addDiagnosticSample( stats, - `source slot changed payload: ${observation.uri} time_us=${event.time_us}`, + `source slot changed payload: ${observation.uri} seq=${event.seq}`, ); fingerprints.add(observation.fingerprint); } else { @@ -608,7 +551,7 @@ export async function ingestEvents( ...(sourceEpoch === null ? {} : { epoch: sourceEpoch }), time_us: event.time_us, revision: commit.rev, - cursor: String(event.time_us), + cursor: String(event.seq), }, }); collected.push(candidate); @@ -618,7 +561,8 @@ export async function ingestEvents( (candidate.record === null ? 0 : utf8.encode(candidate.record).byteLength); } else if (event.kind === "identity") { stats.identityObservations++; - if (await accountSourceItem(event, normalizedJson(event))) return; + if (accountSourceItem(event)) return; + if (event.identity.handle === undefined) return; if ( identityUpdates.has(event.did) || identityUpdates.size < budget.maxIdentityUpdates @@ -631,75 +575,68 @@ export async function ingestEvents( stats.identityUpdatesOmitted++; } } else { - await accountSourceItem(event, normalizedJson(event)); + accountSourceItem(event); } }; - for (;;) { - // Check the deadline before requesting another item. A hot iterator must not - // get one extra next() after any threshold has been reached. - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) { - stats.stopReason = "drain-time"; - break; - } - - const step = await nextWithDeadline(iterator, remainingMs); - if (step === INGEST_TIMEOUT) { - stats.stopReason = "drain-time"; - break; - } - if (step.done) { - stats.stopReason = "idle"; - break; - } - const event = step.value; - stats.observedSourceItems++; + try { + for (;;) { + // Check the deadline before requesting another item. A hot iterator must + // not get one extra next() after any threshold has been reached. + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + stats.stopReason = "drain-time"; + break; + } - await handleEvent(event); - // handleEvent either retained, filtered, deduplicated, or deliberately - // handled the item. Only now is its cursor safe to checkpoint. Keep the - // representable checkpoint monotonic across deliberate overlap replay. - stats.lastAccountedCursor = Math.max( - stats.lastAccountedCursor ?? event.time_us, - event.time_us, - ); + const step = await nextWithDeadline(iterator, remainingMs); + if (step === INGEST_TIMEOUT) { + stats.stopReason = "drain-time"; + break; + } + if (step.done) { + stats.stopReason = "idle"; + break; + } + const event = step.value; + stats.observedSourceItems++; + + await handleEvent(event); + // handleEvent either retained, filtered, deduplicated, or deliberately + // handled the item. Only now is its cursor safe to checkpoint. Keep the + // representable checkpoint monotonic across deliberate overlap replay. + stats.lastAccountedCursor = advanceJetstreamCursor( + stats.lastAccountedCursor, + event.seq, + ); - if (stats.retainedCandidates >= budget.maxCandidates) { - stats.stopReason = "count"; - break; - } - if (stats.serializedCandidateBytes >= budget.maxSerializedBytes) { - stats.stopReason = "bytes"; - break; - } - if (event.time_us >= startTimeUs) { - stats.stopReason = "head"; - break; + if (stats.retainedCandidates >= budget.maxCandidates) { + stats.stopReason = "count"; + break; + } + if (stats.serializedCandidateBytes >= budget.maxSerializedBytes) { + stats.stopReason = "bytes"; + break; + } } + } finally { + // Abort first so a pending async-generator pull and the transport's + // reconnect loop settle before return() is queued behind that pull. + subscriptionAbort.abort(); + Promise.resolve(iterator.return?.()).catch(() => {}); } - // Close the subscription's socket, but fire-and-forget: awaiting the - // iterator's return on a quiet stream could itself block (the hang we fix). - Promise.resolve(iterator.return?.()).catch(() => {}); - - // Never read Atcute's cursor again here: it may have moved when a frame was - // buffered but not yielded. The constructor cursor captured above and the - // maximum fully-accounted yielded cursor are the only safe positions. + // Never read the transport cursor again here: it may have moved when a frame + // was buffered but not yielded. The constructor cursor captured above and the + // maximum fully-accounted yielded seq are the only safe positions. const lastCursor = - effectiveStartCursor === null - ? stats.lastAccountedCursor - : Math.max( + stats.lastAccountedCursor === null + ? effectiveStartCursor + : advanceJetstreamCursor( effectiveStartCursor, - stats.lastAccountedCursor ?? effectiveStartCursor, + stats.lastAccountedCursor, ); stats.safeEndingCursor = lastCursor; - if (lastCursor !== boundaryCursor) { - // This occurs only for an older deliberate overlap. Leave the durable - // boundary unchanged rather than attaching old hashes to a newer cursor. - cursorObservations = new Set(); - } - return { events: collected, lastCursor, @@ -767,12 +704,7 @@ export async function runIngestCycle( const log = getLogger(config); const budget = resolveScheduledIngestBudget(budgetInput); const scheduledUrls = config.jetstreams ?? DEFAULT_JETSTREAMS; - if (scheduledUrls.length !== 1) { - throw new TypeError( - "scheduled ingestion requires exactly one pinned Jetstream endpoint; " + - "use runPersistent() for a failover pool", - ); - } + const service = jetstreamService(scheduledUrls); const wallStartedAt = Date.now(); const finishCpuUsage = runtimeCpuUsage(); const s = state ?? createIngestState(); @@ -781,10 +713,10 @@ export async function runIngestCycle( await initSchema(db, config); s.schemaInitialized = true; } + await assertServingSourceCompatibility(db, config.orderedSource); + await assertJetstreamServiceCompatibility(db, service); const cursor = await getLastCursor(db); - const startingCursorObservations = - cursor === null ? new Set() : await getCursorObservations(db, cursor); // Load known DIDs for filtering dependent collections const dependentCollections = getDependentNsids(config); @@ -810,7 +742,6 @@ export async function runIngestCycle( cursor, budget, knownDids, - startingCursorObservations, ); const accepted: IngestEvent[] = []; @@ -834,7 +765,7 @@ export async function runIngestCycle( trailingStatements: isFinalBatch && lastCursor !== null ? [ - saveCursorStatement(db, lastCursor), + saveJetstreamCursorStatement(db, lastCursor), ...(config.orderedSource ? [ saveOrderedSourcePositionStatement( @@ -844,7 +775,7 @@ export async function runIngestCycle( ), ] : []), - ...saveCursorObservationStatements( + ...saveJetstreamCursorObservationStatements( db, lastCursor, cursorObservations, @@ -904,7 +835,7 @@ export async function runIngestCycle( // empty first drain so the next cron cannot silently start later. (stats.lastAccountedCursor !== null || cursor === null) ) { - await saveCursor( + await saveJetstreamCursor( db, lastCursor, config.orderedSource, diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index 6b19d96..6002848 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -1,14 +1,18 @@ -import type { JetstreamSubscription } from "@atcute/jetstream"; +import { + JetstreamLiveHistoryExpiredError, + JetstreamLiveSubscription, + type JetstreamLiveEvent, +} from "./jetstream-live"; import type { ContrailConfig, IngestEvent, Database, Logger, ResolvedContrailConfig } from "./types"; import { getCollectionNsids, getDependentNsids, buildFeedTargetCaps, getFeedMutatingNsids, - jetstreamUrlOption, + jetstreamService, resolveConfig, } from "./types"; -import { initSchema, getLastCursor, loadKnownActorDids, saveCursorStatement, saveOrderedSourcePositionStatement } from "./db"; +import { assertJetstreamServiceCompatibility, assertServingSourceCompatibility, initSchema, getLastCursor, loadKnownActorDids, saveJetstreamCursorStatement, saveJetstreamCursorObservationStatements, saveOrderedSourcePositionStatement } from "./db"; import { createIngestEvent, ingestRecords, recordTimeUs } from "./ingest"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; @@ -25,12 +29,21 @@ import type { IngestState } from "./jetstream"; * the recycling cron isolate). */ const FEED_SWEEP_INTERVAL_MS = 10_000; +export interface PersistentJetstreamSubscription + extends AsyncIterable { + /** Effective initial source coordinate, when the transport exposes one. */ + cursor?: number | null; +} + export interface PersistentIngestOptions { batchSize?: number; flushIntervalMs?: number; signal?: AbortSignal; - /** Override subscription creation for testing */ - createSubscription?: (cursor: number | null) => JetstreamSubscription; + /** Override subscription creation for testing or a custom transport. */ + createSubscription?: ( + cursor: number | null, + signal?: AbortSignal, + ) => PersistentJetstreamSubscription; logger?: Logger; } @@ -55,12 +68,15 @@ export async function runPersistent( const flushIntervalMs = options?.flushIntervalMs ?? 5_000; const signal = options?.signal; const state = createIngestState(); + const service = jetstreamService(config.jetstreams ?? []); // Init schema once if (!state.schemaInitialized) { await initSchema(db, config); state.schemaInitialized = true; } + await assertServingSourceCompatibility(db, config.orderedSource); + await assertJetstreamServiceCompatibility(db, service); // Load known DIDs for dependent collection filtering const dependentCollections: Set = new Set(getDependentNsids(config)); @@ -88,11 +104,13 @@ export async function runPersistent( newlyKnownDids: new Set(), state, log, + service, createSubscription: options?.createSubscription, }); reconnectAttempts = 0; } catch (err) { if (signal?.aborted) break; + if (err instanceof JetstreamLiveHistoryExpiredError) throw err; log.error(`Jetstream connection error: ${err}`); const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); reconnectAttempts++; @@ -108,6 +126,7 @@ interface StreamOptions { batchSize: number; flushIntervalMs: number; signal?: AbortSignal; + service: string; collections: string[]; knownDids?: Set; /** DIDs that crossed from unknown→known during this stream's lifetime. @@ -115,7 +134,7 @@ interface StreamOptions { newlyKnownDids?: Set; state: IngestState; log: Logger; - createSubscription?: (cursor: number | null) => any; + createSubscription?: PersistentIngestOptions["createSubscription"]; } async function streamAndFlush( @@ -127,19 +146,47 @@ async function streamAndFlush( const { batchSize, flushIntervalMs, signal, collections, knownDids, state, log } = opts; const subscription = opts.createSubscription - ? opts.createSubscription(cursor) - : new (await import("@atcute/jetstream")).JetstreamSubscription({ - // Single-instance config → string, so @atcute skips its array-only - // first-connect cursor rollback (see jetstreamUrlOption). - url: jetstreamUrlOption(config.jetstreams ?? []), + ? opts.createSubscription(cursor, signal) + : new JetstreamLiveSubscription({ + url: opts.service, wantedCollections: collections, ...(cursor !== null ? { cursor } : {}), - onConnectionOpen() { log.log("Connected to Jetstream"); }, - onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, - onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, + signal, + onConnectionOpen() { log.log("Connected to Jetstream v2"); }, + onConnectionClose(event) { log.log(`Disconnected: ${event.code ?? ""} ${event.reason ?? ""}`); }, + onConnectionError(event) { log.error("Jetstream v2 error:", event.error); }, }); + // A fresh v2 subscription begins from a timestamp-domain overlap bridge. + // Persist that exact lower bound before the first pull so failed connection + // attempts cannot create progressively newer bridges and skip the outage. + const initialCursor = cursor === null ? subscription.cursor : null; + if (initialCursor !== null && initialCursor !== undefined) { + if (!Number.isSafeInteger(initialCursor) || initialCursor < 0) { + throw new TypeError( + "persistent Jetstream subscription cursor must be a non-negative safe integer", + ); + } + await db.batch([ + saveJetstreamCursorStatement(db, initialCursor), + ...(config.orderedSource + ? [ + saveOrderedSourcePositionStatement( + db, + config.orderedSource, + initialCursor, + ), + ] + : []), + ...saveJetstreamCursorObservationStatements(db, initialCursor, []), + ]); + } + const buffer: IngestEvent[] = []; + // Highest fully handled v2 seq not yet committed. This includes identity + // events, so a commit-quiet stream still advances and does not replay global + // identity traffic forever after restart. + let pendingCursor: number | null = null; // Guards against overlap between the periodic timer flush and a main-loop // batchSize-driven flush. The main loop only ever awaits flush() sequentially, // but the setInterval callback is a second entry point on another tick. @@ -150,10 +197,13 @@ async function streamAndFlush( flushing = true; try { + const checkpoint = pendingCursor; + if (checkpoint !== null) pendingCursor = null; if (buffer.length > 0) { const batch = buffer.splice(0); - const lastTimeUs = Math.max( - ...batch.map((event) => event.source?.time_us ?? event.time_us), + const lastCursor = Math.max( + checkpoint ?? 0, + ...batch.map((event) => Number(event.source?.cursor)), ); let ingestResult: Awaited>; try { @@ -161,22 +211,24 @@ async function streamAndFlush( phase: "live", knownDids, trailingStatements: [ - saveCursorStatement(db, lastTimeUs), + saveJetstreamCursorStatement(db, lastCursor), ...(config.orderedSource ? [ saveOrderedSourcePositionStatement( db, config.orderedSource, - lastTimeUs, + lastCursor, ), ] : []), + ...saveJetstreamCursorObservationStatements(db, lastCursor, []), ], }); } catch (error) { // The cursor transaction failed, so keep this exact batch at the front // for the timer's next attempt rather than losing it in memory. buffer.unshift(...batch); + pendingCursor = Math.max(pendingCursor ?? 0, lastCursor); throw error; } const { accepted, discoveredDids } = ingestResult; @@ -224,8 +276,27 @@ async function streamAndFlush( await maybeOptimize(db, config, log); log.log( - `Flushed ${accepted.length}/${batch.length} records. Cursor: ${lastTimeUs}`, + `Flushed ${accepted.length}/${batch.length} records. Cursor: ${lastCursor}`, ); + } else if (checkpoint !== null) { + try { + await db.batch([ + saveJetstreamCursorStatement(db, checkpoint), + ...(config.orderedSource + ? [ + saveOrderedSourcePositionStatement( + db, + config.orderedSource, + checkpoint, + ), + ] + : []), + ...saveJetstreamCursorObservationStatements(db, checkpoint, []), + ]); + } catch (error) { + pendingCursor = Math.max(pendingCursor ?? 0, checkpoint); + throw error; + } } // Bounded, cursored feed prune (see sweepFeedItems / runFeedPruneSlice). @@ -328,12 +399,15 @@ async function streamAndFlush( : {}), time_us: event.time_us, revision: commit.rev, - cursor: String(event.time_us), + cursor: String(event.seq), }, }), ); - } else if (event.kind === "identity") { + } else if ( + event.kind === "identity" && + event.identity.handle !== undefined + ) { try { await applyIdentityEvent(db, event.did, event.identity.handle); } catch (err) { @@ -341,6 +415,8 @@ async function streamAndFlush( } } + pendingCursor = Math.max(pendingCursor ?? 0, event.seq); + if (buffer.length >= batchSize) { await flush(); } diff --git a/packages/contrail/src/core/router/diagnostics.ts b/packages/contrail/src/core/router/diagnostics.ts index a080011..2ae8e07 100644 --- a/packages/contrail/src/core/router/diagnostics.ts +++ b/packages/contrail/src/core/router/diagnostics.ts @@ -4,6 +4,7 @@ import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../ import { getLastCursor, getServingSourcePosition } from "../db"; import { getBackfillStatus } from "../status"; import { getRequiredChangeConsumerReadiness } from "../changes"; +import { isJetstreamTimestampCursor } from "../jetstream-live"; export interface CursorStatus { cursor: number | null; @@ -23,6 +24,9 @@ export async function getCursorStatus(db: Database): Promise { return { cursor: null, date: null, seconds_ago: null }; } + if (!isJetstreamTimestampCursor(cursor)) { + return { cursor, date: null, seconds_ago: null }; + } const dateMs = Math.floor(cursor / 1000); return { cursor, @@ -80,13 +84,9 @@ export function registerCursorRoute( app.get(`/xrpc/${ns}.getCursor`, async (c) => { if (!config.orderedSource) { - const legacy = await getCursorStatus(db); - if (legacy.cursor === null) return c.json({}); - return c.json({ - time_us: legacy.cursor, - date: legacy.date, - seconds_ago: legacy.seconds_ago, - }); + const current = await getCursorStatus(db); + if (current.cursor === null) return c.json({}); + return c.json({ cursor: current.cursor }); } const current = await getServingSourcePosition(db); diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index b42bcb2..e1fcef9 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -199,11 +199,46 @@ export function deriveShortName(nsid: string): string { } export const DEFAULT_JETSTREAMS = [ - "wss://jetstream1.us-east.bsky.network", + "https://jetstream.us-east.bsky.network", ]; +/** Canonical source identity used for both the v2 client and its durable + * service binding. The official client addresses XRPC at the origin, so path, + * query, credentials, and fragments are rejected instead of being discarded + * ambiguously. WebSocket and HTTP spellings of the same origin are equivalent. */ +export function normalizeJetstreamService(service: string): string { + const url = new URL(service); + if (url.protocol === "wss:") url.protocol = "https:"; + if (url.protocol === "ws:") url.protocol = "http:"; + if (url.protocol !== "https:" && url.protocol !== "http:") { + throw new TypeError("Jetstream v2 service must use HTTP(S) or WS(S)"); + } + if ( + url.username || + url.password || + (url.pathname !== "" && url.pathname !== "/") || + url.search || + url.hash + ) { + throw new TypeError("Jetstream v2 service must be an origin URL"); + } + return url.origin; +} + +/** Return the one pinned Jetstream v2 service used by live ingestion. + * Sequence cursors are instance-local and cannot fail over between services. */ +export function jetstreamService(jetstreams: string[]): string { + if (jetstreams.length !== 1 || !jetstreams[0]) { + throw new TypeError( + "Jetstream v2 ingestion requires exactly one pinned service", + ); + } + return normalizeJetstreamService(jetstreams[0]); +} + /** - * Shape a configured jetstream list for `@atcute/jetstream`'s `url` option. + * Shape a configured jetstream list for the legacy `@atcute/jetstream` adapter + * still used by v1 archive/bootstrap integrations. * * @atcute distinguishes a string url (one fixed instance) from an array url (a * pool it picks from at random each connect). For an array it seeds @@ -294,14 +329,11 @@ export interface ContrailConfig { validation?: IngestValidationConfig; profiles?: (string | ProfileConfig)[]; relays?: string[]; - /** Jetstream endpoints to ingest from (defaults to {@link DEFAULT_JETSTREAMS}). - * Prefer a single endpoint: one instance has no clock skew, so `@atcute` takes - * no cursor rollback (see {@link jetstreamUrlOption}) — important for the cron - * model, which rebuilds the subscription every cycle. Scheduled ingestion - * requires exactly one pinned endpoint because a pooled connection rolls its - * timestamp cursor back on every new cycle and cannot make bounded progress - * through a dense overlap. Use 2+ only with `runPersistent()`, where the - * connection and its failover cursor remain long-lived. */ + /** Jetstream v2 service used for live ingestion (defaults to + * {@link DEFAULT_JETSTREAMS}). Exactly one service is required: v2 sequence + * cursors are instance-local and cannot fail over between servers. The array + * shape is retained for configuration compatibility and will be simplified in + * a later API cleanup. */ jetstreams?: string[]; /** Identity of the ordered source consumed by live ingestion. Its opaque * cursor is persisted atomically with projected mutations and may be exposed diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index cd07755..1553ecc 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -25,6 +25,10 @@ export * from "./core/generations"; export * from "./core/pds-snapshot"; export * from "./core/jetstream-source"; export * from "./core/jetstream"; +export { + JetstreamLiveHistoryExpiredError, +} from "./core/jetstream-live"; +export type { JetstreamLiveEvent } from "./core/jetstream-live"; export * from "./core/persistent"; export * from "./core/backfill"; export * from "./core/status"; diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index e525263..207a439 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -696,10 +696,8 @@ export function extractXrpcMethods( } function cursorLexicon(config: ContrailConfig): object { - const legacyProperties = { - time_us: { type: "integer" }, - date: { type: "string" }, - seconds_ago: { type: "integer" }, + const unscopedProperties = { + cursor: { type: "integer" }, }; if (!config.orderedSource) { return { @@ -708,10 +706,10 @@ function cursorLexicon(config: ContrailConfig): object { defs: { main: { type: "query", - description: "Get the current ingestion observation time", + description: "Get the current Jetstream live cursor", output: { encoding: "application/json", - schema: { type: "object", properties: legacyProperties }, + schema: { type: "object", properties: unscopedProperties }, }, }, }, diff --git a/packages/contrail/src/workers/backfill.ts b/packages/contrail/src/workers/backfill.ts index 5d2cd40..f2883b4 100644 --- a/packages/contrail/src/workers/backfill.ts +++ b/packages/contrail/src/workers/backfill.ts @@ -106,11 +106,18 @@ export async function backfillAll( ); } +export const DEFAULT_ALLUVIUM_SOURCE_URL = + "wss://jetstream1.us-east.bsky.network"; + export interface AlluviumBootstrapOptions { /** Alluvium HTTP origin. */ endpoint: string | URL; /** Logical source ID advertised by the selected manifests. */ sourceId: string; + /** Exact legacy Jetstream v1 URL advertised by the manifests. This is + * independent of `config.jetstreams`, which selects the v2 live service. + * Default: {@link DEFAULT_ALLUVIUM_SOURCE_URL}. */ + sourceUrl?: string; /** Operator-owned continuity epoch. Alluvium protocol v1 does not publish it. */ sourceEpoch: string; /** Explicitly accept manifests that report historical omissions. */ @@ -171,7 +178,6 @@ async function runAlluviumBootstrap( db: Database, opts: AlluviumBootstrapOptions, ): Promise { - const jetstreams = contrail.config.jetstreams ?? []; if ( contrail.config.orderedSource && (contrail.config.orderedSource.source !== opts.sourceId || @@ -181,16 +187,14 @@ async function runAlluviumBootstrap( "Alluvium source ID/epoch must match the configured orderedSource", ); } - if (jetstreams.length !== 1) { - throw new Error( - "Alluvium bootstrap requires exactly one configured Jetstream endpoint " + - "so scheduled ingestion resumes against the manifest's exact source.", - ); + const sourceUrl = opts.sourceUrl ?? DEFAULT_ALLUVIUM_SOURCE_URL; + if (!sourceUrl.trim()) { + throw new TypeError("Alluvium source URL must not be empty"); } const source = { id: opts.sourceId, epoch: opts.sourceEpoch, - url: jetstreams[0]!, + url: sourceUrl, }; const target = new DatabaseBootstrapTarget(db, contrail.config, { deferDerivedProjections: true, diff --git a/packages/contrail/tests/ingest-hang.test.ts b/packages/contrail/tests/ingest-hang.test.ts index 1dd0782..4201f69 100644 --- a/packages/contrail/tests/ingest-hang.test.ts +++ b/packages/contrail/tests/ingest-hang.test.ts @@ -6,25 +6,37 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; const jetstream = vi.hoisted(() => ({ script: null as | null - | ((self: { cursor: number | null }) => AsyncGenerator), + | ((self: { + cursor: number | null; + signal?: AbortSignal; + }) => AsyncGenerator), abort: false, + lastSignal: null as AbortSignal | null, + pendingPullSettled: false, })); // Replace the real WebSocket-backed subscription with one driven by the test's // `script`. `self.cursor` mirrors the real subscription's progress cursor, // which ingestEvents reads back as `lastCursor`. -vi.mock("@atcute/jetstream", () => { +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { cursor: number | null = null; - constructor(opts: { cursor?: number }) { + readonly signal?: AbortSignal; + constructor(opts: { cursor?: number; signal?: AbortSignal }) { this.cursor = typeof opts?.cursor === "number" ? opts.cursor : null; + this.signal = opts.signal; + jetstream.lastSignal = opts.signal ?? null; } async *[Symbol.asyncIterator]() { if (!jetstream.script) throw new Error("test did not set jetstream.script"); - yield* jetstream.script(this); + for await (const value of jetstream.script(this)) { + const event = value as { seq?: number; time_us: number }; + yield { ...event, seq: event.seq ?? event.time_us }; + } } } - return { JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); import { @@ -126,15 +138,22 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { beforeEach(() => { jetstream.script = null; jetstream.abort = false; + jetstream.lastSignal = null; + jetstream.pendingPullSettled = false; }); it("returns within the safety timeout when the stream replays history then goes quiet", async () => { - jetstream.script = async function* (self: { cursor: number | null }) { + jetstream.script = async function* (self) { // One historical commit (time_us in the past, so never "caught up"). self.cursor = 1_000_000; yield commitEvent("did:plc:author", "community.lexicon.calendar.event", 1_000_000, "evt1"); - // Quiet stream: no further events ever arrive. - await new Promise(() => {}); + // Quiet stream: no further events arrive, but transport cancellation must + // settle the pending pull rather than leave a reconnect loop behind. + await new Promise((resolve) => { + if (self.signal?.aborted) return resolve(); + self.signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + jetstream.pendingPullSettled = true; }; const result = await withTimeout( @@ -156,6 +175,8 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { // The replayed cursor must come back so the caller can persist it. expect(result.lastCursor).toBe(1_000_000); expect(result.stats.stopReason).toBe("drain-time"); + expect(jetstream.lastSignal?.aborted).toBe(true); + await vi.waitFor(() => expect(jetstream.pendingPullSettled).toBe(true)); }); it("never checkpoints past the last yielded event when the subscription buffers ahead", async () => { @@ -225,13 +246,16 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { 1_000_000, "primary", ); - self.cursor = liveUs; - yield commitEvent( - "did:plc:new", - "app.bsky.graph.follow", - liveUs, - "dependent", - ); + self.cursor = 1_000_001; + yield { + ...commitEvent( + "did:plc:new", + "app.bsky.graph.follow", + liveUs, + "dependent", + ), + seq: 1_000_001, + }; }; const result = await ingestEvents( @@ -246,56 +270,48 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { ]); }); - it("breaks via 'caught up to present' and returns the batch + cursor when a kept event reaches the live edge", async () => { - // `live` is >= ingestEvents' startTimeUs (captured at the call below), so the - // 5s safety timeout is irrelevant — the exit must be the caught-up break. - const liveUs = Date.now() * 1000 + 5_000_000; + it("does not infer v2 head progress from the display timestamp", async () => { + // V2 `time` may be operator-imported and is not a resume coordinate. Even a + // future-looking display time must not terminate collection before the next + // seq has been accounted for. + const displayUs = Date.now() * 1000 + 5_000_000; jetstream.script = async function* (self) { self.cursor = 1_000_000; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", 1_000_000, "hist"); - self.cursor = liveUs; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", liveUs, "live"); - // Must NOT be reached: the caught-up break fires on `live` before this. - self.cursor = liveUs + 1_000; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", liveUs + 1_000, "after"); + yield { ...commitEvent("did:plc:author", "community.lexicon.calendar.event", displayUs, "first"), seq: 1_000_000 }; + self.cursor = 1_000_001; + yield { ...commitEvent("did:plc:author", "community.lexicon.calendar.event", displayUs, "second"), seq: 1_000_001 }; await new Promise(() => {}); }; const result = await withTimeout( - ingestEvents(discoverableConfig(), 999_999, 5_000), + ingestEvents(discoverableConfig(), 999_999, 150), 2_000, - "caught-up kept", + "display time is not head", ); - expect(result.events.map((e) => e.rkey)).toEqual(["hist", "live"]); - expect(result.lastCursor).toBe(liveUs); - expect(result.stats.stopReason).toBe("head"); + expect(result.events.map((e) => e.rkey)).toEqual(["first", "second"]); + expect(result.lastCursor).toBe(1_000_001); + expect(result.stats.stopReason).toBe("drain-time"); }); - it("caught-up break fires even on a filtered live event, deferring a following kept event to the next cycle", async () => { - // Behavior change vs the pre-fix loop: filtering now uses early `return`, so - // the caught-up check runs after a filtered event too. A filtered event at - // the live edge breaks the loop BEFORE the kept event that follows it. - // Pre-fix (`continue`) would have skipped the check, collected `evt`, and - // broken on it with cursor liveUs+1000. - const liveUs = Date.now() * 1000 + 5_000_000; - const knownDids = new Set(); // empty -> the follow is filtered (unknown DID) + it("accounts a filtered seq without deferring the following kept seq", async () => { + const knownDids = new Set(); jetstream.script = async function* (self) { - self.cursor = liveUs; - yield commitEvent("did:plc:stranger", "app.bsky.graph.follow", liveUs, "f1"); - self.cursor = liveUs + 1_000; - yield commitEvent("did:plc:author", "community.lexicon.calendar.event", liveUs + 1_000, "evt"); + self.cursor = 1_000_000; + yield commitEvent("did:plc:stranger", "app.bsky.graph.follow", 1_000_000, "f1"); + self.cursor = 1_000_001; + yield commitEvent("did:plc:author", "community.lexicon.calendar.event", 1_000_001, "evt"); await new Promise(() => {}); }; const result = await withTimeout( - ingestEvents(dependentConfig(), 999_999, 5_000, knownDids), + ingestEvents(dependentConfig(), 999_999, 150, knownDids), 2_000, - "caught-up filtered", + "filtered then kept", ); - expect(result.events).toHaveLength(0); // kept `evt` deferred, not collected this cycle - expect(result.lastCursor).toBe(liveUs); // broke at the filtered event, before `evt` + expect(result.events.map((event) => event.rkey)).toEqual(["evt"]); + expect(result.lastCursor).toBe(1_000_001); }); it("collects every event when they flow fast but within the safety timeout (the next()/timeout race drops nothing)", async () => { @@ -613,32 +629,15 @@ describe("ingestEvents — bounded by the safety timeout (om-dua7)", () => { expect(result.lastCursor).toBe(4_000_001); }); - it("never regresses the safe cursor when a pooled transport replays older overlap", async () => { - const startingCursor = 6_000_000; - jetstream.script = async function* (self) { - self.cursor = startingCursor - 500; - yield commitEvent( - "did:plc:author", - "community.lexicon.calendar.event", - startingCursor - 500, - "rolled-back", - ); - }; + it("rejects pooled v2 services because seq cursors are instance-local", async () => { const config = { ...discoverableConfig(), - jetstreams: ["wss://one.test", "wss://two.test"], + jetstreams: ["https://one.test", "https://two.test"], }; - const result = await ingestEvents( - config, - startingCursor, - budget({ maxCandidates: 1 }), - ); - - expect(result.events).toHaveLength(1); - expect(result.stats.lastAccountedCursor).toBe(startingCursor - 500); - expect(result.lastCursor).toBe(startingCursor); - expect(result.stats.safeEndingCursor).toBe(startingCursor); + await expect( + ingestEvents(config, 6_000_000, budget({ maxCandidates: 1 })), + ).rejects.toThrow("exactly one pinned service"); }); it("keeps source-inconsistency diagnostics and normal logs bounded", async () => { diff --git a/packages/contrail/tests/jetstream-single-url.test.ts b/packages/contrail/tests/jetstream-single-url.test.ts index 23cc167..62e07f2 100644 --- a/packages/contrail/tests/jetstream-single-url.test.ts +++ b/packages/contrail/tests/jetstream-single-url.test.ts @@ -1,31 +1,23 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -// Records what `url` shape ingestEvents hands to @atcute's JetstreamSubscription. -// @atcute treats a string url and an array url differently: for an ARRAY it seeds -// `#lastUsedUrl = ''`, so the very first connect satisfies its "switched instance" -// check and rolls the cursor back 10s — even when the array holds a single URL and -// nothing switched. Passing a string seeds `#lastUsedUrl = undefined`, so no -// spurious rollback (and thus no re-delivered events / DUPLICATE-in-cycle flood). const recorder = vi.hoisted(() => ({ url: undefined as unknown })); -vi.mock("@atcute/jetstream", () => { +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { - cursor: number | null = null; + cursor: number | null; constructor(opts: { url?: unknown; cursor?: number }) { - recorder.url = opts?.url; - this.cursor = typeof opts?.cursor === "number" ? opts.cursor : null; + recorder.url = opts.url; + this.cursor = opts.cursor ?? 0; } async *[Symbol.asyncIterator]() { - // Quiet stream: block so ingestEvents exits via its safety timeout. await new Promise(() => {}); } } - return { JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); -import { ingestEvents } from "../src/index"; -import { resolveConfig } from "../src/index"; -import type { ContrailConfig } from "../src/index"; +import { ingestEvents, resolveConfig, type ContrailConfig } from "../src/index"; const silentLogger = { log() {}, warn() {}, error() {} }; @@ -42,26 +34,27 @@ function configWithJetstreams(jetstreams: string[]): ContrailConfig { }; } -describe("ingestEvents — jetstream url shape (om-6zdo)", () => { +describe("Jetstream v2 service pinning", () => { beforeEach(() => { recorder.url = undefined; }); - it("passes a single configured jetstream as a string (avoids @atcute's array-only cursor rollback)", async () => { + it("passes the one configured service to the v2 live adapter", async () => { await ingestEvents( - configWithJetstreams(["wss://jetstream.example/subscribe"]), + configWithJetstreams(["https://jetstream.example"]), 999_999, - 50, + 25, ); - expect(recorder.url).toBe("wss://jetstream.example/subscribe"); + expect(recorder.url).toBe("https://jetstream.example"); }); - it("passes multiple configured jetstreams as an array (preserves failover + intentional switch rollback)", async () => { - await ingestEvents( - configWithJetstreams(["wss://a.example", "wss://b.example"]), - 999_999, - 50, - ); - expect(recorder.url).toEqual(["wss://a.example", "wss://b.example"]); + it("rejects a service pool because v2 seq cursors are instance-local", async () => { + await expect( + ingestEvents( + configWithJetstreams(["https://a.example", "https://b.example"]), + 999_999, + 25, + ), + ).rejects.toThrow("exactly one pinned service"); }); }); diff --git a/packages/contrail/tests/jetstream-v2-live.test.ts b/packages/contrail/tests/jetstream-v2-live.test.ts new file mode 100644 index 0000000..f24a821 --- /dev/null +++ b/packages/contrail/tests/jetstream-v2-live.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + JETSTREAM_V2_SEQ_THRESHOLD, + JetstreamLiveHistoryExpiredError, + JetstreamLiveSubscription, + advanceJetstreamCursor, + isJetstreamTimestampCursor, + jetstreamDatetimeToMicroseconds, +} from "../src/core/jetstream-live"; +import { + assertServingSourceCompatibility, + getLastCursor, + getServingSourcePosition, + saveCursor, +} from "../src/index"; +import { saveJetstreamCursor } from "../src/core/db/records"; +import { createTestDbWithSchema } from "./helpers"; + +describe("Jetstream v2 live cursor semantics", () => { + it("rejects a stale seq through HTTP before an opaque browser transport can retry", async () => { + let transportStarted = false; + const subscription = new JetstreamLiveSubscription({ + url: "https://jetstream.example", + cursor: 42, + fetchImpl: (async (input) => { + expect(String(input)).toContain( + "/xrpc/network.bsky.jetstream.subscribeEvents?cursor=42&kinds=commit", + ); + return new Response( + JSON.stringify({ + error: "CursorTooOld", + message: "cursor 42 below lookback floor 100", + }), + { + status: 400, + headers: { + "Access-Control-Allow-Origin": "*", + "Content-Type": "application/json", + }, + }, + ); + }) as typeof fetch, + liveTransport: { + stream() { + transportStarted = true; + return (async function* () { + await new Promise(() => {}); + })(); + }, + }, + }); + + await expect( + subscription[Symbol.asyncIterator]().next(), + ).rejects.toBeInstanceOf(JetstreamLiveHistoryExpiredError); + expect(transportStarted).toBe(false); + }); + + it("does not misclassify an unverified WebSocket 400 as expired history", async () => { + const handshakeError = new Error("Unexpected server response: 400"); + const subscription = new JetstreamLiveSubscription({ + url: "https://jetstream.example", + cursor: 42, + // Both the initial probe and the post-failure probe prove only that the + // cursor is accepted. The actual filtered WebSocket request may still be + // rejected for an unrelated InvalidRequest. + fetchImpl: (async () => + new Response("WebSocket upgrade required", { status: 426 })) as typeof fetch, + liveTransport: { + stream() { + return (async function* () { + throw handshakeError; + })(); + }, + }, + }); + + const error = await subscription[Symbol.asyncIterator]().next().catch( + (value: unknown) => value, + ); + expect(error).toBe(handshakeError); + expect(error).not.toBeInstanceOf(JetstreamLiveHistoryExpiredError); + }); + + it("preserves microsecond precision from the v2 datetime", () => { + expect( + jetstreamDatetimeToMicroseconds("2026-08-29T13:08:26.865272Z"), + ).toBe(1_788_008_906_865_272); + expect( + jetstreamDatetimeToMicroseconds("2026-08-29T15:08:26.865272+02:00"), + ).toBe(1_788_008_906_865_272); + }); + + it("moves one way from a timestamp bridge to monotonic seqs", () => { + const timestamp = 1_788_008_906_865_272; + expect(isJetstreamTimestampCursor(timestamp)).toBe(true); + expect(isJetstreamTimestampCursor(25_304_406_049)).toBe(false); + expect(advanceJetstreamCursor(timestamp, 25_304_406_049)).toBe( + 25_304_406_049, + ); + expect(advanceJetstreamCursor(25_304_406_049, 25_304_406_048)).toBe( + 25_304_406_049, + ); + expect(JETSTREAM_V2_SEQ_THRESHOLD).toBe(1_000_000_000_000_000); + }); + + it("atomically replaces a legacy timestamp and rejects stale domain rollback", async () => { + const db = await createTestDbWithSchema(); + const timestamp = 1_788_008_906_865_272; + const source = { source: "jetstream", epoch: "v2" }; + + await saveCursor(db, timestamp, { source: "jetstream", epoch: "v1" }, [ + "legacy-observation", + ]); + await assertServingSourceCompatibility(db, source); + + await saveJetstreamCursor(db, 25_304_406_049, source, ["v2-observation"]); + expect(await getLastCursor(db)).toBe(25_304_406_049); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "v2", + cursor: "25304406049", + }, + }); + + await saveJetstreamCursor(db, timestamp + 1, source, ["stale-timestamp"]); + expect(await getLastCursor(db)).toBe(25_304_406_049); + expect( + ( + await db + .prepare( + "SELECT time_us, observation FROM cursor_observations ORDER BY observation", + ) + .all() + ).results, + ).toEqual([{ time_us: 25_304_406_049, observation: "v2-observation" }]); + }); + + it("prevents a legacy writer from rolling a v2 cursor and epoch back", async () => { + const db = await createTestDbWithSchema(); + const timestamp = 1_788_008_906_865_272; + await saveJetstreamCursor(db, 42, { + source: "jetstream", + epoch: "v2-epoch", + }); + + await saveCursor( + db, + timestamp, + { source: "jetstream", epoch: "stale-v1-epoch" }, + ["stale-v1-observation"], + ); + + expect(await getLastCursor(db)).toBe(42); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "v2-epoch", + cursor: "42", + }, + }); + expect( + ( + await db + .prepare("SELECT observation FROM cursor_observations") + .all() + ).results, + ).toEqual([]); + }); + + it("still rejects an epoch mismatch after the cursor is in the seq domain", async () => { + const db = await createTestDbWithSchema(); + await saveJetstreamCursor(db, 42, { + source: "jetstream", + epoch: "first-v2-epoch", + }); + await expect( + assertServingSourceCompatibility(db, { + source: "jetstream", + epoch: "different-v2-epoch", + }), + ).rejects.toThrow("does not match durable source position"); + }); + + it("rejects an epoch change when an unscoped writer already moved the live cursor to v2", async () => { + const db = await createTestDbWithSchema(); + const timestamp = 1_788_008_906_865_272; + await saveCursor(db, timestamp, { + source: "jetstream", + epoch: "legacy-epoch", + }); + + // Ingestion can temporarily run without orderedSource. That advances the + // authoritative live cursor while leaving source_position untouched. + await saveJetstreamCursor(db, 42); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "legacy-epoch", + cursor: String(timestamp), + }, + }); + + await expect( + assertServingSourceCompatibility(db, { + source: "jetstream", + epoch: "new-epoch", + }), + ).rejects.toThrow("does not match durable source position"); + }); +}); diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts index 2203c59..c081b29 100644 --- a/packages/contrail/tests/lexicon-generation.test.ts +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -185,7 +185,7 @@ describe("Contrail Lexicon generation", () => { expect(result.generated["example.public.profile.listRecords"]).toBeUndefined(); }); - it("keeps the legacy cursor contract without an ordered source", () => { + it("publishes an unscoped v2 seq cursor without an ordered source", () => { const { root, config } = fixture(); delete config.orderedSource; const result = generateLexicons({ @@ -197,9 +197,7 @@ describe("Contrail Lexicon generation", () => { const cursor = result.generated["example.public.getCursor"] as any; expect(cursor.defs.sourcePosition).toBeUndefined(); expect(cursor.defs.main.output.schema.properties).toEqual({ - time_us: { type: "integer" }, - date: { type: "string" }, - seconds_ago: { type: "integer" }, + cursor: { type: "integer" }, }); }); diff --git a/packages/contrail/tests/network-overrides-appview.test.ts b/packages/contrail/tests/network-overrides-appview.test.ts index 31ee305..4d844d9 100644 --- a/packages/contrail/tests/network-overrides-appview.test.ts +++ b/packages/contrail/tests/network-overrides-appview.test.ts @@ -9,8 +9,8 @@ import type { ContrailConfig } from "../src/index"; // commit without opening a real WebSocket. Everything else in the appview // ingest path runs for real, so this exercises the live-ingest refresh path // end-to-end (the smoking-gun call site `refreshStaleIdentities(db, dids)`). -vi.mock("@atcute/jetstream", async (importOriginal) => { - const actual = await importOriginal(); +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { cursor: number | null = null; constructor(_opts: unknown) {} @@ -18,6 +18,7 @@ vi.mock("@atcute/jetstream", async (importOriginal) => { yield { kind: "commit", // A past time_us so the ingest loop doesn't treat it as "caught up". + seq: 1_000_000, time_us: 1_000_000, did: "did:plc:ingest", commit: { @@ -30,7 +31,7 @@ vi.mock("@atcute/jetstream", async (importOriginal) => { }; } } - return { ...actual, JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); const silentLogger = { log() {}, warn() {}, error() {} }; diff --git a/packages/contrail/tests/persistent.test.ts b/packages/contrail/tests/persistent.test.ts index 47767fd..59332dd 100644 --- a/packages/contrail/tests/persistent.test.ts +++ b/packages/contrail/tests/persistent.test.ts @@ -1,14 +1,20 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import type { ContrailConfig, Database } from "../src/index"; +import type { + ContrailConfig, + Database, + PersistentJetstreamSubscription, +} from "../src/index"; import { resolveConfig } from "../src/index"; import { createTestDb, createTestDbWithSchema, TEST_CONFIG } from "./helpers"; import { runPersistent } from "../src/index"; +import { JetstreamLiveHistoryExpiredError } from "../src/core/jetstream-live"; import { getLastCursor, getServingSourcePosition, queryRecords, } from "../src/index"; import { initSchema } from "../src/index"; +import { saveJetstreamCursor } from "../src/core/db/records"; const applyIdentityEventMock = vi.fn().mockResolvedValue(undefined); vi.mock("../src/core/identity", async (importOriginal) => { @@ -27,7 +33,14 @@ beforeEach(async () => { }); // Helper: create a mock async iterable that yields events then hangs until aborted -function mockSubscription(events: Array<{ kind: string; did: string; time_us: number; commit?: any }>) { +function mockSubscription( + events: Array<{ + kind: string; + did: string; + time_us: number; + commit?: any; + }>, +): PersistentJetstreamSubscription { let aborted = false; return { cursor: 0, @@ -37,7 +50,11 @@ function mockSubscription(events: Array<{ kind: string; did: string; time_us: nu next: async () => { if (aborted) return { value: undefined, done: true as const }; if (i < events.length) { - return { value: events[i++], done: false as const }; + const event = events[i++]!; + return { + value: { ...event, seq: event.time_us }, + done: false as const, + }; } // Hang until iterator is returned (abort) return new Promise>(() => {}); @@ -52,6 +69,73 @@ function mockSubscription(events: Array<{ kind: string; did: string; time_us: nu } describe("runPersistent", () => { + it("rejects a raw-entry ordered-source mismatch before opening the stream", async () => { + await saveJetstreamCursor(db, 42, { + source: "jetstream", + epoch: "original-epoch", + }); + let subscriptionCreated = false; + + await expect( + runPersistent( + db, + { + ...TEST_CONFIG, + orderedSource: { source: "jetstream", epoch: "changed-epoch" }, + }, + { + createSubscription: () => { + subscriptionCreated = true; + return mockSubscription([]); + }, + }, + ), + ).rejects.toThrow("does not match durable source position"); + expect(subscriptionCreated).toBe(false); + }); + + it("persists the initial timestamp bridge before its first source pull", async () => { + const bridge = 1_788_008_906_865_272; + const controller = new AbortController(); + let durableCursorDuringPull: number | null = null; + + await runPersistent(db, TEST_CONFIG, { + signal: controller.signal, + createSubscription: (cursor) => { + expect(cursor).toBeNull(); + return { + cursor: bridge, + [Symbol.asyncIterator]() { + return { + next: async () => { + durableCursorDuringPull = await getLastCursor(db); + controller.abort(); + return { value: undefined, done: true as const }; + }, + }; + }, + }; + }, + }); + + expect(durableCursorDuringPull).toBe(bridge); + expect(await getLastCursor(db)).toBe(bridge); + }); + + it("fails instead of retrying forever when live retention has expired", async () => { + const expired = new JetstreamLiveHistoryExpiredError("cursor too old"); + await expect( + runPersistent(db, TEST_CONFIG, { + createSubscription: () => + ({ + async *[Symbol.asyncIterator]() { + throw expired; + }, + }) as any, + }), + ).rejects.toBe(expired); + }); + it("flushes records and the ordered source position atomically", async () => { const config = resolveConfig({ ...TEST_CONFIG, @@ -77,7 +161,10 @@ describe("runPersistent", () => { batchSize: 50, flushIntervalMs: 60_000, // high so only batch size triggers flush signal: controller.signal, - createSubscription: () => mockSubscription(events) as any, + createSubscription: (_cursor, subscriptionSignal) => { + expect(subscriptionSignal).toBe(controller.signal); + return mockSubscription(events); + }, }); // Wait for flush to complete @@ -488,6 +575,33 @@ describe("runPersistent", () => { ); }); + it("checkpoints an identity-only v2 range", async () => { + const events = [ + { + kind: "identity" as const, + did: "did:plc:identity-only", + time_us: 4_100, + identity: { + did: "did:plc:identity-only", + handle: "identity-only.test", + time: "2026-04-01T10:00:00Z", + }, + }, + ]; + const controller = new AbortController(); + const promise = runPersistent(db, TEST_CONFIG, { + batchSize: 100, + flushIntervalMs: 50, + signal: controller.signal, + createSubscription: () => mockSubscription(events) as any, + }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(await getLastCursor(db)).toBe(4_100); + controller.abort(); + await promise; + }); + it("runs the recovery feed sweep on a fully idle stream", async () => { // Regression: the feed sweep used to sit behind flush()'s early return when // the buffer was empty, so over-cap rows (a lowered cap, a bulk import) diff --git a/packages/contrail/tests/scheduled-ingest.test.ts b/packages/contrail/tests/scheduled-ingest.test.ts index 8092753..a148058 100644 --- a/packages/contrail/tests/scheduled-ingest.test.ts +++ b/packages/contrail/tests/scheduled-ingest.test.ts @@ -5,6 +5,7 @@ const source = vi.hoisted(() => ({ requestedCursors: [] as Array, events: [] as Array<{ kind: "commit"; + seq?: number; time_us: number; did: string; commit: { @@ -18,7 +19,8 @@ const source = vi.hoisted(() => ({ }>, })); -vi.mock("@atcute/jetstream", () => { +vi.mock("../src/core/jetstream-live", async (importOriginal) => { + const actual = await importOriginal(); class MockJetstreamSubscription { cursor: number | null; private readonly start: number | null; @@ -31,16 +33,20 @@ vi.mock("@atcute/jetstream", () => { async *[Symbol.asyncIterator]() { for (const event of source.events) { - // Model the supported source's resume coordinate: only events after the - // committed cursor are delivered on the next scheduled connection. - if (this.start !== null && event.time_us <= this.start) continue; - this.cursor = event.time_us; - yield event; + const seq = (event as { seq?: number }).seq ?? event.time_us; + // V2 replay is inclusive at the server and de-duplicated by the client. + if ( + this.start !== null && + this.start < 1_000_000_000_000_000 && + seq <= this.start + ) continue; + this.cursor = seq; + yield { ...event, seq }; } } } - return { JetstreamSubscription: MockJetstreamSubscription }; + return { ...actual, JetstreamLiveSubscription: MockJetstreamSubscription }; }); import { @@ -51,13 +57,15 @@ import { type Database, type Logger, } from "../src/index"; +import { saveJetstreamCursor } from "../src/core/db/records"; import { createTestDb } from "./helpers"; const COLLECTION = "com.example.event"; -function commit(time_us: number, rkey: string) { +function commit(time_us: number, rkey: string, seq = time_us) { return { kind: "commit" as const, + seq, time_us, did: "actor", commit: { @@ -146,54 +154,26 @@ describe("bounded scheduled ingest cycles", () => { ).toEqual(["one", "two", "three"]); }); - it("resumes before a capped coarse cursor without skipping or recounting siblings", async () => { + it("uses unique v2 seqs when display timestamps are equal", async () => { const db = createTestDb(); const output = logger(); const configured = config(output.value); source.events = [ - commit(1_100_000, "same-cursor-a"), - commit(1_100_000, "same-cursor-b"), - commit(1_100_001, "after"), + commit(1_100_000, "same-time-a", 1_100_001), + commit(1_100_000, "same-time-b", 1_100_002), + commit(1_100_001, "after", 1_100_003), ]; const oneCandidate = { ...budget, maxCandidates: 1 }; - await runIngestCycle(db, configured, oneCandidate); - expect(await getLastCursor(db)).toBe(1_100_000); - expect( - (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) - .results?.map((row) => row.rkey), - ).toEqual(["same-cursor-a"]); - - await runIngestCycle(db, configured, oneCandidate); - expect(await getLastCursor(db)).toBe(1_100_000); - expect( - (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) - .results?.map((row) => row.rkey), - ).toEqual(["same-cursor-a", "same-cursor-b"]); - - await runIngestCycle(db, configured, oneCandidate); - expect(await getLastCursor(db)).toBe(1_100_001); + for (const cursor of [1_100_001, 1_100_002, 1_100_003]) { + await runIngestCycle(db, configured, oneCandidate); + expect(await getLastCursor(db)).toBe(cursor); + } expect( (await db.prepare("SELECT rkey FROM records_event ORDER BY rkey").all<{ rkey: string }>()) .results?.map((row) => row.rkey), - ).toEqual(["after", "same-cursor-a", "same-cursor-b"]); - expect(source.requestedCursors).toEqual([0, 1_099_999, 1_099_999]); - - const summaries = output.lines - .filter((line) => line.text.startsWith("[ingest] cycle summary ")) - .map((line) => - JSON.parse(line.text.slice("[ingest] cycle summary ".length)) as { - retained_candidates: number; - cursor_boundary_duplicates_dropped: number; - }, - ); - expect(summaries.map((summary) => summary.retained_candidates)).toEqual([ - 1, - 1, - 1, - ]); - expect(summaries[1]?.cursor_boundary_duplicates_dropped).toBe(1); - expect(summaries[2]?.cursor_boundary_duplicates_dropped).toBe(2); + ).toEqual(["after", "same-time-a", "same-time-b"]); + expect(source.requestedCursors).toEqual([0, 1_100_001, 1_100_002]); }); it("restores actor scope from a projected record before replaying dependent siblings", async () => { @@ -202,11 +182,11 @@ describe("bounded scheduled ingest cycles", () => { const configured = config(output.value, true); const timeUs = 1_200_000; source.events = [ - commit(timeUs, "discover-actor"), + commit(timeUs, "discover-actor", 1_200_001), { - ...commit(timeUs, "dependent-sibling"), + ...commit(timeUs, "dependent-sibling", 1_200_002), commit: { - ...commit(timeUs, "dependent-sibling").commit, + ...commit(timeUs, "dependent-sibling", 1_200_002).commit, collection: "app.bsky.graph.follow", }, }, @@ -230,25 +210,25 @@ describe("bounded scheduled ingest cycles", () => { ).toEqual({ rkey: "dependent-sibling" }); }); - it("persists Atcute's captured initial cursor after an empty first drain", async () => { + it("persists a timestamp bridge, then transitions to a v2 seq", async () => { const db = createTestDb(); const output = logger(); - source.initialCursor = 7_000_000; + const timestampCursor = 1_788_000_000_000_000; + source.initialCursor = timestampCursor; await runIngestCycle(db, config(output.value), budget); - expect(await getLastCursor(db)).toBe(7_000_000); + expect(await getLastCursor(db)).toBe(timestampCursor); const summary = output.lines.find((line) => line.text.startsWith("[ingest] cycle summary "), ); expect(summary?.text).toContain('"starting_cursor":null'); - expect(summary?.text).toContain('"safe_ending_cursor":7000000'); + expect(summary?.text).toContain(`"safe_ending_cursor":${timestampCursor}`); - // An event sharing the captured microsecond is not skipped by an exclusive - // resume API: the next cycle requests one microsecond earlier. - source.events = [commit(7_000_000, "between-empty-cycles")]; + source.events = [commit(timestampCursor, "between-empty-cycles", 7_000_000)]; await runIngestCycle(db, config(output.value), budget); - expect(source.requestedCursors).toEqual([7_000_000, 6_999_999]); + expect(source.requestedCursors).toEqual([timestampCursor, timestampCursor]); + expect(await getLastCursor(db)).toBe(7_000_000); expect( await db .prepare("SELECT rkey FROM records_event WHERE rkey = ?") @@ -257,6 +237,73 @@ describe("bounded scheduled ingest cycles", () => { ).toEqual({ rkey: "between-empty-cycles" }); }); + it("rejects a raw-entry ordered-source mismatch before opening the stream", async () => { + const db = createTestDb(); + const output = logger(); + const original = { + ...config(output.value), + orderedSource: { source: "jetstream", epoch: "original-epoch" }, + }; + await initSchema(db, original); + await saveJetstreamCursor(db, 42, original.orderedSource); + + await expect( + runIngestCycle( + db, + { + ...original, + orderedSource: { source: "jetstream", epoch: "changed-epoch" }, + }, + budget, + ), + ).rejects.toThrow("does not match durable source position"); + expect(source.requestedCursors).toHaveLength(0); + }); + + it("rejects an existing seq cursor whose originating service was never bound", async () => { + const db = createTestDb(); + const output = logger(); + const configured = config(output.value); + await initSchema(db, configured); + await saveJetstreamCursor(db, 42); + + await expect(runIngestCycle(db, configured, budget)).rejects.toThrow( + "has no pinned service identity", + ); + expect(source.requestedCursors).toHaveLength(0); + }); + + it("binds seq cursors to one normalized durable service origin", async () => { + const db = createTestDb(); + const output = logger(); + const base = config(output.value); + source.initialCursor = 0; + + await runIngestCycle( + db, + { ...base, jetstreams: ["https://pinned.example/"] }, + budget, + ); + expect(await getLastCursor(db)).toBe(0); + + // WS(S) and HTTP(S) spellings normalize to the same client origin. + await runIngestCycle( + db, + { ...base, jetstreams: ["wss://pinned.example"] }, + budget, + ); + const opened = source.requestedCursors.length; + + await expect( + runIngestCycle( + db, + { ...base, jetstreams: ["https://different.example"] }, + budget, + ), + ).rejects.toThrow("does not match durable service"); + expect(source.requestedCursors).toHaveLength(opened); + }); + it("rejects endpoint pools for scheduled ingestion before rollback can starve the cap", async () => { const db = createTestDb(); const output = logger(); @@ -266,7 +313,7 @@ describe("bounded scheduled ingest cycles", () => { }); await expect(runIngestCycle(db, configured, budget)).rejects.toThrow( - "scheduled ingestion requires exactly one pinned Jetstream endpoint", + "Jetstream v2 ingestion requires exactly one pinned service", ); expect(source.requestedCursors).toHaveLength(0); }); diff --git a/packages/contrail/tests/types.test.ts b/packages/contrail/tests/types.test.ts index c0cdec3..2b8a8ee 100644 --- a/packages/contrail/tests/types.test.ts +++ b/packages/contrail/tests/types.test.ts @@ -216,7 +216,9 @@ describe("resolveConfig", () => { it("applies default jetstreams and relays", () => { const resolved = resolveConfig({ namespace: "test", collections: {} }); - expect(resolved.jetstreams).toHaveLength(1); + expect(resolved.jetstreams).toEqual([ + "https://jetstream.us-east.bsky.network", + ]); expect(resolved.relays).toHaveLength(1); }); diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts index 49cedd6..ea7a782 100644 --- a/packages/contrail/tests/worker.test.ts +++ b/packages/contrail/tests/worker.test.ts @@ -387,7 +387,7 @@ describe("createWorker", () => { expect(response.status).toBe(404); }); - it("preserves the legacy cursor response without an ordered source", async () => { + it("returns the unscoped v2 seq without an ordered source", async () => { const config: ContrailConfig = { ...MINIMAL_CONFIG, orderedSource: undefined, @@ -402,10 +402,7 @@ describe("createWorker", () => { new Request("https://api.example.com/xrpc/com.example.getCursor"), env, ); - expect(await response.json()).toMatchObject({ - time_us: 1_234_000, - date: new Date(1234).toISOString(), - }); + expect(await response.json()).toEqual({ cursor: 1_234_000 }); }); it("keeps profiles, feeds, custom queries, and configured notify routes", async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a049815..f076074 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -118,12 +118,15 @@ importers: apps/sveltekit-cloudflare-workers: dependencies: - '@atcute/jetstream': - specifier: ^2.0.2 - version: 2.0.2(@atcute/lexicons@2.0.3)(typescript@6.0.3) '@atmo-dev/contrail': specifier: workspace:* version: link:../../packages/contrail + '@atproto/lex': + specifier: ^0.3.8 + version: 0.3.8 + '@bsky/jetstream': + specifier: ^1.0.0 + version: 1.0.0(@atproto/lex@0.3.8) '@foxui/core': specifier: ^0.9.1 version: 0.9.1(@internationalized/date@3.12.1)(@sveltejs/kit@2.70.2(@sveltejs/vite-plugin-svelte@7.2.0(svelte@5.56.8(@typescript-eslint/types@8.66.0))(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)))(svelte@5.56.8(@typescript-eslint/types@8.66.0))(typescript@6.0.3)(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)))(svelte@5.56.8(@typescript-eslint/types@8.66.0))(tailwindcss@4.3.3) @@ -281,6 +284,12 @@ importers: '@atcute/xrpc-server': specifier: ^2.0.2 version: 2.0.2(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.3)(typescript@6.0.3) + '@atproto/lex': + specifier: ^0.3.8 + version: 0.3.8 + '@bsky/jetstream': + specifier: ^1.0.0 + version: 1.0.0(@atproto/lex@0.3.8) cac: specifier: ^7.0.0 version: 7.0.0 @@ -464,10 +473,105 @@ packages: peerDependencies: '@atcute/lexicons': ^2.0.0 + '@atproto-labs/did-resolver@0.3.7': + resolution: {integrity: sha512-F3M3U5cU1QSAXX3J5auGEc5N2pgDgpirAjvyDFsytXuyWfrjWfTPd/H+DZrrED7gK+noW0cAWtxxjdX7/cTSVQ==} + engines: {node: '>=22'} + + '@atproto-labs/fetch@0.3.5': + resolution: {integrity: sha512-iDFZEoNqfL17EVKE+uNzdUD7YF8LWhEhxeBtP6x+PNuAxoXv3ip9vLmB8nNzNxFwVn++FipfDtSiPqmziv1bdA==} + engines: {node: '>=22'} + + '@atproto-labs/pipe@0.2.4': + resolution: {integrity: sha512-n67jCcrC+ouAeO10cWkpPzzLMlDi/lDCU30Us+LGqhOPhT6c4t5ASdBLQi9W3jUQtRzQBt3G9zipF+xKWNvVbw==} + engines: {node: '>=22'} + + '@atproto-labs/simple-store-memory@0.2.6': + resolution: {integrity: sha512-DD1v7MEfYF3BAcEpMTTpzfBLWLoI2HuyBhku2YpjHoqPrRrhCtE1alkxfPHjtjJVIjuU/XNU0cF/wB3+5FiKsA==} + engines: {node: '>=22'} + + '@atproto-labs/simple-store@0.5.1': + resolution: {integrity: sha512-vfvoDhu6ds6BT3Pqe+d2/LBU1WpDRtt69y6zltlfMCoucPm82m1j50/Wb6xTvv+oZ+2j0JyZVXsmXQ12SWYQJg==} + engines: {node: '>=22'} + + '@atproto/common-web@0.5.10': + resolution: {integrity: sha512-w4JUdsJ3VXt8ewkavYh5m/u0UbxDtFdCKhNZPEpJ0U1vcWIjDaSInIexteETDgD7fBJAhidIEi30MGz1kk49ug==} + engines: {node: '>=22'} + + '@atproto/common@0.8.1': + resolution: {integrity: sha512-WvgY0CDgodmTS7B1KDVFt868h1b85iQtQZeoDVI+6oOZmg1jINWUl+bZ1rwf59Tk1PqcMz5OzH6OS2J7Mb9/Iw==} + engines: {node: '>=22'} + + '@atproto/crypto@0.5.4': + resolution: {integrity: sha512-UR0BkuYNYuFtw+dA+y/oPPxzX0SWRnGJ+1Cfh/jGP1BvjRUyezK3omjpeLms5fYrXbM9vnfX+ckJFJqkBgLOdw==} + engines: {node: '>=22'} + + '@atproto/did@0.5.4': + resolution: {integrity: sha512-BlnwQ+obL+4ZA71KH/EzZ3TY+cpSxnLiUI85mjBJIQwvDF/oN2sQA18wIj9jpduviIt2b/cMtlJuzHjzBkfXvw==} + engines: {node: '>=22'} + + '@atproto/lex-builder@0.1.13': + resolution: {integrity: sha512-YHSONY88fWdMcN6Xa/Noap98DNW2m5zVZGspQWaROsX2lFx1mLf7g/XKLBcPRVDpE3OWRZ98QkkSZN3SgbOXrg==} + engines: {node: '>=22'} + + '@atproto/lex-cbor@0.1.6': + resolution: {integrity: sha512-PxLQy7jzV8u9zTGURNF8ZczLPk+ZH+WNx/2z/z7RacJbu7sWNpx26U2pC7cIkWtt4jyDaruaLhssw3w3atLotA==} + engines: {node: '>=22'} + + '@atproto/lex-client@0.3.5': + resolution: {integrity: sha512-IOYjgQ2H+pw8QhV0vMMG9PTRg+oh5In9APbGBvotNZlbF4SCZNDVd7dONFY4rU2Dg6BHSkiqbqoNUHNgTC+ITA==} + engines: {node: '>=22'} + + '@atproto/lex-data@0.1.7': + resolution: {integrity: sha512-kW/dPLqo/WgCLV+XESR4JKwV6c1rZWJGOfuPupZGTjEDAKoBbKXdaEzX9/1vKQYbZ9U3j0DS/n7OFFK7wBugyQ==} + engines: {node: '>=22'} + + '@atproto/lex-document@0.1.10': + resolution: {integrity: sha512-7ivrxWttSxInDksR2rtM24xGdntY/3Ew5j08W5SiG45B+T2nyARL5EOk2xMXEm3MLjPrpNYAybA21ZEDYhGG0Q==} + engines: {node: '>=22'} + + '@atproto/lex-installer@0.1.17': + resolution: {integrity: sha512-XSXiFLoDT7YEe+GuUdwczKQmhuLrA2CHVVlL8s5U8FLtWC5Cr3ELmz1t1b/Y8dV+R31hWyvdsk/t992B6+WW3Q==} + engines: {node: '>=22'} + + '@atproto/lex-json@0.1.6': + resolution: {integrity: sha512-mvrAd0lbyuecIHjyld8QN6MN6CBf4j0GCxLzegsvLh0SvDf+GbYWklkcQqmITL44yFQOwmA/QNIQj0Uvh7+R/g==} + engines: {node: '>=22'} + + '@atproto/lex-resolver@0.2.10': + resolution: {integrity: sha512-0Ml37emPqi6zr7hBAVR8fi7XDgG6xWsksBotoMU4Doh/PNFNt/H79Zopt/jnmDRcl8DGj9PNqZOZjzcAr2IuFw==} + engines: {node: '>=22'} + + '@atproto/lex-schema@0.2.6': + resolution: {integrity: sha512-8fe/gmhjMImBsy073jMANkbHHQLe/w4s9XLATpAz+Qd6WrR9RYTqc2DU8uZDL13F5bfdGTj8KECt/Xs4CoPXdQ==} + engines: {node: '>=22'} + + '@atproto/lex@0.3.8': + resolution: {integrity: sha512-8X7f6wxVDm4IVNR80Yg3BV3IXb89sKo/KIZeYb4or/EHP7ljazVU4qU+RPXmlELnWiIcw+PiJE+hPSkGLnMPOA==} + engines: {node: '>=22'} + hasBin: true + + '@atproto/repo@0.10.12': + resolution: {integrity: sha512-SnDSoFi1bRAfN0IcDjSPFcefknDCIIjKgJXgFsd5jvktCkopmzml8BpQEP5t2/mcZ7NvEn5onQ0kaWkXhgL+5g==} + engines: {node: '>=22'} + + '@atproto/syntax@0.7.5': + resolution: {integrity: sha512-6vnLQK8OAzg0dO6z/xnvuXn5zMV0UMI54zbxk7G7BXhGlIlLMb1yo+JVYtAlv8Nxzr+AaFdNZ8AJt+L/QhJMFQ==} + engines: {node: '>=22'} + + '@atproto/ws-client@0.2.1': + resolution: {integrity: sha512-HCGkqH/Ksxpgg/xCyIvH5hK6o8Lg2PxcEA790GCEV9MspCZB/xIesHZVgJN0cEvklGe+6pNCOZE1B5kj9q1ONA==} + engines: {node: '>=22'} + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} + '@bsky/jetstream@1.0.0': + resolution: {integrity: sha512-YtQYhG6NEo7GevmVhRAUp8koyjoj+zf7u+Mo5/Of8xmHFVGLhHyQ57jck8JTPjizFW59G3fqM1fifMCuHFzL0w==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@atproto/lex': ^0.3.0 + '@changesets/apply-release-plan@7.1.1': resolution: {integrity: sha512-9qPCm/rLx/xoOFXIHGB229+4GOL76S4MC+7tyOuTsR6+1jYlfFDQORdvwR5hDA6y4FL2BPt3qpbcQIS+dW85LA==} @@ -1568,6 +1672,14 @@ packages: os: [linux] libc: [glibc] + '@noble/curves@1.9.7': + resolution: {integrity: sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==} + engines: {node: ^14.21.3 || >=16} + + '@noble/hashes@1.8.0': + resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} + engines: {node: ^14.21.3 || >=16} + '@noble/secp256k1@3.1.0': resolution: {integrity: sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==} @@ -1602,6 +1714,9 @@ packages: '@oxc-project/types@0.142.0': resolution: {integrity: sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ==} + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} @@ -2177,6 +2292,9 @@ packages: '@tiptap/core': 3.29.2 '@tiptap/pm': 3.29.2 + '@ts-morph/common@0.28.1': + resolution: {integrity: sha512-W74iWf7ILp1ZKNYXY5qbddNaml7e9Sedv5lvU1V8lftlitkc9Pq1A+jlH23ltDgWYeZFFEqGCD1Ies9hqu3O+g==} + '@turbo/darwin-64@2.10.8': resolution: {integrity: sha512-po+7rfJfUnFXjWlcoN2RwhErgzCdRtBc1T26vYPcywHlggmCQiQe1uWaE4j+BibI2uY9/2pDoFzMN0rmSaPFOw==} cpu: [x64] @@ -2358,6 +2476,14 @@ packages: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + ansi-regex@6.3.0: + resolution: {integrity: sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==} + engines: {node: '>=12'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} @@ -2379,6 +2505,10 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -2423,6 +2553,10 @@ packages: resolution: {integrity: sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==} engines: {node: '>=20.19.0'} + cborg@4.5.8: + resolution: {integrity: sha512-6/viltD51JklRhq4L7jC3zgy6gryuG5xfZ3kzpE+PravtyeQLeQmCYLREhQH7pWENg5pY4Yu/XCd6a7dKScVlw==} + hasBin: true + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -2434,10 +2568,17 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + clsx@2.1.1: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + code-block-writer@13.0.3: + resolution: {integrity: sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==} + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} @@ -2463,6 +2604,9 @@ packages: core-js@3.49.0: resolution: {integrity: sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg==} + core-js@3.50.0: + resolution: {integrity: sha512-BRWgOLKkFeCgRudR6zrs8p9XJZcE14grzKMMssoYrk6krtuEZ7MTKPIY5RzOnqsEKIR9kst7wNzphttraT+Yqw==} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -2519,6 +2663,9 @@ packages: emoji-picker-element@1.29.1: resolution: {integrity: sha512-TOiHzu9Dqib3x4MwcAi3wi3RdyT4SoeB4b15AvH1ks4SBwTl7DeebhZ0d3x6dNi4XfNU7IGRZ7NBQllj0RqwrQ==} + emoji-regex@10.6.0: + resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + enhanced-resolve@5.24.5: resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} engines: {node: '>=10.13.0'} @@ -2548,6 +2695,10 @@ packages: engines: {node: '>=18'} hasBin: true + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -2715,6 +2866,14 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-east-asian-width@1.6.0: + resolution: {integrity: sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==} + engines: {node: '>=18'} + glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -2801,6 +2960,9 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + iso-datestring-validator@2.2.2: + resolution: {integrity: sha512-yLEMkBbLZTlVQqOnQ4FiMujR6T4DEcCb1xizmvXS+OxuhwcbtynoosRzdMA69zZCShCNAbi+gJ71FxZBBXx1SA==} + jiti@2.7.0: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true @@ -3029,6 +3191,9 @@ packages: lowlight@3.3.0: resolution: {integrity: sha512-0JNhgFoPvP6U6lE/UdVsSq99tn6DhjjpAj5MxG49ewd2mOBVtwWYIT8ClyABhq198aXXODMU6Ox8DrGy/CpTZQ==} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} hasBin: true @@ -3085,6 +3250,9 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + multiformats@13.4.2: + resolution: {integrity: sha512-eh6eHCrRi1+POZ3dA+Dq1C6jhP1GNtr9CRINMb67OKzqW9I5DUuZM/3jLPlzhgpGeiNUlEGEbkCYChXMCc/8DQ==} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -3117,6 +3285,10 @@ packages: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3166,6 +3338,9 @@ packages: react: optional: true + path-browserify@1.0.1: + resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -3233,6 +3408,16 @@ packages: resolution: {integrity: sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==} engines: {node: '>=6'} + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@10.3.1: + resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} + hasBin: true + pirates@4.0.7: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} @@ -3385,6 +3570,9 @@ packages: engines: {node: '>=14'} hasBin: true + process-warning@5.1.0: + resolution: {integrity: sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==} + prosemirror-changeset@2.4.1: resolution: {integrity: sha512-96WBLhOaYhJ+kPhLg3uW359Tz6I/MfcrQfL4EGv4SrcqKEMC1gmoGrXHecPE8eOwTVCJ4IwgfzM8fFad25wNfw==} @@ -3434,6 +3622,9 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + rangetouch@2.0.1: resolution: {integrity: sha512-sln+pNSc8NGaHoLzwNBssFSf/rSYkqeBXzX1AtJlkJiUaVSJSbRAWJk+4omsXkN+EJalzkZhWQ3th1m0FpR5xA==} @@ -3445,6 +3636,13 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + real-require@1.0.0: + resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==} + regexparam@3.0.0: resolution: {integrity: sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==} engines: {node: '>=8'} @@ -3501,6 +3699,10 @@ packages: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} @@ -3543,6 +3745,9 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -3567,10 +3772,22 @@ packages: std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + + string-width@8.2.2: + resolution: {integrity: sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==} + engines: {node: '>=20'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -3671,6 +3888,10 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + thread-stream@4.2.0: + resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} + engines: {node: '>=20'} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3710,6 +3931,9 @@ packages: ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + ts-morph@27.0.2: + resolution: {integrity: sha512-fhUhgeljcrdZ+9DZND1De1029PrE+cMkIP7ooqkLRTrRLTqcki2AstsyJm0vRNbTbVCNJ0idGlbBrfqc7/nA8w==} + tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} @@ -3764,6 +3988,9 @@ packages: ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + uint8arrays@5.1.1: + resolution: {integrity: sha512-9muQwa4wZG4dKi9gMAIBtnk2Pw87SRpvWTH6lOGm19V2Uqxr4uomUf2PGqPnWc+qs06sN8owUU4jfcoWOcfwVQ==} + undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} @@ -3778,6 +4005,9 @@ packages: unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + unicode-segmenter@0.14.5: + resolution: {integrity: sha512-jHGmj2LUuqDcX3hqY12Ql+uhUTn8huuxNZGq7GvtF6bSybzH3aFgedYu/KTzQStEgt1Ra2F3HxadNXsNjb3m3g==} + unicode-segmenter@0.17.3: resolution: {integrity: sha512-hKZwqBjJDmqNrq1+LjDxck1qLzJFcLLJM2Xq92ORRHOuau6GSHeELmn+uyk6zNH1CK9mogrUJGP9WJCJUQXMAw==} @@ -3802,6 +4032,9 @@ packages: typescript: optional: true + varint@6.0.0: + resolution: {integrity: sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==} + vite@8.2.0: resolution: {integrity: sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3962,6 +4195,10 @@ packages: '@cloudflare/workers-types': optional: true + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + ws@8.18.0: resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} engines: {node: '>=10.0.0'} @@ -3990,10 +4227,22 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yaml@1.10.3: resolution: {integrity: sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==} engines: {node: '>= 6'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + + yargs@18.1.0: + resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -4011,6 +4260,9 @@ packages: zimmerframe@1.1.4: resolution: {integrity: sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ==} + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + snapshots: '@atcute/atproto@3.1.12(@atcute/lexicons@2.0.3)': @@ -4237,8 +4489,166 @@ snapshots: - '@atcute/cid' - typescript + '@atproto-labs/did-resolver@0.3.7': + dependencies: + '@atproto-labs/fetch': 0.3.5 + '@atproto-labs/pipe': 0.2.4 + '@atproto-labs/simple-store': 0.5.1 + '@atproto-labs/simple-store-memory': 0.2.6 + '@atproto/did': 0.5.4 + zod: 3.25.76 + + '@atproto-labs/fetch@0.3.5': + dependencies: + '@atproto-labs/pipe': 0.2.4 + + '@atproto-labs/pipe@0.2.4': {} + + '@atproto-labs/simple-store-memory@0.2.6': + dependencies: + '@atproto-labs/simple-store': 0.5.1 + lru-cache: 10.4.3 + + '@atproto-labs/simple-store@0.5.1': {} + + '@atproto/common-web@0.5.10': + dependencies: + '@atproto/lex-data': 0.1.7 + '@atproto/lex-json': 0.1.6 + '@atproto/syntax': 0.7.5 + zod: 3.25.76 + + '@atproto/common@0.8.1': + dependencies: + '@atproto/common-web': 0.5.10 + '@atproto/lex-cbor': 0.1.6 + '@atproto/lex-data': 0.1.7 + multiformats: 13.4.2 + pino: 10.3.1 + + '@atproto/crypto@0.5.4': + dependencies: + '@noble/curves': 1.9.7 + '@noble/hashes': 1.8.0 + uint8arrays: 5.1.1 + + '@atproto/did@0.5.4': + dependencies: + zod: 3.25.76 + + '@atproto/lex-builder@0.1.13': + dependencies: + '@atproto/lex-document': 0.1.10 + '@atproto/lex-schema': 0.2.6 + prettier: 3.9.6 + ts-morph: 27.0.2 + tslib: 2.8.1 + + '@atproto/lex-cbor@0.1.6': + dependencies: + '@atproto/lex-data': 0.1.7 + cborg: 4.5.8 + tslib: 2.8.1 + + '@atproto/lex-client@0.3.5': + dependencies: + '@atproto/lex-data': 0.1.7 + '@atproto/lex-json': 0.1.6 + '@atproto/lex-schema': 0.2.6 + tslib: 2.8.1 + + '@atproto/lex-data@0.1.7': + dependencies: + multiformats: 13.4.2 + tslib: 2.8.1 + unicode-segmenter: 0.14.5 + + '@atproto/lex-document@0.1.10': + dependencies: + '@atproto/lex-schema': 0.2.6 + core-js: 3.50.0 + tslib: 2.8.1 + + '@atproto/lex-installer@0.1.17': + dependencies: + '@atproto/lex-builder': 0.1.13 + '@atproto/lex-cbor': 0.1.6 + '@atproto/lex-data': 0.1.7 + '@atproto/lex-document': 0.1.10 + '@atproto/lex-resolver': 0.2.10 + '@atproto/lex-schema': 0.2.6 + '@atproto/syntax': 0.7.5 + tslib: 2.8.1 + + '@atproto/lex-json@0.1.6': + dependencies: + '@atproto/lex-data': 0.1.7 + tslib: 2.8.1 + + '@atproto/lex-resolver@0.2.10': + dependencies: + '@atproto-labs/did-resolver': 0.3.7 + '@atproto/crypto': 0.5.4 + '@atproto/lex-client': 0.3.5 + '@atproto/lex-data': 0.1.7 + '@atproto/lex-document': 0.1.10 + '@atproto/lex-schema': 0.2.6 + '@atproto/repo': 0.10.12 + '@atproto/syntax': 0.7.5 + tslib: 2.8.1 + + '@atproto/lex-schema@0.2.6': + dependencies: + '@atproto/lex-data': 0.1.7 + '@atproto/syntax': 0.7.5 + '@standard-schema/spec': 1.1.0 + tslib: 2.8.1 + + '@atproto/lex@0.3.8': + dependencies: + '@atproto/lex-builder': 0.1.13 + '@atproto/lex-client': 0.3.5 + '@atproto/lex-data': 0.1.7 + '@atproto/lex-installer': 0.1.17 + '@atproto/lex-json': 0.1.6 + '@atproto/lex-schema': 0.2.6 + tslib: 2.8.1 + yargs: 18.1.0 + + '@atproto/repo@0.10.12': + dependencies: + '@atproto/common': 0.8.1 + '@atproto/common-web': 0.5.10 + '@atproto/crypto': 0.5.4 + '@atproto/lex-cbor': 0.1.6 + '@atproto/lex-data': 0.1.7 + '@atproto/syntax': 0.7.5 + varint: 6.0.0 + zod: 3.25.76 + + '@atproto/syntax@0.7.5': + dependencies: + iso-datestring-validator: 2.2.2 + tslib: 2.8.1 + + '@atproto/ws-client@0.2.1': + dependencies: + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@babel/runtime@7.29.7': {} + '@bsky/jetstream@1.0.0(@atproto/lex@0.3.8)': + dependencies: + '@atproto/lex': 0.3.8 + '@atproto/lex-cbor': 0.1.6 + '@atproto/ws-client': 0.2.1 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@changesets/apply-release-plan@7.1.1': dependencies: '@changesets/config': 3.1.4 @@ -5117,6 +5527,12 @@ snapshots: '@napi-rs/lzma-linux-x64-gnu@1.5.1': optional: true + '@noble/curves@1.9.7': + dependencies: + '@noble/hashes': 1.8.0 + + '@noble/hashes@1.8.0': {} + '@noble/secp256k1@3.1.0': {} '@nodelib/fs.scandir@2.1.5': @@ -5147,6 +5563,8 @@ snapshots: '@oxc-project/types@0.142.0': {} + '@pinojs/redact@0.4.0': {} + '@polka/url@1.0.0-next.29': {} '@poppinss/colors@4.1.6': @@ -5595,6 +6013,12 @@ snapshots: '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) '@tiptap/pm': 3.29.2 + '@ts-morph/common@0.28.1': + dependencies: + minimatch: 10.2.6 + path-browserify: 1.0.1 + tinyglobby: 0.2.17 + '@turbo/darwin-64@2.10.8': optional: true @@ -5811,6 +6235,10 @@ snapshots: ansi-regex@5.0.1: {} + ansi-regex@6.3.0: {} + + ansi-styles@6.2.3: {} + any-promise@1.3.0: {} argparse@1.0.10: @@ -5825,6 +6253,8 @@ snapshots: assertion-error@2.0.1: {} + atomic-sleep@1.0.0: {} + axobject-query@4.1.0: {} balanced-match@4.0.4: {} @@ -5865,6 +6295,8 @@ snapshots: cac@7.0.0: {} + cborg@4.5.8: {} + chai@6.2.2: {} chardet@2.2.0: {} @@ -5873,8 +6305,16 @@ snapshots: dependencies: readdirp: 4.1.2 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.2.0 + wrap-ansi: 9.0.2 + clsx@2.1.1: {} + code-block-writer@13.0.3: {} + commander@4.1.1: {} confbox@0.1.8: {} @@ -5889,6 +6329,8 @@ snapshots: core-js@3.49.0: {} + core-js@3.50.0: {} + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -5929,6 +6371,8 @@ snapshots: emoji-picker-element@1.29.1: {} + emoji-regex@10.6.0: {} + enhanced-resolve@5.24.5: dependencies: graceful-fs: 4.2.11 @@ -6030,6 +6474,8 @@ snapshots: '@esbuild/win32-ia32': 0.28.1 '@esbuild/win32-x64': 0.28.1 + escalade@3.2.0: {} + escape-string-regexp@4.0.0: {} eslint-config-prettier@10.1.8(eslint@10.8.0(jiti@2.7.0)): @@ -6221,6 +6667,10 @@ snapshots: fsevents@2.3.3: optional: true + get-caller-file@2.0.5: {} + + get-east-asian-width@1.6.0: {} + glob-parent@5.1.2: dependencies: is-glob: 4.0.3 @@ -6286,6 +6736,8 @@ snapshots: isexe@2.0.0: {} + iso-datestring-validator@2.2.2: {} + jiti@2.7.0: {} joycon@3.1.1: {} @@ -6450,6 +6902,8 @@ snapshots: devlop: 1.1.0 highlight.js: 11.11.1 + lru-cache@10.4.3: {} + lz-string@1.5.0: {} magic-string@0.30.21: @@ -6514,6 +6968,8 @@ snapshots: ms@2.1.3: {} + multiformats@13.4.2: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -6536,6 +6992,8 @@ snapshots: obug@2.1.4: {} + on-exit-leak-free@2.1.2: {} + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -6581,6 +7039,8 @@ snapshots: dependencies: event-target-polyfill: 0.0.4 + path-browserify@1.0.1: {} + path-exists@4.0.0: {} path-key@3.1.1: {} @@ -6634,6 +7094,26 @@ snapshots: pify@4.0.1: {} + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-std-serializers@7.1.0: {} + + pino@10.3.1: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.1.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 4.2.0 + pirates@4.0.7: {} pkg-types@1.3.1: @@ -6711,6 +7191,8 @@ snapshots: prettier@3.9.6: {} + process-warning@5.1.0: {} + prosemirror-changeset@2.4.1: dependencies: prosemirror-transform: 1.12.0 @@ -6791,6 +7273,8 @@ snapshots: queue-microtask@1.2.3: {} + quick-format-unescaped@4.0.4: {} + rangetouch@2.0.1: {} read-yaml-file@1.1.0: @@ -6802,6 +7286,10 @@ snapshots: readdirp@4.1.2: {} + real-require@0.2.0: {} + + real-require@1.0.0: {} + regexparam@3.0.0: {} resolve-from@5.0.0: {} @@ -6894,6 +7382,8 @@ snapshots: dependencies: mri: 1.2.0 + safe-stable-stringify@2.5.0: {} + safer-buffer@2.1.2: {} semver@7.8.5: {} @@ -6981,6 +7471,10 @@ snapshots: slash@3.0.0: {} + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + source-map-js@1.2.1: {} source-map@0.7.6: {} @@ -6998,10 +7492,25 @@ snapshots: std-env@4.2.0: {} + string-width@7.2.0: + dependencies: + emoji-regex: 10.6.0 + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + + string-width@8.2.2: + dependencies: + get-east-asian-width: 1.6.0 + strip-ansi: 7.2.0 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.3.0 + strip-bom@3.0.0: {} style-to-object@1.0.14: @@ -7119,6 +7628,10 @@ snapshots: dependencies: any-promise: 1.3.0 + thread-stream@4.2.0: + dependencies: + real-require: 1.0.0 + tinybench@2.9.0: {} tinyexec@0.3.2: {} @@ -7146,6 +7659,11 @@ snapshots: ts-interface-checker@0.1.13: {} + ts-morph@27.0.2: + dependencies: + '@ts-morph/common': 0.28.1 + code-block-writer: 13.0.3 + tslib@2.8.1: {} tsup@8.5.1(jiti@2.7.0)(postcss@8.5.25)(tsx@4.23.6)(typescript@6.0.3): @@ -7212,6 +7730,10 @@ snapshots: ufo@1.6.4: {} + uint8arrays@5.1.1: + dependencies: + multiformats: 13.4.2 + undici-types@8.3.0: {} undici@7.24.8: {} @@ -7222,6 +7744,8 @@ snapshots: dependencies: pathe: 2.0.3 + unicode-segmenter@0.14.5: {} + unicode-segmenter@0.17.3: {} universalify@0.1.2: {} @@ -7238,6 +7762,8 @@ snapshots: optionalDependencies: typescript: 6.0.3 + varint@6.0.0: {} + vite@8.2.0(@types/node@26.1.2)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.23.6): dependencies: lightningcss: 1.33.0 @@ -7396,14 +7922,33 @@ snapshots: - bufferutil - utf-8-validate + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.3 + string-width: 7.2.0 + strip-ansi: 7.2.0 + ws@8.18.0: {} ws@8.21.0: {} xtend@4.0.2: {} + y18n@5.0.8: {} + yaml@1.10.3: {} + yargs-parser@22.0.0: {} + + yargs@18.1.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 8.2.2 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yocto-queue@0.1.0: {} yocto-queue@1.2.2: {} @@ -7422,3 +7967,5 @@ snapshots: youch-core: 0.3.3 zimmerframe@1.1.4: {} + + zod@3.25.76: {} -- 2.51.2 From cd9545e41731039940e00c55194b7871d6e6c5a7 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:04:56 +0200 Subject: [PATCH 26/34] fix clean-install lexicon checks --- apps/cloudflare-workers/package.json | 2 +- apps/sveltekit-cloudflare-workers/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cloudflare-workers/package.json b/apps/cloudflare-workers/package.json index 8b6926d..5e215fd 100644 --- a/apps/cloudflare-workers/package.json +++ b/apps/cloudflare-workers/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "contrail dev", "deploy": "wrangler deploy", - "typecheck": "contrail lexicons check --no-atcute-config && tsc --noEmit" + "typecheck": "node ../../packages/contrail/dist/cli.js lexicons check --no-atcute-config && tsc --noEmit" }, "dependencies": { "@atmo-dev/contrail": "workspace:*" diff --git a/apps/sveltekit-cloudflare-workers/package.json b/apps/sveltekit-cloudflare-workers/package.json index dbcacdb..a33e14b 100644 --- a/apps/sveltekit-cloudflare-workers/package.json +++ b/apps/sveltekit-cloudflare-workers/package.json @@ -12,7 +12,7 @@ "preview": "vite preview", "prepare": "svelte-kit sync || echo ''", "check": "pnpm generate:types && svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "typecheck": "contrail lexicons check --no-atcute-config && pnpm check", + "typecheck": "node ../../packages/contrail/dist/cli.js lexicons check --no-atcute-config && pnpm check", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "format": "prettier --write .", "lint": "prettier --check . && eslint .", -- 2.51.2 From 280c36a65b7efa62f57c3776a3498a6fc4778268 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:37:37 +0000 Subject: [PATCH 27/34] Version Packages --- .changeset/calm-winds-connect.md | 5 ----- .changeset/jetstream-v2-live.md | 5 ----- packages/contrail/CHANGELOG.md | 10 ++++++++++ packages/contrail/package.json | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 .changeset/calm-winds-connect.md delete mode 100644 .changeset/jetstream-v2-live.md diff --git a/.changeset/calm-winds-connect.md b/.changeset/calm-winds-connect.md deleted file mode 100644 index 1d9be89..0000000 --- a/.changeset/calm-winds-connect.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": patch ---- - -Allow config-backed `contrail dev` and `contrail connect` to continue after confirmation when configured Lexicons cannot yet be resolved, omitting incomplete schema closures and generating untyped values for affected records. diff --git a/.changeset/jetstream-v2-live.md b/.changeset/jetstream-v2-live.md deleted file mode 100644 index fa487ae..0000000 --- a/.changeset/jetstream-v2-live.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": minor ---- - -Switch scheduled and persistent live ingestion to Jetstream v2 using the official `@bsky/jetstream` client. Live cursors now use instance-local sequence numbers, one normalized v2 service is durably pinned to each generation, legacy timestamp cursors transition atomically into the seq domain, and existing PDS/Alluvium historical acquisition remains unchanged. Alluvium bootstrap now has a separate `sourceUrl`/`--alluvium-source-url` for its legacy v1 manifest identity. Unscoped `getCursor` responses now return `{ cursor }`; ordered-source responses retain their opaque `{ source, epoch, cursor }` position. The minimum supported Node.js version is now 22.15. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 41d6824..13612a8 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,15 @@ # @atmo-dev/contrail +## 0.22.0 + +### Minor Changes + +- b851583: Switch scheduled and persistent live ingestion to Jetstream v2 using the official `@bsky/jetstream` client. Live cursors now use instance-local sequence numbers, one normalized v2 service is durably pinned to each generation, legacy timestamp cursors transition atomically into the seq domain, and existing PDS/Alluvium historical acquisition remains unchanged. Alluvium bootstrap now has a separate `sourceUrl`/`--alluvium-source-url` for its legacy v1 manifest identity. Unscoped `getCursor` responses now return `{ cursor }`; ordered-source responses retain their opaque `{ source, epoch, cursor }` position. The minimum supported Node.js version is now 22.15. + +### Patch Changes + +- 28f94c9: Allow config-backed `contrail dev` and `contrail connect` to continue after confirmation when configured Lexicons cannot yet be resolved, omitting incomplete schema closures and generating untyped values for affected records. + ## 0.21.2 ### Patch Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index b02a1d6..a2fa459 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.21.2", + "version": "0.22.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 825aa8550234741df854ff9164af6cdd30dbd00f Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:04:58 +0200 Subject: [PATCH 28/34] fix cloudflare worker issues --- .changeset/workerd-runtime-compatibility.md | 5 ++ packages/contrail/src/core/jetstream-live.ts | 6 +- packages/contrail/src/core/jetstream.ts | 31 ++++------- .../contrail/src/core/runtime-telemetry.ts | 27 +++++++++ .../contrail/tests/jetstream-v2-live.test.ts | 55 ++++++++++++++++++- .../contrail/tests/runtime-cpu-usage.test.ts | 51 +++++++++++++++++ 6 files changed, 150 insertions(+), 25 deletions(-) create mode 100644 .changeset/workerd-runtime-compatibility.md create mode 100644 packages/contrail/src/core/runtime-telemetry.ts create mode 100644 packages/contrail/tests/runtime-cpu-usage.test.ts diff --git a/.changeset/workerd-runtime-compatibility.md b/.changeset/workerd-runtime-compatibility.md new file mode 100644 index 0000000..bbcea00 --- /dev/null +++ b/.changeset/workerd-runtime-compatibility.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": patch +--- + +Keep optional CPU telemetry and Jetstream v2 cursor preflights compatible with Cloudflare Workers. Exposed but unimplemented `process.cpuUsage` methods now degrade to `null`, and cursor probes use a receiver-safe fetch call with fail-closed manual redirect handling. diff --git a/packages/contrail/src/core/jetstream-live.ts b/packages/contrail/src/core/jetstream-live.ts index 13ae39e..d95b52e 100644 --- a/packages/contrail/src/core/jetstream-live.ts +++ b/packages/contrail/src/core/jetstream-live.ts @@ -132,9 +132,11 @@ async function assertSeqCursorAvailable(options: { // validation is source-global; one valid kind is enough to reach it. url.searchParams.set("kinds", "commit"); - const response = await options.fetchImpl(url, { + const response = await options.fetchImpl.call(globalThis, url, { cache: "no-store", - redirect: "error", + // Workerd does not implement `redirect: "error"`. Manual redirects remain + // fail-closed because only the expected 426 response is accepted below. + redirect: "manual", signal: options.signal, }); const body = (await response.text()).slice(0, 2_000); diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index 6b22083..b4b666c 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -28,6 +28,10 @@ import { applyIdentityEventStatement, } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; +import { + runtimeCpuUsage, + type RuntimeCpuUsageProcess, +} from "./runtime-telemetry"; const BATCH_SIZE = 50; @@ -674,25 +678,6 @@ function admissionFilteredCount(dropped: IngestDropCounts): number { ); } -function runtimeCpuUsage(): (() => number) | null { - const runtimeProcess = ( - globalThis as typeof globalThis & { - process?: { - cpuUsage?: (previous?: { user: number; system: number }) => { - user: number; - system: number; - }; - }; - } - ).process; - if (!runtimeProcess?.cpuUsage) return null; - const start = runtimeProcess.cpuUsage(); - return () => { - const elapsed = runtimeProcess.cpuUsage!(start); - return Math.round((elapsed.user + elapsed.system) / 100) / 10; - }; -} - // Run a full ingest cycle: init schema, load cursor, ingest, apply, save cursor export async function runIngestCycle( db: Database, @@ -706,7 +691,13 @@ export async function runIngestCycle( const scheduledUrls = config.jetstreams ?? DEFAULT_JETSTREAMS; const service = jetstreamService(scheduledUrls); const wallStartedAt = Date.now(); - const finishCpuUsage = runtimeCpuUsage(); + const finishCpuUsage = runtimeCpuUsage( + ( + globalThis as typeof globalThis & { + process?: RuntimeCpuUsageProcess; + } + ).process, + ); const s = state ?? createIngestState(); if (!s.schemaInitialized) { diff --git a/packages/contrail/src/core/runtime-telemetry.ts b/packages/contrail/src/core/runtime-telemetry.ts new file mode 100644 index 0000000..e0e8e72 --- /dev/null +++ b/packages/contrail/src/core/runtime-telemetry.ts @@ -0,0 +1,27 @@ +export interface RuntimeCpuUsageProcess { + cpuUsage?: (previous?: { user: number; system: number }) => { + user: number; + system: number; + }; +} + +/** Optional runtime telemetry must never become an ingestion capability gate. + * Some edge runtimes expose Node-compatible process methods as throwing stubs. */ +export function runtimeCpuUsage( + runtimeProcess: RuntimeCpuUsageProcess | undefined, +): (() => number | null) | null { + if (!runtimeProcess?.cpuUsage) return null; + try { + const start = runtimeProcess.cpuUsage(); + return () => { + try { + const elapsed = runtimeProcess.cpuUsage!(start); + return Math.round((elapsed.user + elapsed.system) / 100) / 10; + } catch { + return null; + } + }; + } catch { + return null; + } +} diff --git a/packages/contrail/tests/jetstream-v2-live.test.ts b/packages/contrail/tests/jetstream-v2-live.test.ts index f24a821..9519b92 100644 --- a/packages/contrail/tests/jetstream-v2-live.test.ts +++ b/packages/contrail/tests/jetstream-v2-live.test.ts @@ -19,10 +19,18 @@ import { createTestDbWithSchema } from "./helpers"; describe("Jetstream v2 live cursor semantics", () => { it("rejects a stale seq through HTTP before an opaque browser transport can retry", async () => { let transportStarted = false; + let fetchReceiver: unknown; + let requestInit: RequestInit | undefined; const subscription = new JetstreamLiveSubscription({ url: "https://jetstream.example", cursor: 42, - fetchImpl: (async (input) => { + fetchImpl: (async function ( + this: unknown, + input, + init, + ) { + fetchReceiver = this; + requestInit = init; expect(String(input)).toContain( "/xrpc/network.bsky.jetstream.subscribeEvents?cursor=42&kinds=commit", ); @@ -53,19 +61,58 @@ describe("Jetstream v2 live cursor semantics", () => { await expect( subscription[Symbol.asyncIterator]().next(), ).rejects.toBeInstanceOf(JetstreamLiveHistoryExpiredError); + expect(fetchReceiver).toBe(globalThis); + expect(requestInit).toMatchObject({ + cache: "no-store", + redirect: "manual", + }); + expect(transportStarted).toBe(false); + }); + + it("rejects a manual redirect instead of following or accepting it", async () => { + let transportStarted = false; + const subscription = new JetstreamLiveSubscription({ + url: "https://jetstream.example", + cursor: 42, + fetchImpl: (async (_input, init) => { + expect(init?.redirect).toBe("manual"); + return new Response("redirected", { + status: 302, + headers: { location: "https://elsewhere.example" }, + }); + }) as typeof fetch, + liveTransport: { + stream() { + transportStarted = true; + return (async function* () { + await new Promise(() => {}); + })(); + }, + }, + }); + + await expect( + subscription[Symbol.asyncIterator]().next(), + ).rejects.toThrow("Jetstream live cursor preflight failed (302)"); expect(transportStarted).toBe(false); }); it("does not misclassify an unverified WebSocket 400 as expired history", async () => { const handshakeError = new Error("Unexpected server response: 400"); + const fetchReceivers: unknown[] = []; + let preflightCalls = 0; const subscription = new JetstreamLiveSubscription({ url: "https://jetstream.example", cursor: 42, // Both the initial probe and the post-failure probe prove only that the // cursor is accepted. The actual filtered WebSocket request may still be // rejected for an unrelated InvalidRequest. - fetchImpl: (async () => - new Response("WebSocket upgrade required", { status: 426 })) as typeof fetch, + fetchImpl: (async function (this: unknown, _input, init) { + preflightCalls++; + fetchReceivers.push(this); + expect(init?.redirect).toBe("manual"); + return new Response("WebSocket upgrade required", { status: 426 }); + }) as typeof fetch, liveTransport: { stream() { return (async function* () { @@ -80,6 +127,8 @@ describe("Jetstream v2 live cursor semantics", () => { ); expect(error).toBe(handshakeError); expect(error).not.toBeInstanceOf(JetstreamLiveHistoryExpiredError); + expect(preflightCalls).toBe(2); + expect(fetchReceivers).toEqual([globalThis, globalThis]); }); it("preserves microsecond precision from the v2 datetime", () => { diff --git a/packages/contrail/tests/runtime-cpu-usage.test.ts b/packages/contrail/tests/runtime-cpu-usage.test.ts new file mode 100644 index 0000000..d538f45 --- /dev/null +++ b/packages/contrail/tests/runtime-cpu-usage.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { runtimeCpuUsage } from "../src/core/runtime-telemetry"; + +describe("optional runtime CPU telemetry", () => { + it("returns null when process or process.cpuUsage is absent", () => { + expect(runtimeCpuUsage(undefined)).toBeNull(); + expect(runtimeCpuUsage({})).toBeNull(); + }); + + it("returns null when the initial cpuUsage call throws", () => { + const finish = runtimeCpuUsage({ + cpuUsage() { + throw new Error("not implemented"); + }, + }); + + expect(finish).toBeNull(); + }); + + it("returns null for the metric when the elapsed cpuUsage call throws", () => { + const finish = runtimeCpuUsage({ + cpuUsage(previous) { + if (previous) throw new Error("elapsed measurement failed"); + return { user: 100, system: 50 }; + }, + }); + + expect(finish).not.toBeNull(); + expect(finish?.()).toBeNull(); + }); + + it("reports numeric milliseconds when cpuUsage is supported", () => { + const start = { user: 100, system: 50 }; + const finish = runtimeCpuUsage({ + cpuUsage(previous) { + if (!previous) return start; + expect(previous).toBe(start); + return { user: 1_200, system: 300 }; + }, + }); + + expect(finish?.()).toBe(1.5); + }); + + it("continues to measure CPU time with Node's implementation", () => { + const finish = runtimeCpuUsage(process); + + expect(finish).not.toBeNull(); + expect(finish?.()).toEqual(expect.any(Number)); + }); +}); -- 2.51.2 From e924bc3e4d039621ae098b7f2726a14f5d7ab6fa Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:36:36 +0200 Subject: [PATCH 29/34] add worker smoke --- .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 3 + package.json | 1 + packages/contrail/package.json | 1 + packages/contrail/tests/workerd-smoke.mjs | 215 ++++++++++++++++++ packages/contrail/tests/workerd/worker.mjs | 67 ++++++ .../contrail/tests/workerd/wrangler.jsonc | 13 ++ 7 files changed, 303 insertions(+) create mode 100644 packages/contrail/tests/workerd-smoke.mjs create mode 100644 packages/contrail/tests/workerd/worker.mjs create mode 100644 packages/contrail/tests/workerd/wrangler.jsonc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8735e58..b6ab8e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,5 +47,8 @@ jobs: - name: Test published adapters run: pnpm test:built + - name: Test built Worker in workerd + run: pnpm test:workerd + - name: Test run: pnpm test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ae1749..3c4d886 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,6 +52,9 @@ jobs: - name: Run tests run: pnpm test + - name: Test built Worker in workerd + run: pnpm test:workerd + - name: Create Release Pull Request or Publish to npm uses: changesets/action@v1 with: diff --git a/package.json b/package.json index 3feede4..5e2ecaf 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "build:apps": "turbo run build", "test": "turbo run test", "test:built": "pnpm --filter @atmo-dev/contrail test:built", + "test:workerd": "pnpm --filter @atmo-dev/contrail test:workerd", "bench": "node --no-warnings --experimental-strip-types apps/benchmark/src/bench.ts", "typecheck": "turbo run typecheck", "dev": "turbo run dev", diff --git a/packages/contrail/package.json b/packages/contrail/package.json index a2fa459..9f5c0a1 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -77,6 +77,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run", "test:built": "node tests/built-sqlite.mjs && node tests/built-alluvium.mjs && node tests/built-lexicons.mjs && node tests/built-client.mjs && node tests/built-changes.mjs", + "test:workerd": "node tests/workerd-smoke.mjs", "test:watch": "vitest" }, "dependencies": { diff --git a/packages/contrail/tests/workerd-smoke.mjs b/packages/contrail/tests/workerd-smoke.mjs new file mode 100644 index 0000000..0f87b4a --- /dev/null +++ b/packages/contrail/tests/workerd-smoke.mjs @@ -0,0 +1,215 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +const PACKAGE_ROOT = resolve(import.meta.dirname, ".."); +const STATE_DIR = await mkdtemp( + resolve(tmpdir(), "contrail-workerd-runtime-smoke-"), +); +const MARKER = "[ingest] cycle summary "; +const ANSI = /\x1b\[[0-?]*[ -/]*[@-~]/g; + +async function availablePort() { + const server = createServer(); + await new Promise((resolveReady, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolveReady); + }); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : null; + await new Promise((resolveClosed) => server.close(resolveClosed)); + if (port === null) throw new Error("Could not allocate a local port"); + return port; +} + +const port = await availablePort(); +const inspectorPort = await availablePort(); +const origin = `http://127.0.0.1:${port}`; +const wrangler = resolve(PACKAGE_ROOT, "node_modules/.bin/wrangler"); +let output = ""; + +const child = spawn( + wrangler, + [ + "dev", + "--config", + "tests/workerd/wrangler.jsonc", + "--port", + String(port), + "--inspector-port", + String(inspectorPort), + "--test-scheduled", + "--persist-to", + STATE_DIR, + "--log-level", + "log", + ], + { + cwd: PACKAGE_ROOT, + env: { ...process.env, CI: "true", NO_COLOR: "1" }, + stdio: ["ignore", "pipe", "pipe"], + }, +); +child.stdout.setEncoding("utf8"); +child.stderr.setEncoding("utf8"); +child.stdout.on("data", (chunk) => { + output += chunk; +}); +child.stderr.on("data", (chunk) => { + output += chunk; +}); + +function summaries() { + const parsed = []; + for (const line of output.replace(ANSI, "").split("\n")) { + const marker = line.indexOf(MARKER); + if (marker === -1) continue; + try { + parsed.push(JSON.parse(line.slice(marker + MARKER.length))); + } catch { + // Ignore a line while Wrangler is still streaming it. + } + } + return parsed; +} + +async function state() { + const response = await fetch(`${origin}/__contrail_workerd_smoke`, { + signal: AbortSignal.timeout(2_000), + }); + if (!response.ok) { + throw new Error(`Smoke state endpoint returned ${response.status}`); + } + return response.json(); +} + +async function waitFor(read, predicate, label, timeoutMs = 30_000) { + const deadline = Date.now() + timeoutMs; + let lastError; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`Wrangler exited early with code ${child.exitCode}`); + } + try { + const value = await read(); + if (predicate(value)) return value; + } catch (error) { + lastError = error; + } + await new Promise((resolveWait) => setTimeout(resolveWait, 250)); + } + throw new Error( + `Timed out waiting for ${label}${lastError ? `: ${lastError}` : ""}`, + ); +} + +async function triggerScheduled() { + const response = await fetch(`${origin}/__scheduled`, { + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error(`Scheduled trigger returned ${response.status}`); + } +} + +async function advanceCursor(predicate, label) { + for (let attempt = 1; attempt <= 3; attempt++) { + await triggerScheduled(); + try { + return await waitFor(state, predicate, label, 20_000); + } catch (error) { + if (attempt === 3) throw error; + } + } + throw new Error(`Could not ${label}`); +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +async function stopWrangler() { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolveExit) => child.once("exit", resolveExit)), + new Promise((resolveTimeout) => setTimeout(resolveTimeout, 5_000)), + ]); + if (child.exitCode === null) child.kill("SIGKILL"); +} + +try { + const initial = await waitFor(state, () => true, "Wrangler startup"); + assert( + initial.cpuUsage?.exposed === true && initial.cpuUsage?.throws === true, + `Expected workerd to expose a throwing process.cpuUsage stub, got ${JSON.stringify(initial.cpuUsage)}`, + ); + + const first = await advanceCursor( + (value) => value.cursorDomain === "seq", + "the timestamp bridge to commit a sequence cursor", + ); + assert( + Number.isSafeInteger(first.cursor) && first.cursor >= 0, + `Invalid first sequence cursor: ${first.cursor}`, + ); + + const second = await advanceCursor( + (value) => + value.cursorDomain === "seq" && + Number.isSafeInteger(value.cursor) && + value.cursor > first.cursor, + "an existing sequence cursor to pass preflight and advance", + ); + + await waitFor( + async () => summaries(), + (values) => + values.some( + (summary) => + summary.starting_cursor === first.cursor && + summary.safe_ending_cursor === second.cursor, + ), + "the resumed sequence-cycle summary", + 5_000, + ); + + const cycleSummaries = summaries(); + const firstSummary = cycleSummaries.find( + (summary) => summary.safe_ending_cursor === first.cursor, + ); + const secondSummary = cycleSummaries.find( + (summary) => + summary.starting_cursor === first.cursor && + summary.safe_ending_cursor === second.cursor, + ); + assert(firstSummary, "Missing timestamp-bridge cycle summary"); + assert(secondSummary, "Missing resumed sequence-cycle summary"); + for (const [label, summary] of [ + ["bridge", firstSummary], + ["resumed seq", secondSummary], + ]) { + assert( + summary.connection_errors === 0, + `${label} cycle reported connection_errors=${summary.connection_errors}`, + ); + assert( + summary.cpu_ms === null, + `${label} cycle did not degrade throwing CPU telemetry to null`, + ); + } + + console.log( + `workerd smoke passed: ${first.cursor} -> ${second.cursor}; ` + + "host fetch preflight succeeded and cpu_ms=null", + ); +} catch (error) { + console.error(error); + console.error("\n--- Wrangler output ---\n" + output.slice(-30_000)); + process.exitCode = 1; +} finally { + await stopWrangler(); + await rm(STATE_DIR, { recursive: true, force: true }); +} diff --git a/packages/contrail/tests/workerd/worker.mjs b/packages/contrail/tests/workerd/worker.mjs new file mode 100644 index 0000000..07b8fc8 --- /dev/null +++ b/packages/contrail/tests/workerd/worker.mjs @@ -0,0 +1,67 @@ +import { createWorker } from "../../dist/worker/index.js"; + +const JETSTREAM_V2_SEQ_THRESHOLD = 1_000_000_000_000_000; + +const config = { + namespace: "com.example", + profiles: [], + constellation: false, + jetstreams: ["https://jetstream.us-east.bsky.network"], + orderedSource: { + source: "jetstream", + epoch: "contrail-workerd-runtime-smoke-v1", + }, + collections: { + post: { collection: "app.bsky.feed.post" }, + }, + logger: console, +}; + +const worker = createWorker(config, { + scheduledIngest: { + maxDrainMs: 8_000, + maxCandidates: 1, + maxIdentityUpdates: 1, + maxSerializedBytes: 1024 * 1024, + }, + backfillRetries: false, +}); + +function cpuUsageCapability() { + const runtimeProcess = globalThis.process; + const exposed = typeof runtimeProcess?.cpuUsage === "function"; + if (!exposed) return { exposed, throws: false }; + try { + runtimeProcess.cpuUsage(); + return { exposed, throws: false }; + } catch (error) { + return { exposed, throws: true, error: String(error) }; + } +} + +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + if (url.pathname === "/__contrail_workerd_smoke") { + // Exercise the built Worker's lazy initialization before inspecting its + // durable cursor directly through the real local D1 binding. + await worker.fetch(new Request(new URL("/health", url)), env, ctx); + const row = await env.DB.prepare( + "SELECT time_us FROM cursor WHERE id = 1", + ).first(); + const cursor = row?.time_us ?? null; + return Response.json({ + cursor, + cursorDomain: + cursor === null + ? null + : cursor < JETSTREAM_V2_SEQ_THRESHOLD + ? "seq" + : "timestamp", + cpuUsage: cpuUsageCapability(), + }); + } + return worker.fetch(request, env, ctx); + }, + scheduled: worker.scheduled, +}; diff --git a/packages/contrail/tests/workerd/wrangler.jsonc b/packages/contrail/tests/workerd/wrangler.jsonc new file mode 100644 index 0000000..9d2e896 --- /dev/null +++ b/packages/contrail/tests/workerd/wrangler.jsonc @@ -0,0 +1,13 @@ +{ + "name": "contrail-workerd-runtime-smoke", + "main": "worker.mjs", + "compatibility_date": "2026-08-06", + "compatibility_flags": ["nodejs_compat"], + "d1_databases": [ + { + "binding": "DB", + "database_name": "contrail-workerd-runtime-smoke", + "database_id": "00000000-0000-0000-0000-000000000001" + } + ] +} -- 2.51.2 From 0d923bf9f79e816f98f23b59cabad80b33b17998 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:56:08 +0200 Subject: [PATCH 30/34] pin workerd smoke to verified runtime --- packages/contrail/package.json | 4 +- .../contrail/tests/workerd/wrangler.jsonc | 2 +- pnpm-lock.yaml | 126 +++++++++++++++++- 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 9f5c0a1..b7c7587 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -104,14 +104,14 @@ }, "devDependencies": { "@atcute/crypto": "2.4.4", - "@cloudflare/workers-types": "^5.20260804.1", + "@cloudflare/workers-types": "5.20260815.1", "@types/node": "^26.1.2", "@types/pg": "^8.20.4", "pg": "^8.22.0", "tsup": "^8.5.1", "typescript": "^6.0.3", "vitest": "^4.1.10", - "wrangler": "^4.118.0" + "wrangler": "4.123.0" }, "peerDependencies": { "pg": "^8.0.0", diff --git a/packages/contrail/tests/workerd/wrangler.jsonc b/packages/contrail/tests/workerd/wrangler.jsonc index 9d2e896..bfb46f6 100644 --- a/packages/contrail/tests/workerd/wrangler.jsonc +++ b/packages/contrail/tests/workerd/wrangler.jsonc @@ -1,7 +1,7 @@ { "name": "contrail-workerd-runtime-smoke", "main": "worker.mjs", - "compatibility_date": "2026-08-06", + "compatibility_date": "2026-08-15", "compatibility_flags": ["nodejs_compat"], "d1_databases": [ { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f076074..6f6451a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -307,8 +307,8 @@ importers: specifier: 2.4.4 version: 2.4.4 '@cloudflare/workers-types': - specifier: ^5.20260804.1 - version: 5.20260804.1 + specifier: 5.20260815.1 + version: 5.20260815.1 '@types/node': specifier: ^26.1.2 version: 26.1.2 @@ -328,8 +328,8 @@ importers: specifier: ^4.1.10 version: 4.1.10(@types/node@26.1.2)(vite@8.2.0(@types/node@26.1.2)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.23.6)) wrangler: - specifier: ^4.118.0 - version: 4.118.0(@cloudflare/workers-types@5.20260804.1) + specifier: 4.123.0 + version: 4.123.0(@cloudflare/workers-types@5.20260815.1) packages: @@ -665,6 +665,12 @@ packages: cpu: [x64] os: [darwin] + '@cloudflare/workerd-darwin-64@1.20260811.1': + resolution: {integrity: sha512-i5jqz+ywtOefr0AJbiAc8qxBLfSim/B0WJG7aW3B+pWnoVfMJdUQvi+BWcFKZJ0MoCci3KadTx6g31VfuEEqpQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + '@cloudflare/workerd-darwin-arm64@1.20260421.1': resolution: {integrity: sha512-Trotq3xRAkIcpC505WoxM8+kIH4JIvOJCNuRatyHcz9uF5S+ukgiVUFUlM+GIjw1uCM/Bda2St+vSniX1RZdpw==} engines: {node: '>=16'} @@ -677,6 +683,12 @@ packages: cpu: [arm64] os: [darwin] + '@cloudflare/workerd-darwin-arm64@1.20260811.1': + resolution: {integrity: sha512-NoOUM/nvaDdm2Onlnz33FikWjtatzulNtvwvy4xs0IrHaTCHwC0c8NwIt6s+AI13FkDs02/vm2I3GTPLCT9+hQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + '@cloudflare/workerd-linux-64@1.20260421.1': resolution: {integrity: sha512-938QjUv0z+QqK6BAvgwX/lCIZ2b224ZXoXtGTbhyNVMhB+mt4Dj24cj9qca4ekNXjVM7uTKp1yOHZO97fVSacw==} engines: {node: '>=16'} @@ -689,6 +701,12 @@ packages: cpu: [x64] os: [linux] + '@cloudflare/workerd-linux-64@1.20260811.1': + resolution: {integrity: sha512-sdYq2jL1AD1supa3fsi5O4zTB28wSjvTHj7Migh6/ts8EROPdvrSwv+rdGHhv8HJNAz/wbIAY3wZsi1Rw4uUIg==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + '@cloudflare/workerd-linux-arm64@1.20260421.1': resolution: {integrity: sha512-YI4+mLfwnJcKJ+iPyxzx+tp2Jy4o29BxBPSQGZxl/AZyvZ9eTKsmNZmtjEiT4i3O/M0tdO/B/d9ESDHbRCs2rQ==} engines: {node: '>=16'} @@ -701,6 +719,12 @@ packages: cpu: [arm64] os: [linux] + '@cloudflare/workerd-linux-arm64@1.20260811.1': + resolution: {integrity: sha512-RIRv4shbu1kg05sD+DHTpSFCNnb5Dl2SkPDMUykqZa508tkPqe7VVw7gO0Q5msTBGyL0FfFrLuRxwwfA8u5Sow==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + '@cloudflare/workerd-windows-64@1.20260421.1': resolution: {integrity: sha512-q1SFgwlNH9lFmw74vh7EJbJtduo92Nx51mNOfd3/u6pux6AldcwRviYzKEEv3FEbtv6OBB7J8D5f8vtZj7Z6Sg==} engines: {node: '>=16'} @@ -713,12 +737,21 @@ packages: cpu: [x64] os: [win32] + '@cloudflare/workerd-windows-64@1.20260811.1': + resolution: {integrity: sha512-g6VquwjASlYAibcNW/0E6Zszht4qLkmnXOGwIjjRHl2A0Qz48kVeMcGvyH6eA0G9U3OzZojjYFpP+YeyQmmdjw==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + '@cloudflare/workers-types@4.20260424.1': resolution: {integrity: sha512-0DLJ9yEk1KKzPbqop80Gw/P1wkKKzawmipULiJWdBXIBCoMvE0OVWms3IrL/Q/G7tfmPop9yF4XlZ69k9JLYng==} '@cloudflare/workers-types@5.20260804.1': resolution: {integrity: sha512-B1dwxpN6e5RZXZkE5zpZj+ooNeNZ1mwavLIyHDYe10ojhlGTwDfe8sAl7R1mMXc1cyIsbr+jKVdvmMEyVcdTdg==} + '@cloudflare/workers-types@5.20260815.1': + resolution: {integrity: sha512-btpW8rEgeJKcxDlBWKxzjLLUHxzUWqXZY8O+DmM6NcSHadUQN5I71z6Q/yAKCTJyGS7adBpXzlsZSFbosLnECQ==} + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -3227,6 +3260,10 @@ packages: resolution: {integrity: sha512-8/dspSXDshP6nSkCpjKO7BYc2qZoYSXm7iM+QxY7qJyJpAB3onnQSaiu0cvKJlfuMGwULl55hG69FJCcCMXU1Q==} engines: {node: '>=22.0.0'} + miniflare@5.20260811.1-alpha: + resolution: {integrity: sha512-DtOG0BeanIxs2sH0smFvExZD89cBQwGckbHiFkRJrrNAUu3NGClZkUxqu+zy7HYfKBAgq935EMY49vIPm3JVdA==} + engines: {node: '>=22.0.0'} + minimatch@10.2.6: resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} engines: {node: 18 || 20 || >=22} @@ -4002,6 +4039,10 @@ packages: resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} engines: {node: '>=20.18.1'} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -4171,6 +4212,11 @@ packages: engines: {node: '>=16'} hasBin: true + workerd@1.20260811.1: + resolution: {integrity: sha512-kh+FFm55JQ4ssxhHZV9VPdMQq3D1nHxNJgwxMtWGD4dGppJvLySdguTRDKgeNTvgq6heSz+6TTXyPSDGj8Yllw==} + engines: {node: '>=16'} + hasBin: true + worktop@0.8.0-next.18: resolution: {integrity: sha512-+TvsA6VAVoMC3XDKR5MoC/qlLqDixEfOBysDEKnPIPou/NvoPWCAuXHXMsswwlvmEuvX56lQjvELLyLuzTKvRw==} engines: {node: '>=12'} @@ -4185,6 +4231,16 @@ packages: '@cloudflare/workers-types': optional: true + wrangler@4.123.0: + resolution: {integrity: sha512-VXo2I1oa0x9aGAKIFPRSQPqTh0RBY5Ktl44YOhNmsJQFUdJKDA2vVTU6Xj+FC2koll6orJqWZN8jbXVIk9O67Q==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260811.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + wrangler@4.84.1: resolution: {integrity: sha512-Xe1S/Bik7pNdtdJ+asHsEZC2dX9k3WxYn2BbxFtOrrLVxN/LKi750zsrjX41jSAk00M/O1l7jzyQV4sQqw8ftg==} engines: {node: '>=20.3.0'} @@ -4808,40 +4864,63 @@ snapshots: optionalDependencies: workerd: 1.20260730.1 + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260811.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260811.1 + '@cloudflare/workerd-darwin-64@1.20260421.1': optional: true '@cloudflare/workerd-darwin-64@1.20260730.1': optional: true + '@cloudflare/workerd-darwin-64@1.20260811.1': + optional: true + '@cloudflare/workerd-darwin-arm64@1.20260421.1': optional: true '@cloudflare/workerd-darwin-arm64@1.20260730.1': optional: true + '@cloudflare/workerd-darwin-arm64@1.20260811.1': + optional: true + '@cloudflare/workerd-linux-64@1.20260421.1': optional: true '@cloudflare/workerd-linux-64@1.20260730.1': optional: true + '@cloudflare/workerd-linux-64@1.20260811.1': + optional: true + '@cloudflare/workerd-linux-arm64@1.20260421.1': optional: true '@cloudflare/workerd-linux-arm64@1.20260730.1': optional: true + '@cloudflare/workerd-linux-arm64@1.20260811.1': + optional: true + '@cloudflare/workerd-windows-64@1.20260421.1': optional: true '@cloudflare/workerd-windows-64@1.20260730.1': optional: true + '@cloudflare/workerd-windows-64@1.20260811.1': + optional: true + '@cloudflare/workers-types@4.20260424.1': {} '@cloudflare/workers-types@5.20260804.1': {} + '@cloudflare/workers-types@5.20260815.1': {} + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -6945,6 +7024,18 @@ snapshots: - bufferutil - utf-8-validate + miniflare@5.20260811.1-alpha: + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.35.2 + undici: 7.29.0 + workerd: 1.20260811.1 + ws: 8.21.0 + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + minimatch@10.2.6: dependencies: brace-expansion: 5.0.9 @@ -7740,6 +7831,8 @@ snapshots: undici@7.28.0: {} + undici@7.29.0: {} + unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 @@ -7883,6 +7976,14 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20260730.1 '@cloudflare/workerd-windows-64': 1.20260730.1 + workerd@1.20260811.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260811.1 + '@cloudflare/workerd-darwin-arm64': 1.20260811.1 + '@cloudflare/workerd-linux-64': 1.20260811.1 + '@cloudflare/workerd-linux-arm64': 1.20260811.1 + '@cloudflare/workerd-windows-64': 1.20260811.1 + worktop@0.8.0-next.18: dependencies: mrmime: 2.0.1 @@ -7905,6 +8006,23 @@ snapshots: - bufferutil - utf-8-validate + wrangler@4.123.0(@cloudflare/workers-types@5.20260815.1): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260811.1) + blake3-wasm: 2.1.5 + esbuild: 0.28.1 + miniflare: 5.20260811.1-alpha + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.24 + workerd: 1.20260811.1 + optionalDependencies: + '@cloudflare/workers-types': 5.20260815.1 + fsevents: 2.3.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + wrangler@4.84.1(@cloudflare/workers-types@4.20260424.1): dependencies: '@cloudflare/kv-asset-handler': 0.4.2 -- 2.51.2 From a80f74c42065a094b9c6b83b3abbeb599ecf1108 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:22:43 +0000 Subject: [PATCH 31/34] Version Packages --- .changeset/workerd-runtime-compatibility.md | 5 ----- packages/contrail/CHANGELOG.md | 6 ++++++ packages/contrail/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/workerd-runtime-compatibility.md diff --git a/.changeset/workerd-runtime-compatibility.md b/.changeset/workerd-runtime-compatibility.md deleted file mode 100644 index bbcea00..0000000 --- a/.changeset/workerd-runtime-compatibility.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": patch ---- - -Keep optional CPU telemetry and Jetstream v2 cursor preflights compatible with Cloudflare Workers. Exposed but unimplemented `process.cpuUsage` methods now degrade to `null`, and cursor probes use a receiver-safe fetch call with fail-closed manual redirect handling. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 13612a8..e1ed69b 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.22.1 + +### Patch Changes + +- 825aa85: Keep optional CPU telemetry and Jetstream v2 cursor preflights compatible with Cloudflare Workers. Exposed but unimplemented `process.cpuUsage` methods now degrade to `null`, and cursor probes use a receiver-safe fetch call with fail-closed manual redirect handling. + ## 0.22.0 ### Minor Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index b7c7587..c7246b7 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.22.0", + "version": "0.22.1", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 25fb67f62e1d52e3a7de00cb236942b6eac511f2 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:31:55 +0200 Subject: [PATCH 32/34] add docs --- README.md | 15 +++++++++ docs/01-getting-started.md | 57 ++++++++++++++++++++++++++++------ docs/03-configure-and-query.md | 22 +++++++++++++ packages/contrail/README.md | 26 +++++++++++----- 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 639f67c..7ba3363 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,21 @@ It provides: Cloudflare Workers with D1 is the primary deployment target. Node.js with SQLite or PostgreSQL is also supported. +## Quick start from a Lexicon prefix + +```bash +pnpx @atmo-dev/contrail init my-appview \ + --prefix community.lexicon.calendar. \ + --namespace com.example.calendar +cd my-appview +pnpx @atmo-dev/contrail dev +``` + +`--prefix` selects the source record Lexicons to import; `--namespace` sets the +separate XRPC namespace for the generated AppView methods. See the +[prefix setup guide](docs/01-getting-started.md#initialize-from-your-lexicon-prefix) +for choosing a prefix, generated files, and import options. + ## Documentation 1. [Get started locally](docs/01-getting-started.md) diff --git a/docs/01-getting-started.md b/docs/01-getting-started.md index c7e64ea..bc6582b 100644 --- a/docs/01-getting-started.md +++ b/docs/01-getting-started.md @@ -2,7 +2,9 @@ Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. -Create a project from a verified Lexicon namespace: +## Initialize from your Lexicon prefix + +Pass the prefix to `contrail init`: ```bash pnpx @atmo-dev/contrail init my-appview \ @@ -11,12 +13,32 @@ pnpx @atmo-dev/contrail init my-appview \ cd my-appview ``` -Contrail queries `https://lex.atmo.tools`, selects record Lexicons under the +Replace `community.lexicon.calendar.` with your source Lexicon prefix. The value +is matched as a literal NSID prefix, so include the trailing `.` when you mean a +namespace boundary. For example, this prefix selects record Lexicons such as +`community.lexicon.calendar.event` and +`community.lexicon.calendar.rsvp`. + +The two namespace options serve different purposes: + +- `--prefix` selects the record Lexicons Contrail imports. +- `--namespace` sets the generated AppView XRPC method namespace. It is optional, + defaults to `com.example`, and does not need to match the imported prefix. + +Omit `my-appview` to initialize the current directory. Contrail refuses to +replace an existing config or pinned import. + +### What initialization creates + +Contrail queries `https://lex.atmo.tools`, selects record Lexicons matching the prefix, pins their complete dependency graph under `lexicons/pinned/`, and -creates `contrail.config.ts`. String fields become equality filters and sort -options; datetime fields become range filters and chronological sort options. -Arrays, unions, numeric fields, and other unsupported shapes are reported rather -than configured incorrectly. +creates `contrail.config.ts`. Every imported schema is recorded with its CID and +authority in `lexicons/pinned.lock`; commit the pinned schemas and lock alongside +the config. + +String fields become equality filters and sort options; datetime fields become +range filters and chronological sort options. Arrays, unions, numeric fields, +and other unsupported shapes are reported rather than configured incorrectly. When a strongRef or AT URI could point at another imported collection, an interactive terminal asks whether to add forward hydration and an inverse @@ -24,17 +46,32 @@ relation. The schema does not encode its target, so non-interactive runs leave ambiguous references unconfigured. Use `--no-interactive` to request that behavior explicitly. -The import requires complete verification and catalog indexing. Use +### Import options + +| Option | Purpose | Default | +|---|---|---| +| `--prefix ` | Import verified record Lexicons matching this NSID prefix | none (uses the offline starter) | +| `--namespace ` | Namespace for generated AppView XRPC methods | `com.example` | +| `--lexicon-api ` | Use another Lexicon registry | `https://lex.atmo.tools` | +| `--timeout ` | Readiness and dependency lookup budget | `60` | +| `--allow-partial` | Accept incomplete prefix verification or catalog indexing | off | +| `--no-interactive` | Skip reference and inverse-relation questions | off | + +The default import requires complete verification and catalog indexing. Use `--allow-partial` only when you deliberately accept that some prefix documents -may be absent; required dependencies must always resolve. Every imported schema -is recorded with its CID and authority in `lexicons/pinned.lock`. +may be absent; required dependencies must always resolve. -For an offline example config with no registry request, omit `--prefix`: +## Offline starter + +For an example config with no registry request, omit `--prefix`: ```bash pnpx @atmo-dev/contrail init my-appview +cd my-appview ``` +## Run the local AppView + Then run: ```bash diff --git a/docs/03-configure-and-query.md b/docs/03-configure-and-query.md index a5d3bae..56c27b7 100644 --- a/docs/03-configure-and-query.md +++ b/docs/03-configure-and-query.md @@ -51,6 +51,13 @@ This produces the following client parameters: Dotted fields become camel case: `subject.uri` becomes `subjectUri`. Every list method also supports `actor` (a DID or handle), `sort`, `order`, `limit`, `cursor`, and `profiles`. +A reference hydrates the current indexed target record identified by the URI in +`field`. If that field comes from a strongRef, Contrail does not retrieve the +historical record version named by the strongRef's CID. Lexicons also do not +usually identify a reference's target collection or desired inverse relation; +configure those semantics explicitly. Prefix initialization prompts for these +choices on a TTY and otherwise leaves them unconfigured. + ## Query from the client After changing the config, re-run `contrail connect` in your app. The generated Atcute client now knows the new parameters and response types: @@ -79,3 +86,18 @@ Every record has `uri`, `cid`, and its original record body in `value`. Hydrated Pass the returned opaque `cursor` into the same query to get the next page. `limit` defaults to 50 and may be 1–200. Full-text `search` works with D1 and PostgreSQL. The zero-config local SQLite AppView does not provide full-text search. + +## Lexicon source files + +Contrail resolves checked-in Lexicon documents in this order: + +1. `lexicons/custom/` — application-owned schemas; +2. `lexicons/pinned/` — verified, CID-pinned schemas created by prefix + initialization; and +3. `lexicons/pulled/` — mutable schemas fetched by `contrail lexicons all`. + +The first document for an NSID wins. Remote pull generation does not request an +NSID already supplied by `custom` or `pinned`, and cleaning pulled output does +not replace either directory. Keep `lexicons/pinned.lock` with the pinned files; +it records the registry snapshot, verification state, source CID, authority, +and role of each imported document. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index b58d992..115e13b 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -176,14 +176,24 @@ cd my-appview contrail dev ``` -Prefix initialization uses `https://lex.atmo.tools` by default. It requires a -stable, fully verified and indexed catalog snapshot, pins each root and -transitive dependency by CID under `lexicons/pinned/`, writes provenance to -`lexicons/pinned.lock`, and generates equality/range fields which are safe on -all supported databases. Use `--lexicon-api` for a different registry, -`--timeout` to change the 60-second readiness budget, or `--allow-partial` to -explicitly accept incomplete prefix discovery. Missing required dependencies -always fail. +The prefix is matched literally against source Lexicon NSIDs. Include a trailing +`.` to select a namespace boundary. `--namespace` is independent: it controls +the generated AppView XRPC method IDs and defaults to `com.example`. + +| Init option | Purpose | +|---|---| +| `--prefix ` | Import verified record Lexicons matching an NSID prefix | +| `--namespace ` | Set the generated AppView XRPC namespace | +| `--lexicon-api ` | Select a registry (default: `https://lex.atmo.tools`) | +| `--timeout ` | Set the readiness/dependency budget (default: `60`) | +| `--allow-partial` | Explicitly accept incomplete prefix discovery | +| `--no-interactive` | Leave ambiguous references unconfigured | + +Prefix initialization requires a stable, fully verified and indexed catalog +snapshot, pins each root and transitive dependency by CID under +`lexicons/pinned/`, writes provenance to `lexicons/pinned.lock`, and generates +equality/range fields which are safe on all supported databases. Missing +required dependencies always fail, including with `--allow-partial`. In an interactive terminal, reference-shaped fields can be connected to an imported target collection and optionally exposed as inverse relations. These -- 2.51.2 From 1abe8cb1bc5d47c694b7710b87070279fab7dd56 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:36:34 +0000 Subject: [PATCH 33/34] Version Packages --- .changeset/lexicon-prefix-init.md | 5 ----- packages/contrail/CHANGELOG.md | 6 ++++++ packages/contrail/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/lexicon-prefix-init.md diff --git a/.changeset/lexicon-prefix-init.md b/.changeset/lexicon-prefix-init.md deleted file mode 100644 index 2e7014f..0000000 --- a/.changeset/lexicon-prefix-init.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@atmo-dev/contrail": minor ---- - -Add verified Lexicon-prefix initialization with CID-pinned dependency bundles, automatic safe query fields, and optional reference/relation prompts. diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index e1ed69b..448dff6 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail +## 0.23.0 + +### Minor Changes + +- e050b0b: Add verified Lexicon-prefix initialization with CID-pinned dependency bundles, automatic safe query fields, and optional reference/relation prompts. + ## 0.22.1 ### Patch Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index c7246b7..9b3e037 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.22.1", + "version": "0.23.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, -- 2.51.2 From 1806573a29d94a1ff43ab417a745c34fd0e10148 Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Sat, 12 Sep 2026 20:38:46 -0400 Subject: [PATCH 34/34] Regenerate atmo-rsvp lexicon config for 0.23 CI caught what my local check missed. I verified the two packages I touched; CI runs typecheck across all nine workspaces, and apps/atmo-rsvp failed with "generated Contrail Lexicons are stale: lex.config.js". 0.23's generator emits an extra lexicons/pinned/**/*.json glob, so any config generated before the bump is stale by definition. The branch was written against 0.18, so merging main made it stale. One line, produced by running the generator rather than hand-edited. Verified across the whole workspace this time, not just the two packages: pnpm typecheck 10/10 tasks, pnpm build clean, pnpm test 5/5 tasks with contrail 598 passed / 3 skipped. --- apps/atmo-rsvp/lex.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/atmo-rsvp/lex.config.js b/apps/atmo-rsvp/lex.config.js index 2447175..f2233cd 100644 --- a/apps/atmo-rsvp/lex.config.js +++ b/apps/atmo-rsvp/lex.config.js @@ -5,6 +5,7 @@ export default defineLexiconConfig({ generate: { files: [ "lexicons/custom/**/*.json", + "lexicons/pinned/**/*.json", "lexicons/pulled/**/*.json", "lexicons/generated/**/*.json", ],