diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md new file mode 100644 index 0000000..6a3e1e5 --- /dev/null +++ b/.changeset/durable-projection-log.md @@ -0,0 +1,7 @@ +--- +"@atmo-dev/contrail": minor +--- + +Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. + +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 8a1e40d..99614f1 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -72,6 +72,32 @@ After a write to a user's PDS, `contrail.notify(uri)` can fetch the authoritativ Contrail stores source event time, repository revision, source cursor, CID, and local index time separately from record/application time. Durable tombstones reject stale resurrection, and live Jetstream projection commits its exact yielded cursor in the same transaction. A successful PDS `listRecords` page is a current authoritative observation, so it supersedes older durable state without a redundant version read; its version writes and page cursor still commit atomically. Tombstones are retained indefinitely; authoritative rebuild/retention tooling is planned separately. +Projection winner selection is guarded again inside the write transaction. Overlapping cron, persistent, notify, or backfill writers cannot commit a stale canonical row, derived projection, tombstone, or source checkpoint; a changed predecessor rolls the complete attempt back and retries from fresh durable state. + +## Transactional change log (experimental) + +Fresh empty generations may opt into a compact transactional projection change log: + +```ts +const config = { + // ... + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "current", + requiredForActivation: true, + }, + }, + }, +} satisfies ContrailConfig; +``` + +Static definitions contain no handlers, URLs, clients, credentials, or secrets. Contrail registers them with a random database-generation ID and collection/phase coverage ledger. A winning logical put/delete appends one compact URI/version reference in the same transaction as canonical and derived state plus the source checkpoint. Duplicate, stale, same-CID, absent-delete, rejected, and rolled-back mutations append nothing. Record bodies are hydrated from current state by the later delivery layer rather than copied into the log. + +This milestone intentionally exposes only the atomic log foundation; leased claim/hydrate/ack delivery and current-state bootstrap APIs follow separately. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. + ## Local development A project containing only `contrail.config.ts` can start a complete local service: diff --git a/packages/contrail/src/adapters/postgres.ts b/packages/contrail/src/adapters/postgres.ts index 310be5d..32586e2 100644 --- a/packages/contrail/src/adapters/postgres.ts +++ b/packages/contrail/src/adapters/postgres.ts @@ -16,6 +16,21 @@ const BIGINT_COLUMNS = new Set([ "last_seen_at", "resolved_at", "total", + "revision", + "head_position", + "retained_floor_position", + "position", + "acknowledged_position", + "bootstrap_anchor_position", + "bootstrap_target_position", + "from_position", + "through_position", + "lease_expires_at", + "next_attempt_at", + "last_success_at", + "last_error_at", + "created_at", + "updated_at", ]); function normalizeRow(row: any): any { diff --git a/packages/contrail/src/adapters/sqlite.ts b/packages/contrail/src/adapters/sqlite.ts index 34b2511..57dee1e 100644 --- a/packages/contrail/src/adapters/sqlite.ts +++ b/packages/contrail/src/adapters/sqlite.ts @@ -9,6 +9,7 @@ interface SqliteStatement extends Statement { export function createSqliteDatabase(path: string): Database { const raw = new DatabaseSync(path); raw.exec("PRAGMA journal_mode = WAL"); + raw.exec("PRAGMA busy_timeout = 5000"); function wrapStatement( sql: string, diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index f4d799f..e84ad58 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -429,6 +429,7 @@ async function backfillUserAttempt( ) .bind(nextCursor ?? null, pageDone ? 1 : 0, now, did, collection); const result = await ingestRecords(db, events, config, { + phase: "historical", skipReplayDetection: options?.skipReplayDetection, skipFeedFanout: true, knownDids: options?.knownDids, diff --git a/packages/contrail/src/core/bootstrap.ts b/packages/contrail/src/core/bootstrap.ts index 7071cbe..e4e5757 100644 --- a/packages/contrail/src/core/bootstrap.ts +++ b/packages/contrail/src/core/bootstrap.ts @@ -496,6 +496,7 @@ export class DatabaseBootstrapTarget implements BootstrapTarget { ): Promise { const knownDids = await this.getKnownDids(); const result = await ingestRecords(this.db, events, this.config, { + phase: "historical", knownDids, skipDerivedProjections: this.options.deferDerivedProjections === true, authoritativeSourceObservation, diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts new file mode 100644 index 0000000..6505ce9 --- /dev/null +++ b/packages/contrail/src/core/change-log.ts @@ -0,0 +1,520 @@ +import type { SqlDialect } from "./dialect"; +import type { + ContrailConfig, + Database, + IngestEvent, + ProjectionPhase, + Statement, +} from "./types"; +import { + canonicalChangeDefinitions, + changeConsumerPhases, + changeLogCoverage, + changesEnabled, +} from "./types"; + +export const MAX_CHANGE_BATCH_CHANGES = 500; +export const MAX_CHANGE_BATCH_BYTES = 512_000; + +export interface RecordChange { + id: string; + kind: "record"; + operation: "put" | "delete"; + uri: string; + did: string; + collection: string; + rkey: string; + cid: string | null; + version: { + sourceId: string; + sourceEpoch: string | null; + sourceRevision: string | null; + sourceTimeUs: number; + sourceCursor: string | null; + }; +} + +type StoredRecordChange = Omit; + +export interface ChangeLogState { + generation: string; + head: string; + retainedFloor: string; + createdAt: number; +} + +interface ChangeLogStateRow { + generation_id: string; + head_position: number | string; + retained_floor_position: number | string; + definitions_json: string; + created_at: number | string; +} + +interface ChangeConsumerRow { + consumer_id: string; + generation_id: string; + acknowledged_position: number | string; + configured_collections_json: string; + configured_phases_json: string; + initial_mode: string; + required_for_activation: number | string; + bootstrap_state: string; + bootstrap_anchor_position: number | string | null; +} + +interface ChangeCoverageRow { + generation_id: string; + collection: string; + phase: ProjectionPhase; + from_position: number | string; + through_position: number | string | null; +} + +export interface ChangeLogSchemaProbe { + exists: boolean; + state: ChangeLogStateRow | null; +} + +/** Optional physical schema. It is absent when no consumers are configured. */ +export function buildChangeLogSchema( + config: ContrailConfig, + dialect: SqlDialect, +): string[] { + if (!changesEnabled(config)) return []; + const bigint = dialect.bigintType; + return [ + `CREATE TABLE IF NOT EXISTS change_log_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + generation_id TEXT NOT NULL, + head_position ${bigint} NOT NULL, + retained_floor_position ${bigint} NOT NULL, + definitions_json TEXT NOT NULL, + created_at ${bigint} NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS change_batches ( + generation_id TEXT NOT NULL, + position ${bigint} NOT NULL, + projection_transaction_id TEXT NOT NULL, + source_id TEXT NOT NULL, + source_epoch TEXT, + source_cursor TEXT, + phase TEXT NOT NULL CHECK (phase IN ('historical', 'live')), + changes_json TEXT NOT NULL, + change_count INTEGER NOT NULL, + created_at ${bigint} NOT NULL, + PRIMARY KEY (generation_id, position) + )`, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_change_batches_transaction ON change_batches(generation_id, projection_transaction_id)", + "CREATE INDEX IF NOT EXISTS idx_change_batches_created ON change_batches(generation_id, created_at)", + `CREATE TABLE IF NOT EXISTS change_consumers ( + consumer_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + acknowledged_position ${bigint} NOT NULL, + configured_collections_json TEXT NOT NULL, + configured_phases_json TEXT NOT NULL, + initial_mode TEXT NOT NULL CHECK (initial_mode IN ('current', 'future', 'history')), + required_for_activation INTEGER NOT NULL DEFAULT 0, + bootstrap_state TEXT NOT NULL CHECK (bootstrap_state IN ('pending', 'scanning', 'catching-up', 'activating', 'ready', 'error', 'reset-required')), + bootstrap_anchor_position ${bigint}, + bootstrap_scan_collection TEXT, + bootstrap_scan_cursor TEXT, + bootstrap_target_position ${bigint}, + bootstrap_token TEXT, + lease_owner TEXT, + lease_expires_at ${bigint}, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at ${bigint}, + last_success_at ${bigint}, + last_error_code TEXT, + last_error_at ${bigint}, + updated_at ${bigint} NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS change_log_coverage ( + generation_id TEXT NOT NULL, + collection TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('historical', 'live')), + from_position ${bigint} NOT NULL, + through_position ${bigint}, + PRIMARY KEY (generation_id, collection, phase, from_position) + )`, + ]; +} + +function missingTable(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + if ((error as { code?: unknown }).code === "42P01") return true; + return /no such table|does not exist/i.test( + String((error as { message?: unknown }).message ?? ""), + ); +} + +export async function probeChangeLogSchema( + db: Database, +): Promise { + try { + const state = await db + .prepare( + `SELECT generation_id, head_position, retained_floor_position, + definitions_json, created_at + FROM change_log_state WHERE id = 1`, + ) + .first(); + return { exists: true, state }; + } catch (error) { + if (missingTable(error)) return { exists: false, state: null }; + throw error; + } +} + +/** Enabling the first log without an old-writer quiet boundary is supported + * only on an empty projection generation in this milestone. */ +export async function assertFreshChangeLogGeneration( + db: Database, +): Promise { + const row = await db + .prepare( + `SELECT CASE WHEN + EXISTS (SELECT 1 FROM record_versions LIMIT 1) OR + EXISTS (SELECT 1 FROM cursor LIMIT 1) OR + EXISTS (SELECT 1 FROM source_position LIMIT 1) OR + EXISTS (SELECT 1 FROM bootstrap_state LIMIT 1) OR + EXISTS (SELECT 1 FROM backfills LIMIT 1) OR + EXISTS (SELECT 1 FROM discovery LIMIT 1) OR + EXISTS (SELECT 1 FROM identities LIMIT 1) + THEN 1 ELSE 0 END AS active`, + ) + .first<{ active: number | string }>(); + if (Number(row?.active ?? 0) !== 0) { + throw new Error( + "Transactional change logging can currently be enabled only on a fresh empty generation; build a fresh generation instead of racing existing projection writers", + ); + } +} + +function canonicalCollections(collections: string[]): string { + return JSON.stringify([...collections].sort()); +} + +function canonicalPhases(phases: ProjectionPhase[]): string { + return JSON.stringify([...phases].sort()); +} + +/** Initialize one immutable milestone-1 logging definition. Later milestones + * add explicit quiet-boundary operations for changing this durable definition. */ +export async function initializeChangeLog( + db: Database, + config: ContrailConfig, +): Promise { + if (!changesEnabled(config)) return; + + const definitions = canonicalChangeDefinitions(config); + const now = Date.now(); + const candidateGeneration = crypto.randomUUID(); + const statements: Statement[] = [ + db + .prepare( + `INSERT INTO change_log_state + (id, generation_id, head_position, retained_floor_position, + definitions_json, created_at) + VALUES (1, ?, 0, 0, ?, ?) + ON CONFLICT(id) DO NOTHING`, + ) + .bind(candidateGeneration, definitions, now), + ]; + + for (const [consumerId, consumer] of Object.entries( + config.changes?.consumers ?? {}, + ).sort(([left], [right]) => left.localeCompare(right))) { + const initialReady = consumer.initial === "current" ? "pending" : "ready"; + statements.push( + db + .prepare( + `INSERT INTO change_consumers + (consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + bootstrap_anchor_position, attempts, updated_at) + SELECT ?, generation_id, head_position, ?, ?, ?, ?, ?, + CASE WHEN ? = 'current' THEN head_position ELSE NULL END, + 0, ? + FROM change_log_state + WHERE id = 1 AND definitions_json = ? + ON CONFLICT(consumer_id) DO NOTHING`, + ) + .bind( + consumerId, + canonicalCollections(consumer.collections), + canonicalPhases(changeConsumerPhases(consumer)), + consumer.initial, + consumer.requiredForActivation === true ? 1 : 0, + initialReady, + consumer.initial, + now, + definitions, + ), + ); + } + + for (const item of changeLogCoverage(config)) { + statements.push( + db + .prepare( + `INSERT INTO change_log_coverage + (generation_id, collection, phase, from_position, through_position) + SELECT generation_id, ?, ?, head_position, NULL + FROM change_log_state + WHERE id = 1 AND definitions_json = ? + ON CONFLICT(generation_id, collection, phase, from_position) + DO NOTHING`, + ) + .bind(item.collection, item.phase, definitions), + ); + } + + // Keep initialization under conservative D1 statement limits. The durable + // definitions_json winner makes these resumable chunks safe under concurrent + // initialization; init does not return until the complete set verifies. + await db.batch(statements.slice(0, 1)); + for (let index = 1; index < statements.length; index += 50) { + await db.batch(statements.slice(index, index + 50)); + } + await assertChangeLogDefinition(db, config); +} + +async function assertChangeLogDefinition( + db: Database, + config: ContrailConfig, +): Promise { + const state = (await probeChangeLogSchema(db)).state; + const definitions = canonicalChangeDefinitions(config); + if (!state || state.definitions_json !== definitions) { + throw new Error( + "Durable change consumer definitions differ from configuration; changing consumers or coverage requires a fresh generation in this milestone", + ); + } + + const consumers = await db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + bootstrap_anchor_position + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + const expectedConsumers = Object.entries(config.changes?.consumers ?? {}).sort( + ([left], [right]) => left.localeCompare(right), + ); + if (consumers.results.length !== expectedConsumers.length) { + throw new Error("Durable change consumer registration is incomplete"); + } + for (let index = 0; index < expectedConsumers.length; index++) { + const [id, expected] = expectedConsumers[index]!; + const actual = consumers.results[index]!; + const bootstrapState = expected.initial === "current" ? "pending" : "ready"; + if ( + actual.consumer_id !== id || + actual.generation_id !== state.generation_id || + actual.configured_collections_json !== canonicalCollections(expected.collections) || + actual.configured_phases_json !== canonicalPhases(changeConsumerPhases(expected)) || + actual.initial_mode !== expected.initial || + Number(actual.required_for_activation) !== + (expected.requiredForActivation === true ? 1 : 0) || + actual.bootstrap_state !== bootstrapState || + Number(actual.acknowledged_position) !== 0 || + (expected.initial === "current" + ? Number(actual.bootstrap_anchor_position) !== 0 + : actual.bootstrap_anchor_position !== null) + ) { + throw new Error(`Durable change consumer ${id} is incompatible`); + } + } + + const coverage = await db + .prepare( + `SELECT generation_id, collection, phase, from_position, through_position + FROM change_log_coverage + ORDER BY collection, phase, from_position`, + ) + .all(); + const expectedCoverage = changeLogCoverage(config); + if (coverage.results.length !== expectedCoverage.length) { + throw new Error("Durable change-log coverage is incomplete"); + } + for (let index = 0; index < expectedCoverage.length; index++) { + const actual = coverage.results[index]!; + const expected = expectedCoverage[index]!; + if ( + actual.generation_id !== state.generation_id || + actual.collection !== expected.collection || + actual.phase !== expected.phase || + Number(actual.from_position) !== 0 || + actual.through_position !== null + ) { + throw new Error("Durable change-log coverage is incompatible"); + } + } +} + +export async function getChangeLogState( + db: Database, +): Promise { + const probe = await probeChangeLogSchema(db); + if (!probe.state) return null; + return { + generation: probe.state.generation_id, + head: String(probe.state.head_position), + retainedFloor: String(probe.state.retained_floor_position), + createdAt: Number(probe.state.created_at), + }; +} + +function bounded(value: string | null, label: string, maximum: number): void { + if (value !== null && value.length > maximum) { + throw new Error(`${label} exceeds ${maximum} characters`); + } +} + +function logicalChanges( + events: IngestEvent[], + existing: ReadonlyMap, + config: ContrailConfig, + phase: ProjectionPhase, +): StoredRecordChange[] { + const covered = new Set( + changeLogCoverage(config) + .filter((item) => item.phase === phase) + .map((item) => item.collection), + ); + if (covered.size === 0) return []; + + // projectEvents already selects one source winner per URI. Keep this final + // reduction defensive for direct internal callers. + const final = new Map(); + for (const event of events) final.set(event.uri, event); + + const changes: StoredRecordChange[] = []; + for (const event of final.values()) { + if (!covered.has(event.collection)) continue; + const prior = existing.get(event.uri); + const deleted = event.operation === "delete"; + const visibleChange = deleted + ? prior !== undefined + : prior === undefined || + prior.cid !== event.cid || + (event.cid === null && prior.record !== event.record); + if (!visibleChange) continue; + + const source = event.source; + const change: StoredRecordChange = { + kind: "record", + operation: deleted ? "delete" : "put", + uri: event.uri, + did: event.did, + collection: event.collection, + rkey: event.rkey, + cid: deleted ? null : event.cid, + version: { + sourceId: source?.id ?? "legacy-caller", + sourceEpoch: source?.epoch ?? null, + sourceRevision: source?.revision ?? null, + sourceTimeUs: source?.time_us ?? event.time_us, + sourceCursor: source?.cursor ?? null, + }, + }; + bounded(change.uri, "change URI", 2_048); + bounded(change.did, "change DID", 2_048); + bounded(change.collection, "change collection", 512); + bounded(change.rkey, "change rkey", 512); + bounded(change.cid, "change CID", 512); + bounded(change.version.sourceId, "change source ID", 128); + bounded(change.version.sourceEpoch, "change source epoch", 256); + bounded(change.version.sourceRevision, "change source revision", 2_048); + bounded(change.version.sourceCursor, "change source cursor", 2_048); + if ( + !Number.isSafeInteger(change.version.sourceTimeUs) || + change.version.sourceTimeUs < 0 + ) { + throw new Error("change source time must be a non-negative safe integer"); + } + changes.push(change); + } + return changes; +} + +function commonValue( + values: Array, +): string | null { + if (values.length === 0) return null; + const first = values[0]!; + return values.every((value) => value === first) ? first : null; +} + +/** Statements appended after canonical/derived projection and before source + * checkpoints in the same database transaction. */ +export function appendChangeLogStatements( + db: Database, + events: IngestEvent[], + existing: ReadonlyMap, + config: ContrailConfig, + phase: ProjectionPhase, +): Statement[] { + if (!changesEnabled(config)) return []; + const changes = logicalChanges(events, existing, config, phase); + if (changes.length === 0) return []; + if (changes.length > MAX_CHANGE_BATCH_CHANGES) { + throw new Error( + `Projection change batch contains ${changes.length} changes; maximum is ${MAX_CHANGE_BATCH_CHANGES}`, + ); + } + const serialized = JSON.stringify(changes); + const bytes = new TextEncoder().encode(serialized).byteLength; + if (bytes > MAX_CHANGE_BATCH_BYTES) { + throw new Error( + `Projection change batch contains ${bytes} encoded bytes; maximum is ${MAX_CHANGE_BATCH_BYTES}`, + ); + } + + const sourceIds = changes.map((change) => change.version.sourceId); + const sourceId = commonValue(sourceIds) ?? "mixed"; + const sourceEpoch = commonValue( + changes.map((change) => change.version.sourceEpoch), + ); + const sourceCursor = commonValue( + changes.map((change) => change.version.sourceCursor), + ); + const transactionId = crypto.randomUUID(); + const now = Date.now(); + + return [ + db + .prepare( + `UPDATE change_log_state + SET head_position = head_position + 1 + WHERE id = 1`, + ), + db + .prepare( + `INSERT INTO change_batches + (generation_id, position, projection_transaction_id, source_id, + source_epoch, source_cursor, phase, changes_json, change_count, + created_at) + VALUES ( + (SELECT generation_id FROM change_log_state WHERE id = 1), + (SELECT head_position FROM change_log_state WHERE id = 1), + ?, ?, ?, ?, ?, ?, ?, ? + )`, + ) + .bind( + transactionId, + sourceId, + sourceEpoch, + sourceCursor, + phase, + serialized, + changes.length, + now, + ), + ]; +} diff --git a/packages/contrail/src/core/constellation.ts b/packages/contrail/src/core/constellation.ts index 221bd88..1c4495d 100644 --- a/packages/contrail/src/core/constellation.ts +++ b/packages/contrail/src/core/constellation.ts @@ -184,6 +184,7 @@ export async function backfillFollowersFromConstellation( if (events.length > 0) { known.add(subjectDid); const ingest = await ingestRecords(db, events, config, { + phase: "historical", knownDids: known, }); inserted += ingest.accepted.length; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index e06cac0..e74fb60 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -8,6 +8,7 @@ import type { RecordRow, RecordSource, OrderedSourceConfig, + ProjectionPhase, } from "../types"; import { getNestedValue, @@ -22,6 +23,7 @@ import { normalizeFeedTarget, feedTargetMaxItems, DEFAULT_FOLLOW_SHORT, + changesEnabled, } from "../types"; import { getSearchableFields, @@ -35,6 +37,7 @@ import { sqliteFtsContentExpression, } from "../dialect"; import type { SourcePosition } from "../sources"; +import { appendChangeLogStatements } from "../change-log"; // --- Counts --- @@ -700,9 +703,13 @@ export interface RecordVersionInfo { source_time_us: number; source_cursor: string | null; indexed_at: number; + /** Opaque optimistic-concurrency token; not part of source ordering. */ + projection_token: string; } -function versionForEvent(event: IngestEvent): RecordVersionInfo { +type ComparableRecordVersion = Omit; + +function versionForEvent(event: IngestEvent): ComparableRecordVersion { const source = event.source; return { uri: event.uri, @@ -742,8 +749,8 @@ function operationRank(operation: RecordVersionInfo["operation"]): number { * tie-breakers; notably a delete wins an exact tie so replay cannot resurrect it. */ export function compareRecordVersions( - left: RecordVersionInfo, - right: RecordVersionInfo, + left: ComparableRecordVersion, + right: ComparableRecordVersion, ): number { if ( left.source_revision !== null && @@ -793,7 +800,7 @@ export async function lookupRecordVersions( const placeholders = chunk.map(() => "?").join(","); const rows = await db .prepare( - `SELECT uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at FROM record_versions WHERE uri IN (${placeholders})`, + `SELECT uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at, projection_token FROM record_versions WHERE uri IN (${placeholders})`, ) .bind(...chunk) .all(); @@ -807,13 +814,13 @@ export interface MutationSelection { superseded: number; } -function selectMutationWinners( +export function selectMutationWinners( events: IngestEvent[], durable: ReadonlyMap, ): MutationSelection { const winners = new Map< string, - { event: IngestEvent; version: RecordVersionInfo; index: number } + { event: IngestEvent; version: ComparableRecordVersion; index: number } >(); let superseded = 0; @@ -918,10 +925,11 @@ const RECORD_UPSERT_BINDINGS = 7; const RECORD_UPSERT_ROWS = Math.floor( MAX_STATEMENT_BINDINGS / RECORD_UPSERT_BINDINGS ); -const RECORD_VERSION_BINDINGS = 12; +const RECORD_VERSION_BINDINGS = 13; const RECORD_VERSION_ROWS = Math.floor( MAX_STATEMENT_BINDINGS / RECORD_VERSION_BINDINGS, ); +const PROJECTION_GUARD_URIS = 40; interface StorageMutation { event: IngestEvent; @@ -932,17 +940,18 @@ function buildRecordVersionStatements( db: Database, events: IngestEvent[], existing: Map, + projectionTokens: ReadonlyMap, ): Statement[] { const statements: Statement[] = []; for (let index = 0; index < events.length; index += RECORD_VERSION_ROWS) { const chunk = events.slice(index, index + RECORD_VERSION_ROWS); const values = chunk - .map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + .map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") .join(", "); statements.push( db .prepare( - `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at) VALUES ${values} ON CONFLICT(uri) DO UPDATE SET did = excluded.did, collection = excluded.collection, rkey = excluded.rkey, operation = excluded.operation, cid = excluded.cid, source_id = excluded.source_id, source_epoch = excluded.source_epoch, source_revision = excluded.source_revision, source_time_us = excluded.source_time_us, source_cursor = excluded.source_cursor, indexed_at = excluded.indexed_at`, + `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at, projection_token) VALUES ${values} ON CONFLICT(uri) DO UPDATE SET did = excluded.did, collection = excluded.collection, rkey = excluded.rkey, operation = excluded.operation, cid = excluded.cid, source_id = excluded.source_id, source_epoch = excluded.source_epoch, source_revision = excluded.source_revision, source_time_us = excluded.source_time_us, source_cursor = excluded.source_cursor, indexed_at = excluded.indexed_at, projection_token = excluded.projection_token`, ) .bind( ...chunk.flatMap((event) => { @@ -964,6 +973,7 @@ function buildRecordVersionStatements( version.source_time_us, version.source_cursor, version.indexed_at, + projectionTokens.get(event.uri)!, ]; }), ), @@ -1024,6 +1034,71 @@ function buildRecordMutationStatements( return statements; } +function buildProjectionGuardStatements( + db: Database, + events: IngestEvent[], + predecessors: ReadonlyMap, +): Statement[] { + const uris = [...new Set(events.map((event) => event.uri))]; + const statements: Statement[] = [ + db.prepare( + `INSERT INTO _contrail_projection_state (id, revision, guard) + VALUES (1, 0, 1) ON CONFLICT(id) DO NOTHING`, + ), + // PostgreSQL takes a row lock here. The following statement then receives a + // fresh READ COMMITTED snapshot after any earlier projector commits. SQLite + // and D1 already serialize the containing write batch. + db.prepare( + `UPDATE _contrail_projection_state + SET revision = revision + 1 + WHERE id = 1`, + ), + ]; + for (let index = 0; index < uris.length; index += PROJECTION_GUARD_URIS) { + const chunk = uris.slice(index, index + PROJECTION_GUARD_URIS); + const conditions: string[] = []; + const bindings: string[] = []; + for (const uri of chunk) { + const predecessor = predecessors.get(uri); + if (!predecessor) { + conditions.push( + "NOT EXISTS (SELECT 1 FROM record_versions WHERE uri = ?)", + ); + bindings.push(uri); + continue; + } + if (!predecessor.projection_token) { + throw new Error(`Record version ${uri} has no projection token`); + } + conditions.push( + "EXISTS (SELECT 1 FROM record_versions WHERE uri = ? AND projection_token = ?)", + ); + bindings.push(uri, predecessor.projection_token); + } + statements.push( + db + .prepare( + `UPDATE _contrail_projection_state + SET guard = CASE WHEN ${conditions.join(" AND ")} THEN 1 ELSE 0 END + WHERE id = 1`, + ) + .bind(...bindings), + ); + } + return statements; +} + +/** Adapter-neutral classification for the named optimistic guard constraint. */ +export function isProjectionConflictError(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const candidate = error as { code?: unknown; constraint?: unknown; message?: unknown }; + return ( + (candidate.code === "23514" && + candidate.constraint === "projection_guard_valid") || + /projection_guard_valid/i.test(String(candidate.message ?? "")) + ); +} + export async function projectEvents( db: Database, events: IngestEvent[], @@ -1033,19 +1108,26 @@ export async function projectEvents( skipFeedFanout?: boolean; /** Skip FTS and relation-count maintenance during canonical bulk loading. */ skipDerivedProjections?: boolean; - /** Pre-fetched existing records — skips the internal lookup when provided */ + /** @deprecated Existing rows are re-read for transaction conflict safety. */ existing?: Map; /** Statements committed after projection in the same database batch. */ trailingStatements?: Statement[]; /** Internal: ingestRecords already checked durable source order. */ sourceOrderingChecked?: boolean; + /** Durable versions observed while selecting source winners. */ + predecessors?: ReadonlyMap; + /** Acquisition phase persisted on an optional change batch. */ + phase?: ProjectionPhase; }, ): Promise { if (events.length === 0) return { applied: [], superseded: 0 }; + const predecessors = + options?.predecessors ?? + (await lookupRecordVersions(db, events.map((event) => event.uri))); const selection = options?.sourceOrderingChecked ? { applied: events, superseded: 0 } - : await selectCurrentMutations(db, events); + : selectMutationWinners(events, predecessors); events = selection.applied; if (events.length === 0) { if (options?.trailingStatements?.length) { @@ -1062,17 +1144,19 @@ export async function projectEvents( (relation) => relation.count !== false ) ); - const needRecordContent = followCollections.length > 0 || hasCountingRelations; - - // Use pre-fetched data or look up existing records - let existingMap: Map; - if (options?.existing) { - existingMap = options.existing; - } else if (!options?.skipReplayDetection) { - existingMap = await lookupExistingRecords(db, events, needRecordContent, config); - } else { - existingMap = new Map(); - } + const needRecordContent = + followCollections.length > 0 || + hasCountingRelations || + changesEnabled(config); + + // Existing state must be read after predecessor selection. A caller-provided + // map can predate that selection and would make derived changes incorrect + // even when the optimistic token guard itself succeeds. + const needExistingState = + !options?.skipReplayDetection || needRecordContent || changesEnabled(config); + const existingMap = needExistingState + ? await lookupExistingRecords(db, events, needRecordContent, config) + : new Map(); const batch: Statement[] = []; // Keep only the final storage mutation for a URI within this atomic batch. @@ -1122,15 +1206,36 @@ export async function projectEvents( } } - // Storage and durable version/tombstone metadata run first so FTS, feeds, - // and count statements in the same atomic batch observe the final records. + const projectionTokens = new Map( + events.map((event) => [event.uri, crypto.randomUUID()] as const), + ); + + // Lock, verify the exact durable predecessors selected by the caller, then + // write storage and version metadata. Any changed token violates the named + // guard constraint and rolls the complete database batch back. batch.unshift( + ...buildProjectionGuardStatements(db, events, predecessors), ...buildRecordMutationStatements(db, storageMutations.values()), - ...buildRecordVersionStatements(db, events, existingMap), + ...buildRecordVersionStatements( + db, + events, + existingMap, + projectionTokens, + ), ); - // Build deduplicated count statements — one UPDATE per unique target. + // Build deduplicated count statements, then append the compact change batch. + // Caller-provided source checkpoints deliberately remain last. batch.push(...buildBatchCountStatements(db, config, countTargets)); + batch.push( + ...appendChangeLogStatements( + db, + events, + existingMap, + config, + options?.phase ?? "live", + ), + ); if (options?.trailingStatements?.length) { batch.push(...options.trailingStatements); } diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 95fa42d..51dfe2f 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -25,9 +25,19 @@ import { getSearchableFields, } from "../search"; import { buildLabelsSchema } from "../labels/schema"; +import { + assertFreshChangeLogGeneration, + buildChangeLogSchema, + initializeChangeLog, + probeChangeLogSchema, +} from "../change-log"; +import { + canonicalChangeDefinitions, + changesEnabled, +} from "../types"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 11; +export const CONTRAIL_SCHEMA_VERSION = 12; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -130,7 +140,8 @@ CREATE TABLE IF NOT EXISTS record_versions ( source_revision TEXT, source_time_us ${dialect.bigintType} NOT NULL, source_cursor TEXT, - indexed_at ${dialect.bigintType} NOT NULL + indexed_at ${dialect.bigintType} NOT NULL, + projection_token TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_record_versions_collection ON record_versions(collection); CREATE INDEX IF NOT EXISTS idx_record_versions_did ON record_versions(did); @@ -140,6 +151,12 @@ CREATE TABLE IF NOT EXISTS ingest_diagnostics ( total ${dialect.bigintType} NOT NULL, last_seen_at ${dialect.bigintType} NOT NULL ); +CREATE TABLE IF NOT EXISTS _contrail_projection_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision ${dialect.bigintType} NOT NULL DEFAULT 0, + guard INTEGER NOT NULL DEFAULT 1 + CONSTRAINT projection_guard_valid CHECK (guard = 1) +); `; } @@ -576,6 +593,11 @@ const MIGRATIONS: MigrationOp[] = [ column: "source_epoch", columnDef: "TEXT", }, + { + table: "record_versions", + column: "projection_token", + columnDef: "TEXT", + }, { table: "discovery", column: "retries", @@ -639,8 +661,8 @@ async function seedLegacyRecordVersions( const table = recordsTableName(shortName); await db .prepare( - `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at) - SELECT uri, did, ?, rkey, 'update', cid, 'legacy', NULL, NULL, indexed_at, NULL, indexed_at FROM ${table} + `INSERT INTO record_versions (uri, did, collection, rkey, operation, cid, source_id, source_epoch, source_revision, source_time_us, source_cursor, indexed_at, projection_token) + SELECT uri, did, ?, rkey, 'update', cid, 'legacy', NULL, NULL, indexed_at, NULL, indexed_at, uri FROM ${table} WHERE 1 = 1 ON CONFLICT(uri) DO NOTHING`, ) .bind(collection.collection) @@ -671,6 +693,7 @@ function schemaFingerprint( indexes: string[]; feeds: string[]; fts: string[]; + changes: string[]; }, ): string { return hashStrings([ @@ -682,6 +705,8 @@ function schemaFingerprint( ...ddl.indexes, ...ddl.feeds, ...ddl.fts, + ...ddl.changes, + canonicalChangeDefinitions(config), ...buildCountColumns(config), ...(config.labels ? buildLabelsSchema(dialect) : []), JSON.stringify(MIGRATIONS), @@ -702,12 +727,14 @@ export async function initSchema( const indexes = buildDynamicIndexes(config, dialect); const feeds = buildFeedTables(config, dialect); const fts = buildFtsTables(config, dialect); + const changes = buildChangeLogSchema(config, dialect); const fingerprint = schemaFingerprint(config, dialect, { base, collections, indexes, feeds, fts, + changes, }); if ((await getMeta(db, SCHEMA_FINGERPRINT_KEY)) === fingerprint) { @@ -715,6 +742,13 @@ export async function initSchema( return; } + const priorChangeLog = await probeChangeLogSchema(db); + if (!changesEnabled(config) && priorChangeLog.exists) { + throw new Error( + "Durable change logging cannot be disabled or removed during ordinary initialization", + ); + } + for (const statement of [...base, ...collections, ...indexes, ...feeds]) { await runIdempotentDdl(db, statement); } @@ -729,9 +763,29 @@ export async function initSchema( const hasFeeds = !!(config.feeds && Object.keys(config.feeds).length > 0); await runMigrations(db, hasFeeds); + await db + .prepare( + "UPDATE record_versions SET projection_token = uri WHERE projection_token IS NULL", + ) + .run(); + await db + .prepare( + `INSERT INTO _contrail_projection_state (id, revision, guard) + VALUES (1, 0, 1) ON CONFLICT(id) DO NOTHING`, + ) + .run(); await applyCountColumns(db, config); await seedLegacyRecordVersions(db, config); + if (changesEnabled(config)) { + const concurrentChangeLog = await probeChangeLogSchema(db); + if (!priorChangeLog.state && !concurrentChangeLog.state) { + await assertFreshChangeLogGeneration(db); + } + for (const statement of changes) await runIdempotentDdl(db, statement); + await initializeChangeLog(db, config); + } + for (const apply of options.extraSchemas ?? []) await apply(db); await setMeta(db, SCHEMA_FINGERPRINT_KEY, fingerprint); } diff --git a/packages/contrail/src/core/ingest.ts b/packages/contrail/src/core/ingest.ts index 48527cf..fe1392c 100644 --- a/packages/contrail/src/core/ingest.ts +++ b/packages/contrail/src/core/ingest.ts @@ -4,6 +4,7 @@ import type { Database, IngestEvent, MutationSource, + ProjectionPhase, Statement, } from "./types"; import { @@ -11,9 +12,11 @@ import { resolveCollectionKey, } from "./types"; import { + isProjectionConflictError, + lookupRecordVersions, projectEvents, selectAuthoritativeMutations, - selectCurrentMutations, + selectMutationWinners, type ExistingRecordInfo, } from "./db/records"; import { @@ -114,6 +117,9 @@ export interface IngestRecordsOptions { /** The source response is a current authoritative snapshot, so it supersedes * durable observations without a redundant version lookup. */ authoritativeSourceObservation?: boolean; + /** Acquisition phase for optional durable consumers. Defaults to live for + * backwards-compatible direct ingestRecords() calls. */ + phase?: ProjectionPhase; /** @internal Aggregate private diagnostics for one bulk run. The caller * flushes this bounded object once after concurrent page processing. */ aggregateDiagnostics?: IngestDiagnosticCounts; @@ -277,97 +283,131 @@ export async function ingestRecords( ); } - // Reject duplicate/stale source observations before they can admit dependent - // actors in this batch. The winning versions are persisted with projection. - const ordered = options.authoritativeSourceObservation - ? selectAuthoritativeMutations(accepted) - : await selectCurrentMutations(db, accepted); - dropped.superseded += ordered.superseded; + // Winner reads occur before db.batch on D1, so each projection carries the + // exact predecessor tokens it observed. A concurrent projector changes a + // token, the named transaction guard rolls everything back, and this loop + // repeats selection plus derived-state reads from fresh durable state. + const retryBase = { + unknownActor: dropped.unknownActor, + unknownSubject: dropped.unknownSubject, + superseded: dropped.superseded, + }; + const maximumAttempts = 5; + for (let attempt = 1; attempt <= maximumAttempts; attempt++) { + const attemptDropped: IngestDropCounts = { + ...dropped, + unknownActor: retryBase.unknownActor, + unknownSubject: retryBase.unknownSubject, + superseded: retryBase.superseded, + }; + const predecessors = await lookupRecordVersions( + db, + accepted.map((event) => event.uri), + ); + const ordered = options.authoritativeSourceObservation + ? selectAuthoritativeMutations(accepted) + : selectMutationWinners(accepted, predecessors); + attemptDropped.superseded += ordered.superseded; - const projectionExclusions = ordered.applied.filter((event) => - policyExcluded.has(event), - ); - const admitted = ordered.applied.filter((event) => !policyExcluded.has(event)); + const projectionExclusions = ordered.applied.filter((event) => + policyExcluded.has(event), + ); + const admitted = ordered.applied.filter( + (event) => !policyExcluded.has(event), + ); - const effectiveKnownDids = options.knownDids - ? new Set(options.knownDids) - : undefined; - const discoveredDids: string[] = []; - if (effectiveKnownDids) { + const effectiveKnownDids = options.knownDids + ? new Set(options.knownDids) + : undefined; + const discoveredDids: string[] = []; + if (effectiveKnownDids) { + for (const event of admitted) { + if (event.operation === "delete") continue; + const shortName = resolveCollectionKey(config, event.collection); + const collection = shortName ? config.collections[shortName] : undefined; + if (collection?.discover === false || effectiveKnownDids.has(event.did)) { + continue; + } + effectiveKnownDids.add(event.did); + discoveredDids.push(event.did); + } + } + + const actorFiltered: IngestEvent[] = []; for (const event of admitted) { - if (event.operation === "delete") continue; + if (event.operation === "delete" || !effectiveKnownDids) { + actorFiltered.push(event); + continue; + } const shortName = resolveCollectionKey(config, event.collection); const collection = shortName ? config.collections[shortName] : undefined; - if (collection?.discover === false || effectiveKnownDids.has(event.did)) { + if (collection?.discover !== false || effectiveKnownDids.has(event.did)) { + actorFiltered.push(event); continue; } - effectiveKnownDids.add(event.did); - discoveredDids.push(event.did); + attemptDropped.unknownActor++; } - } - const actorFiltered: IngestEvent[] = []; - for (const event of admitted) { - if (event.operation === "delete" || !effectiveKnownDids) { - actorFiltered.push(event); - continue; - } - const shortName = resolveCollectionKey(config, event.collection); - const collection = shortName ? config.collections[shortName] : undefined; - if (collection?.discover !== false || effectiveKnownDids.has(event.did)) { - actorFiltered.push(event); - continue; - } - dropped.unknownActor++; - } + const subjectFiltered = await filterUnknownSubjects( + db, + config, + actorFiltered, + effectiveKnownDids, + attemptDropped, + ); + const projectionEvents = [...subjectFiltered, ...projectionExclusions]; + const diagnosticCounts: IngestDiagnosticCounts = { + unknown_collection: attemptDropped.unknownCollection, + invalid_json: attemptDropped.invalidRecord, + lexicon_validation: attemptDropped.lexiconValidation, + cid_mismatch: attemptDropped.cidMismatch, + cid_encoding: attemptDropped.cidEncoding, + missing_cid: attemptDropped.missingCid, + record_filter: attemptDropped.recordFilter, + unknown_actor: attemptDropped.unknownActor, + unknown_subject: attemptDropped.unknownSubject, + superseded: attemptDropped.superseded, + }; + const diagnostics = options.aggregateDiagnostics + ? null + : ingestDiagnosticsStatement(db, diagnosticCounts); + const trailingStatements = [ + ...(diagnostics ? [diagnostics] : []), + ...(options.trailingStatements ?? []), + ]; - const subjectFiltered = await filterUnknownSubjects( - db, - config, - actorFiltered, - effectiveKnownDids, - dropped, - ); + try { + if (projectionEvents.length > 0) { + await projectEvents(db, projectionEvents, config, { + ...options, + phase: options.phase ?? "live", + // A pre-fetched visible-row map is not safe after a conflict; the + // projector deliberately reloads it under this predecessor attempt. + existing: undefined, + trailingStatements, + sourceOrderingChecked: true, + predecessors, + }); + } else if (trailingStatements.length > 0) { + await db.batch(trailingStatements); + } + } catch (error) { + if (isProjectionConflictError(error) && attempt < maximumAttempts) { + continue; + } + throw error; + } - const projectionEvents = [ - ...subjectFiltered, - ...projectionExclusions, - ]; - const diagnosticCounts: IngestDiagnosticCounts = { - unknown_collection: dropped.unknownCollection, - invalid_json: dropped.invalidRecord, - lexicon_validation: dropped.lexiconValidation, - cid_mismatch: dropped.cidMismatch, - cid_encoding: dropped.cidEncoding, - missing_cid: dropped.missingCid, - record_filter: dropped.recordFilter, - unknown_actor: dropped.unknownActor, - unknown_subject: dropped.unknownSubject, - superseded: dropped.superseded, - }; - const diagnostics = options.aggregateDiagnostics - ? null - : ingestDiagnosticsStatement(db, diagnosticCounts); - const trailingStatements = [ - ...(diagnostics ? [diagnostics] : []), - ...(options.trailingStatements ?? []), - ]; - if (projectionEvents.length > 0) { - await projectEvents(db, projectionEvents, config, { - ...options, - trailingStatements, - sourceOrderingChecked: true, - }); - } else if (trailingStatements.length > 0) { - await db.batch(trailingStatements); - } - // Aggregate only after the canonical projection/checkpoint transaction - // succeeds, so a rolled-back page cannot inflate private diagnostics. - if (options.aggregateDiagnostics) { - addIngestDiagnosticCounts(options.aggregateDiagnostics, diagnosticCounts); + Object.assign(dropped, attemptDropped); + // Aggregate only after the canonical projection/checkpoint transaction + // succeeds, so a rolled-back attempt cannot inflate private diagnostics. + if (options.aggregateDiagnostics) { + addIngestDiagnosticCounts(options.aggregateDiagnostics, diagnosticCounts); + } + return { accepted: subjectFiltered, dropped, discoveredDids }; } - return { accepted: subjectFiltered, dropped, discoveredDids }; + throw new Error("Projection conflict retry limit exhausted"); } function incrementValidationDrop( diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index b29c8d1..e1285e9 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -478,6 +478,7 @@ export async function runIngestCycle( const batch = events.slice(i, i + BATCH_SIZE); const isFinalBatch = i + BATCH_SIZE >= events.length; const result = await ingestRecords(db, batch, config, { + phase: "live", knownDids, // Earlier batches may commit without moving the cursor. A crash replays // them safely; the final batch atomically commits the exact source cursor. diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index 3b96cc6..877526b 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -161,6 +161,7 @@ async function streamAndFlush( let ingestResult: Awaited>; try { ingestResult = await ingestRecords(db, batch, config, { + phase: "live", knownDids, trailingStatements: [ saveCursorStatement(db, lastTimeUs), diff --git a/packages/contrail/src/core/router/notify.ts b/packages/contrail/src/core/router/notify.ts index 1f11282..3688a27 100644 --- a/packages/contrail/src/core/router/notify.ts +++ b/packages/contrail/src/core/router/notify.ts @@ -255,7 +255,7 @@ export async function processNotifyUris( } const appliedEvents = events.length > 0 - ? (await ingestRecords(db, events, config, { existing })).accepted + ? (await ingestRecords(db, events, config, { existing, phase: "live" })).accepted : []; // The shared ingest path fans these records into feed_items exactly like the cron and diff --git a/packages/contrail/src/core/router/profiles.ts b/packages/contrail/src/core/router/profiles.ts index abaa351..5bdf518 100644 --- a/packages/contrail/src/core/router/profiles.ts +++ b/packages/contrail/src/core/router/profiles.ts @@ -185,7 +185,9 @@ async function fetchMissingProfiles( const events = fetched.filter((event) => event !== null); if (events.length === 0) return {}; - const { accepted } = await ingestRecords(db, events, config); + const { accepted } = await ingestRecords(db, events, config, { + phase: "historical", + }); const result: Record = {}; for (const event of accepted) { if (event.operation === "delete") continue; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index c472054..bb5860e 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -249,6 +249,28 @@ export interface OrderedSourceConfig { epoch: string; } +/** How an accepted mutation entered the logical projection. */ +export type ProjectionPhase = "historical" | "live"; + +export type ChangeConsumerInitialMode = "current" | "future" | "history"; + +/** Static, secret-free definition for one durable change-log consumer. */ +export interface ChangeConsumerConfig { + /** Exact configured collection NSIDs. Short aliases are deliberately rejected. */ + collections: string[]; + /** Projection phases to observe. Defaults to both historical and live. */ + phases?: ProjectionPhase[]; + /** How the consumer establishes its first durable position. */ + initial: ChangeConsumerInitialMode; + /** Whether deployment generation activation may require this consumer. */ + requiredForActivation?: boolean; +} + +export interface ChangeLogConfig { + /** Stable consumer IDs mapped to their static delivery policy. */ + consumers: Record; +} + export type AtprotoServiceAuthMethod = "getFeed" | "notifyOfUpdate"; export interface AtprotoServiceAuthConfig { @@ -284,6 +306,9 @@ export interface ContrailConfig { * cursor is persisted atomically with projected mutations and may be exposed * to clients as a cache invalidation coordinate. */ orderedSource?: OrderedSourceConfig; + /** Optional transactional projection change log. Runtime handlers and + * destination credentials are bound separately and never belong here. */ + changes?: ChangeLogConfig; feeds?: Record; logger?: Logger; /** Expose the notifyOfUpdate HTTP endpoint. Off by default. @@ -655,6 +680,58 @@ function validateShortName(short: string): void { } } +const CHANGE_CONSUMER_ID = /^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/; +const MAX_CHANGE_CONSUMERS = 32; +const MAX_CHANGE_CONSUMER_COLLECTIONS = 64; +const MAX_CHANGE_COVERAGE_PAIRS = 256; +const MAX_CHANGE_DEFINITIONS_BYTES = 64 * 1_024; + +/** Whether this configuration requires the optional transactional change log. */ +export function changesEnabled(config: ContrailConfig): boolean { + return Object.keys(config.changes?.consumers ?? {}).length > 0; +} + +/** Canonical phases for a consumer definition. */ +export function changeConsumerPhases( + consumer: ChangeConsumerConfig, +): ProjectionPhase[] { + return consumer.phases ?? ["historical", "live"]; +} + +/** Canonical collection/phase pairs whose changes must be retained. */ +export function changeLogCoverage( + config: ContrailConfig, +): Array<{ collection: string; phase: ProjectionPhase }> { + const pairs = new Map(); + for (const consumer of Object.values(config.changes?.consumers ?? {})) { + for (const collection of consumer.collections) { + for (const phase of changeConsumerPhases(consumer)) { + pairs.set(`${collection}\0${phase}`, { collection, phase }); + } + } + } + return [...pairs.values()].sort( + (left, right) => + left.collection.localeCompare(right.collection) || + left.phase.localeCompare(right.phase), + ); +} + +/** Stable secret-free representation used by schema/config compatibility checks. */ +export function canonicalChangeDefinitions(config: ContrailConfig): string { + return JSON.stringify( + Object.entries(config.changes?.consumers ?? {}) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([id, consumer]) => ({ + id, + collections: [...consumer.collections].sort(), + phases: [...changeConsumerPhases(consumer)].sort(), + initial: consumer.initial, + requiredForActivation: consumer.requiredForActivation === true, + })), + ); +} + export function validateConfig(config: ContrailConfig): void { const shortNames = new Set(); for (const [short, colConfig] of Object.entries(config.collections)) { @@ -715,6 +792,64 @@ export function validateConfig(config: ContrailConfig): void { } } + const consumers = Object.entries(config.changes?.consumers ?? {}); + if (consumers.length > MAX_CHANGE_CONSUMERS) { + throw new Error(`changes supports at most ${MAX_CHANGE_CONSUMERS} consumers`); + } + const configuredNsids = new Set(getCollectionNsids(config)); + for (const [id, consumer] of consumers) { + if (!CHANGE_CONSUMER_ID.test(id)) { + throw new Error( + `Invalid change consumer ID "${id}"; use 1-64 letters, digits, underscores, or hyphens`, + ); + } + if ( + !Array.isArray(consumer.collections) || + consumer.collections.length === 0 || + consumer.collections.length > MAX_CHANGE_CONSUMER_COLLECTIONS || + new Set(consumer.collections).size !== consumer.collections.length + ) { + throw new Error( + `Change consumer "${id}" requires 1-${MAX_CHANGE_CONSUMER_COLLECTIONS} unique collection NSIDs`, + ); + } + for (const collection of consumer.collections) { + if (!configuredNsids.has(collection)) { + throw new Error( + `Change consumer "${id}" references unconfigured collection NSID "${collection}"`, + ); + } + } + const phases = changeConsumerPhases(consumer); + if ( + phases.length === 0 || + phases.length > 2 || + new Set(phases).size !== phases.length || + phases.some((phase) => phase !== "historical" && phase !== "live") + ) { + throw new Error( + `Change consumer "${id}" requires unique historical/live phases`, + ); + } + if (!(["current", "future", "history"] as string[]).includes(consumer.initial)) { + throw new Error(`Change consumer "${id}" has an invalid initial mode`); + } + } + const coveragePairs = changeLogCoverage(config).length; + if (coveragePairs > MAX_CHANGE_COVERAGE_PAIRS) { + throw new Error( + `changes requires ${coveragePairs} collection/phase coverage pairs; maximum is ${MAX_CHANGE_COVERAGE_PAIRS}`, + ); + } + const definitionBytes = new TextEncoder().encode( + canonicalChangeDefinitions(config), + ).byteLength; + if (definitionBytes > MAX_CHANGE_DEFINITIONS_BYTES) { + throw new Error( + `changes definitions contain ${definitionBytes} encoded bytes; maximum is ${MAX_CHANGE_DEFINITIONS_BYTES}`, + ); + } + } // Helpers diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index bd254c2..0e68f6c 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -29,6 +29,12 @@ export * from "./core/persistent"; export * from "./core/backfill"; export * from "./core/status"; export * from "./core/diagnostics"; +export { + getChangeLogState, + MAX_CHANGE_BATCH_BYTES, + MAX_CHANGE_BATCH_CHANGES, +} from "./core/change-log"; +export type { ChangeLogState, RecordChange } from "./core/change-log"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts new file mode 100644 index 0000000..ead13a1 --- /dev/null +++ b/packages/contrail/tests/change-log.test.ts @@ -0,0 +1,493 @@ +import { describe, expect, it } from "vitest"; +import { + Contrail, + createIngestEvent, + getChangeLogState, + ingestRecords, + initSchema, + queryRecords, + resolveConfig, + saveCursorStatement, + type ContrailConfig, + type Database, + type Statement, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const EVENT = "com.example.event"; +const NOTE = "com.example.note"; +const URI = `at://did:plc:alice/${EVENT}/one`; +const logger = { log() {}, warn() {}, error() {} }; + +function config(options: { + changes?: ContrailConfig["changes"]; + includeNote?: boolean; +} = {}) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { + event: { collection: EVENT }, + ...(options.includeNote ? { note: { collection: NOTE } } : {}), + }, + changes: options.changes, + }); +} + +function loggedConfig() { + return config({ + changes: { + consumers: { + search: { + collections: [EVENT], + initial: "current", + requiredForActivation: true, + }, + webhooks: { + collections: [EVENT], + phases: ["live"], + initial: "future", + }, + }, + }, + }); +} + +function mutation(options: { + sourceTime: number; + cid?: string | null; + value?: Record; + operation?: "create" | "update" | "delete"; + revision?: string | null; +}) { + const operation = options.operation ?? "update"; + return createIngestEvent({ + uri: URI, + did: "did:plc:alice", + collection: EVENT, + rkey: "one", + operation, + cid: operation === "delete" ? null : (options.cid ?? `cid-${options.sourceTime}`), + value: + operation === "delete" + ? undefined + : (options.value ?? { name: `event-${options.sourceTime}` }), + timeUs: options.sourceTime, + indexedAt: options.sourceTime + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: options.sourceTime, + revision: options.revision ?? String(options.sourceTime), + cursor: String(options.sourceTime), + }, + }); +} + +async function batches(db: Database) { + return ( + await db + .prepare( + `SELECT generation_id, position, projection_transaction_id, source_id, + source_epoch, source_cursor, phase, changes_json, change_count + FROM change_batches ORDER BY position`, + ) + .all() + ).results; +} + +describe("transactional projection change log", () => { + it("validates bounded static consumer definitions", () => { + const base = { + namespace: "com.example", + profiles: [] as string[], + collections: { event: { collection: EVENT } }, + }; + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + "bad id": { collections: [EVENT], initial: "future" }, + }, + }, + }), + ).toThrow("Invalid change consumer ID"); + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + search: { collections: ["event"], initial: "future" }, + }, + }, + }), + ).toThrow("unconfigured collection NSID"); + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + search: { + collections: [EVENT], + phases: ["live", "live"], + initial: "future", + }, + }, + }, + }), + ).toThrow("unique historical/live phases"); + }); + + it("has no change-log schema or writes when disabled", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config(); + await initSchema(db, resolved); + + const tables = await db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'change_%' ORDER BY name", + ) + .all<{ name: string }>(); + expect(tables.results).toEqual([]); + expect( + await db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '_contrail_projection_state'", + ) + .first(), + ).not.toBeNull(); + + await ingestRecords(db, [mutation({ sourceTime: 1 })], resolved); + expect(await getChangeLogState(db)).toBeNull(); + }); + + it("initializes a fresh generation, registrations, and coverage ledger", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + + const state = await getChangeLogState(db); + expect(state).toMatchObject({ head: "0", retainedFloor: "0" }); + expect(state?.generation).toMatch(/^[0-9a-f-]{36}$/); + + const consumers = await db + .prepare( + `SELECT consumer_id, configured_collections_json, + configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + acknowledged_position, bootstrap_anchor_position + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + expect(consumers.results).toEqual([ + { + consumer_id: "search", + configured_collections_json: JSON.stringify([EVENT]), + configured_phases_json: JSON.stringify(["historical", "live"]), + initial_mode: "current", + required_for_activation: 1, + bootstrap_state: "pending", + acknowledged_position: 0, + bootstrap_anchor_position: 0, + }, + { + consumer_id: "webhooks", + configured_collections_json: JSON.stringify([EVENT]), + configured_phases_json: JSON.stringify(["live"]), + initial_mode: "future", + required_for_activation: 0, + bootstrap_state: "ready", + acknowledged_position: 0, + bootstrap_anchor_position: null, + }, + ]); + + const coverage = await db + .prepare( + `SELECT collection, phase, from_position, through_position + FROM change_log_coverage ORDER BY collection, phase`, + ) + .all(); + expect(coverage.results).toEqual([ + { + collection: EVENT, + phase: "historical", + from_position: 0, + through_position: null, + }, + { + collection: EVENT, + phase: "live", + from_position: 0, + through_position: null, + }, + ]); + + // Initialization is idempotent and retains one random database generation. + await initSchema(db, resolved); + expect((await getChangeLogState(db))?.generation).toBe(state?.generation); + }); + + it("appends only committed logical current-state changes", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + + const original = mutation({ + sourceTime: 100, + cid: "cid-one", + value: { name: "one" }, + }); + await ingestRecords(db, [original], resolved, { phase: "historical" }); + expect((await getChangeLogState(db))?.head).toBe("1"); + + // Exact replay and a newer source observation of the same immutable state + // may update source metadata but do not wake current-state consumers. + await ingestRecords(db, [original], resolved, { phase: "historical" }); + await ingestRecords( + db, + [ + mutation({ + sourceTime: 110, + cid: "cid-one", + value: { name: "one" }, + }), + ], + resolved, + { phase: "live" }, + ); + expect((await getChangeLogState(db))?.head).toBe("1"); + + await ingestRecords( + db, + [ + mutation({ + sourceTime: 200, + cid: "cid-two", + value: { name: "two" }, + }), + ], + resolved, + { phase: "live" }, + ); + await ingestRecords( + db, + [mutation({ sourceTime: 300, operation: "delete" })], + resolved, + { phase: "live" }, + ); + await ingestRecords( + db, + [mutation({ sourceTime: 400, operation: "delete" })], + resolved, + { phase: "live" }, + ); + + const rows = await batches(db); + expect(rows).toHaveLength(3); + expect(rows.map((row) => [row.position, row.phase, row.change_count])).toEqual([ + [1, "historical", 1], + [2, "live", 1], + [3, "live", 1], + ]); + const changes = rows.map((row) => JSON.parse(row.changes_json)[0]); + expect(changes[0]).toMatchObject({ + kind: "record", + operation: "put", + uri: URI, + cid: "cid-one", + version: { sourceId: "source", sourceTimeUs: 100 }, + }); + expect(changes[0]).not.toHaveProperty("id"); + expect(changes[0]).not.toHaveProperty("record"); + expect(changes[2]).toMatchObject({ + operation: "delete", + uri: URI, + cid: null, + }); + expect((await getChangeLogState(db))?.head).toBe("3"); + }); + + it("reduces multiple mutations for one URI to the final state", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + + await ingestRecords( + db, + [ + mutation({ sourceTime: 1, cid: "cid-one" }), + mutation({ sourceTime: 2, cid: "cid-two" }), + mutation({ sourceTime: 3, cid: "cid-three" }), + ], + resolved, + ); + + const rows = await batches(db); + expect(rows).toHaveLength(1); + expect(rows[0].change_count).toBe(1); + expect(JSON.parse(rows[0].changes_json)[0].cid).toBe("cid-three"); + }); + + it("does not log a phase outside every consumer's coverage", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + changes: { + consumers: { + webhook: { + collections: [EVENT], + phases: ["live"], + initial: "future", + }, + }, + }, + }); + await initSchema(db, resolved); + + await ingestRecords(db, [mutation({ sourceTime: 1 })], resolved, { + phase: "historical", + }); + expect((await getChangeLogState(db))?.head).toBe("0"); + expect(await batches(db)).toEqual([]); + }); + + it("rolls projection, log head, batch, and checkpoint back together", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + await db.prepare("CREATE TABLE change_failure (value TEXT UNIQUE)").run(); + await db.prepare("INSERT INTO change_failure VALUES ('duplicate')").run(); + + await expect( + ingestRecords(db, [mutation({ sourceTime: 1 })], resolved, { + trailingStatements: [ + saveCursorStatement(db, 1), + db.prepare("INSERT INTO change_failure VALUES ('duplicate')"), + ], + }), + ).rejects.toThrow(); + + expect((await getChangeLogState(db))?.head).toBe("0"); + expect(await batches(db)).toEqual([]); + expect( + await db.prepare("SELECT uri FROM records_event").first(), + ).toBeNull(); + expect(await db.prepare("SELECT time_us FROM cursor").first()).toBeNull(); + }); + + it("gives each fresh database a distinct generation", async () => { + const resolved = loggedConfig(); + const first = createSqliteDatabase(":memory:"); + const second = createSqliteDatabase(":memory:"); + await initSchema(first, resolved); + await initSchema(second, resolved); + expect((await getChangeLogState(first))?.generation).not.toBe( + (await getChangeLogState(second))?.generation, + ); + }); + + it("fails closed for unsafe enable, disable, and definition changes", async () => { + const populated = createSqliteDatabase(":memory:"); + const disabled = config(); + await initSchema(populated, disabled); + await ingestRecords(populated, [mutation({ sourceTime: 1 })], disabled); + await expect(initSchema(populated, loggedConfig())).rejects.toThrow( + "fresh empty generation", + ); + expect( + await populated + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='change_log_state'", + ) + .first(), + ).toBeNull(); + + const initialized = createSqliteDatabase(":memory:"); + await initSchema(initialized, loggedConfig()); + await expect(initSchema(initialized, config())).rejects.toThrow( + "cannot be disabled", + ); + const changed = config({ + changes: { + consumers: { + replacement: { collections: [EVENT], initial: "future" }, + }, + }, + }); + await expect(initSchema(initialized, changed)).rejects.toThrow( + "definitions differ", + ); + }); + + it("retries a losing overlapping writer from fresh durable state", async () => { + const real = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(real, resolved); + + let arrivals = 0; + let releaseBoth!: () => void; + const both = new Promise((resolve) => { + releaseBoth = resolve; + }); + let releaseNewer!: () => void; + const newerDone = new Promise((resolve) => { + releaseNewer = resolve; + }); + + function overlapping(role: "older" | "newer"): Database { + let projectionBatches = 0; + return { + prepare(sql: string): Statement { + return real.prepare(sql); + }, + async batch(statements: Statement[]) { + projectionBatches++; + if (projectionBatches > 1) return real.batch(statements); + arrivals++; + if (arrivals === 2) releaseBoth(); + await both; + if (role === "older") await newerDone; + try { + return await real.batch(statements); + } finally { + if (role === "newer") releaseNewer(); + } + }, + dialect: real.dialect, + }; + } + + const olderDb = overlapping("older"); + const newerDb = overlapping("newer"); + const [older] = await Promise.all([ + ingestRecords( + olderDb, + [mutation({ sourceTime: 100, cid: "cid-old", value: { name: "old" } })], + resolved, + { trailingStatements: [saveCursorStatement(real, 100)] }, + ), + ingestRecords( + newerDb, + [mutation({ sourceTime: 200, cid: "cid-new", value: { name: "new" } })], + resolved, + { trailingStatements: [saveCursorStatement(real, 200)] }, + ), + ]); + + expect(older.dropped.superseded).toBe(1); + const visible = await queryRecords(real, resolved, { collection: "event" }); + expect(JSON.parse(visible.records[0]!.record!).name).toBe("new"); + expect((await getChangeLogState(real))?.head).toBe("1"); + expect(JSON.parse((await batches(real))[0].changes_json)[0].cid).toBe( + "cid-new", + ); + expect( + await real.prepare("SELECT time_us FROM cursor WHERE id = 1").first(), + ).toEqual({ time_us: 200 }); + }); +}); diff --git a/packages/contrail/tests/ingest.test.ts b/packages/contrail/tests/ingest.test.ts index eefa59b..c70cd74 100644 --- a/packages/contrail/tests/ingest.test.ts +++ b/packages/contrail/tests/ingest.test.ts @@ -106,7 +106,9 @@ describe("ingestRecords", () => { const versionInserts = sql.filter((statement) => statement.startsWith("INSERT INTO record_versions"), ); - expect(versionInserts).toHaveLength(4); + // Thirteen bindings include the optimistic projection token, so seven + // versions fit under D1's 100-binding statement ceiling. + expect(versionInserts).toHaveLength(5); expect( versionInserts.every( (statement) => (statement.match(/\?/g) ?? []).length <= 100, diff --git a/packages/contrail/tests/postgres-concurrent-init.test.ts b/packages/contrail/tests/postgres-concurrent-init.test.ts index 954bbb3..dd6fdae 100644 --- a/packages/contrail/tests/postgres-concurrent-init.test.ts +++ b/packages/contrail/tests/postgres-concurrent-init.test.ts @@ -71,7 +71,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', 'backfills', 'backfill_state', 'discovery', 'cursor', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); diff --git a/packages/contrail/tests/postgres-e2e.test.ts b/packages/contrail/tests/postgres-e2e.test.ts index e5032ea..0e5d499 100644 --- a/packages/contrail/tests/postgres-e2e.test.ts +++ b/packages/contrail/tests/postgres-e2e.test.ts @@ -18,7 +18,7 @@ import { saveCursor, } from "../src/index"; import { resolveConfig } from "../src/index"; -import type { Database } from "../src/index"; +import type { Database, Statement } from "../src/index"; import { resolveHydrates, resolveReferences } from "../src/index"; import { makeEvent } from "./helpers"; @@ -78,7 +78,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', 'backfills', 'backfill_state', 'discovery', 'cursor', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); @@ -171,6 +171,81 @@ if (!PG_URL) { expect(result.records).toHaveLength(0); }); + it("retries an overlapping stale projector after the transaction lock", async () => { + const uri = "at://did:plc:test/community.lexicon.calendar.event/concurrent"; + let arrivals = 0; + let releaseBoth!: () => void; + const both = new Promise((resolve) => { + releaseBoth = resolve; + }); + let releaseNewer!: () => void; + const newerDone = new Promise((resolve) => { + releaseNewer = resolve; + }); + + const overlap = (role: "older" | "newer"): Database => { + let writes = 0; + return { + prepare(sql: string): Statement { + return db.prepare(sql); + }, + async batch(statements: Statement[]) { + writes++; + if (writes > 1) return db.batch(statements); + arrivals++; + if (arrivals === 2) releaseBoth(); + await both; + if (role === "older") await newerDone; + try { + return await db.batch(statements); + } finally { + if (role === "newer") releaseNewer(); + } + }, + dialect: db.dialect, + }; + }; + + const [older] = await Promise.all([ + ingestRecords( + overlap("older"), + [ + makeEvent({ + uri, + rkey: "concurrent", + cid: "cid-old", + record: { name: "old", mode: "online" }, + time_us: 100, + indexed_at: 100, + }), + ], + TEST_CONFIG, + ), + ingestRecords( + overlap("newer"), + [ + makeEvent({ + uri, + rkey: "concurrent", + cid: "cid-new", + record: { name: "new", mode: "online" }, + time_us: 200, + indexed_at: 200, + }), + ], + TEST_CONFIG, + ), + ]); + + expect(older.dropped.superseded).toBe(1); + const row = await pool.query( + "SELECT record, cid FROM records_community_lexicon_calendar_event WHERE uri = $1", + [uri], + ); + expect(row.rows[0]).toMatchObject({ cid: "cid-new" }); + expect(row.rows[0].record.name).toBe("new"); + }); + it("does nothing for empty events", async () => { await ingestRecords(db, []); const cursor = await getLastCursor(db); diff --git a/packages/contrail/tests/postgres.test.ts b/packages/contrail/tests/postgres.test.ts index 2738dc6..0b7b530 100644 --- a/packages/contrail/tests/postgres.test.ts +++ b/packages/contrail/tests/postgres.test.ts @@ -57,7 +57,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', 'backfills', 'backfill_state', 'discovery', 'cursor', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); diff --git a/packages/contrail/tests/schema.test.ts b/packages/contrail/tests/schema.test.ts index 15643dc..3a95149 100644 --- a/packages/contrail/tests/schema.test.ts +++ b/packages/contrail/tests/schema.test.ts @@ -21,6 +21,8 @@ describe("initSchema", () => { expect(names).toContain("identities"); expect(names).toContain("record_versions"); expect(names).toContain("ingest_diagnostics"); + expect(names).toContain("_contrail_projection_state"); + expect(names).not.toContain("change_log_state"); const backfillColumns = await db .prepare("PRAGMA table_info(backfills)") @@ -115,6 +117,52 @@ describe("initSchema", () => { ); }); + it("migrates optimistic tokens onto existing version rows", async () => { + const db = createTestDb(); + await db + .prepare( + `CREATE TABLE record_versions ( + uri TEXT PRIMARY KEY, + did TEXT NOT NULL, + collection TEXT NOT NULL, + rkey TEXT NOT NULL, + operation TEXT NOT NULL, + cid TEXT, + source_id TEXT NOT NULL, + source_revision TEXT, + source_time_us BIGINT NOT NULL, + source_cursor TEXT, + indexed_at BIGINT NOT NULL + )`, + ) + .run(); + await db + .prepare( + `INSERT INTO record_versions + (uri, did, collection, rkey, operation, cid, source_id, + source_revision, source_time_us, source_cursor, indexed_at) + VALUES (?, ?, ?, ?, 'update', ?, 'legacy', NULL, 1, NULL, 1)`, + ) + .bind( + "at://did:plc:legacy/community.lexicon.calendar.event/one", + "did:plc:legacy", + "community.lexicon.calendar.event", + "one", + "cid-legacy", + ) + .run(); + + await initSchema(db, TEST_CONFIG); + expect( + await db + .prepare("SELECT projection_token FROM record_versions") + .first(), + ).toEqual({ + projection_token: + "at://did:plc:legacy/community.lexicon.calendar.event/one", + }); + }); + it("seeds ordering metadata for visible rows from older schemas", async () => { const db = createTestDb(); await db @@ -149,7 +197,7 @@ describe("initSchema", () => { const version = await db .prepare( - "SELECT operation, source_id, source_time_us, cid FROM record_versions WHERE uri = ?", + "SELECT operation, source_id, source_time_us, cid, projection_token FROM record_versions WHERE uri = ?", ) .bind("at://did:plc:legacy/community.lexicon.calendar.event/one") .first<{ @@ -157,12 +205,15 @@ describe("initSchema", () => { source_id: string; source_time_us: number; cid: string; + projection_token: string; }>(); expect(version).toEqual({ operation: "update", source_id: "legacy", source_time_us: 200, cid: "cid-legacy", + projection_token: + "at://did:plc:legacy/community.lexicon.calendar.event/one", }); }); diff --git a/turbo.json b/turbo.json index 94b9292..2279a64 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,7 @@ "dependsOn": ["^build"] }, "test": { - "dependsOn": ["^build"], + "dependsOn": ["^build", "build"], "outputs": [] }, "dev": { -- 2.51.2 From cd0771e3f537e23386aac390cdd0bb7e0819cc2e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:04:25 +0200 Subject: [PATCH 02/10] Add durable change consumer protocol --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 27 +- packages/contrail/package.json | 2 +- packages/contrail/src/cli.ts | 5 +- packages/contrail/src/cli/commands/changes.ts | 127 +++ packages/contrail/src/contrail.ts | 7 + packages/contrail/src/core/change-log.ts | 20 +- packages/contrail/src/core/changes.ts | 981 ++++++++++++++++++ packages/contrail/src/core/db/schema.ts | 13 +- packages/contrail/src/index.ts | 1 + packages/contrail/tests/built-changes.mjs | 42 + packages/contrail/tests/built-sqlite.mjs | 2 + .../contrail/tests/change-consumers.test.ts | 351 +++++++ packages/contrail/tests/postgres-e2e.test.ts | 62 ++ 14 files changed, 1625 insertions(+), 17 deletions(-) create mode 100644 packages/contrail/src/cli/commands/changes.ts create mode 100644 packages/contrail/src/core/changes.ts create mode 100644 packages/contrail/tests/built-changes.mjs create mode 100644 packages/contrail/tests/change-consumers.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 6a3e1e5..b76f0f1 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs plus `contrail changes status/retry` commands. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 99614f1..33d0b54 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -96,7 +96,32 @@ const config = { Static definitions contain no handlers, URLs, clients, credentials, or secrets. Contrail registers them with a random database-generation ID and collection/phase coverage ledger. A winning logical put/delete appends one compact URI/version reference in the same transaction as canonical and derived state plus the source checkpoint. Duplicate, stale, same-CID, absent-delete, rejected, and rolled-back mutations append nothing. Record bodies are hydrated from current state by the later delivery layer rather than copied into the log. -This milestone intentionally exposes only the atomic log foundation; leased claim/hydrate/ack delivery and current-state bootstrap APIs follow separately. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. +Low-level delivery uses bounded leases and compare-and-swap acknowledgement: + +```ts +const claim = await contrail.changes.claim("webhooks", { + maxBatches: 20, + maxChanges: 500, + maxBytes: 512_000, + leaseMs: 30_000, +}); +if (claim) { + try { + const batch = await contrail.changes.hydrate(claim); + await deliverIdempotently(batch); + await contrail.changes.ack(claim); + } catch { + await contrail.changes.fail(claim, { + code: "destination_unavailable", + nextAttemptAt: Date.now() + 30_000, + }); + } +} +``` + +Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and resolve delete/recreate races from newest canonical state. Consumers lease and progress independently; irrelevant position ranges advance without invoking a handler. Delivery is intentionally at least once—a destination success followed by an acknowledgement crash causes duplicate delivery. Handlers must be idempotent by stable record/document key. + +`initial: "future"` and `initial: "history"` consumers can use this API now. `initial: "current"` remains pending until the snapshot-plus-tail bootstrap coordinator lands in the next milestone. `contrail changes status` and `contrail changes retry ` expose private status and manual retry for SQLite or Wrangler D1 deployments. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. ## Local development diff --git a/packages/contrail/package.json b/packages/contrail/package.json index d695f92..86378ce 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,7 +73,7 @@ "clean": "rm -rf dist", "typecheck": "tsc --noEmit", "test": "vitest run", - "test:built": "node tests/built-sqlite.mjs && node tests/built-alluvium.mjs && node tests/built-lexicons.mjs && node tests/built-client.mjs", + "test:built": "node tests/built-sqlite.mjs && node tests/built-alluvium.mjs && node tests/built-lexicons.mjs && node tests/built-client.mjs && node tests/built-changes.mjs", "test:watch": "vitest" }, "dependencies": { diff --git a/packages/contrail/src/cli.ts b/packages/contrail/src/cli.ts index 7299c6b..2aa6d9a 100644 --- a/packages/contrail/src/cli.ts +++ b/packages/contrail/src/cli.ts @@ -11,6 +11,7 @@ import { registerDev } from "./cli/commands/dev.js"; import { registerAppendScheduled } from "./cli/commands/append-scheduled.js"; import { registerConnect } from "./cli/commands/connect.js"; import { registerLexicons } from "./cli/commands/lexicons.js"; +import { registerChanges } from "./cli/commands/changes.js"; const cli = cac("contrail"); @@ -19,11 +20,13 @@ registerDev(cli); registerAppendScheduled(cli); registerConnect(cli); registerLexicons(cli); +registerChanges(cli); cli.help(); try { - cli.parse(); + cli.parse(process.argv, { run: false }); + await cli.runMatchedCommand(); } catch (err) { console.error(err); process.exit(1); diff --git a/packages/contrail/src/cli/commands/changes.ts b/packages/contrail/src/cli/commands/changes.ts new file mode 100644 index 0000000..d035b55 --- /dev/null +++ b/packages/contrail/src/cli/commands/changes.ts @@ -0,0 +1,127 @@ +import type { CAC } from "cac"; +import { Contrail } from "../../contrail.js"; +import type { Database } from "../../core/types.js"; +import { + resolveAndLoadConfig, + resolveValidationLexicons, +} from "../shared.js"; + +interface ChangeCommandOptions { + config?: string; + root?: string; + remote?: boolean; + binding: string; + sqlite?: string; + json?: boolean; +} + +async function withChangesDatabase( + options: ChangeCommandOptions, + callback: (contrail: Contrail, db: Database) => Promise, +): Promise { + if (options.sqlite && options.remote) { + throw new Error("--sqlite cannot be combined with --remote"); + } + const config = await resolveAndLoadConfig(options); + const lexicons = await resolveValidationLexicons(options, config); + const contrail = new Contrail({ ...config, lexicons }); + if (options.sqlite) { + const { createSqliteDatabase } = await import("../../adapters/sqlite.js"); + const db = createSqliteDatabase(options.sqlite); + await contrail.init(db); + return callback(contrail, db); + } + + const { getPlatformProxy } = await import("wrangler"); + const { env, dispose } = await getPlatformProxy({ + environment: options.remote ? "production" : undefined, + }); + const binding = options.binding ?? "DB"; + const db = (env as Record)[binding] as Database | undefined; + if (!db) { + await dispose(); + throw new Error(`No binding named "${binding}" in wrangler env`); + } + try { + await contrail.init(db); + return await callback(contrail, db); + } finally { + await dispose(); + } +} + +function options(command: ReturnType) { + return command + .option("--config ", "Path to Contrail config file (TS or JS)") + .option("--root ", "Project root for auto-detection (default: CWD)") + .option("--remote", "Use production D1 bindings") + .option("--binding ", "D1 binding name in wrangler.jsonc", { + default: "DB", + }) + .option( + "--sqlite ", + "Use a local SQLite database instead of a Wrangler D1 binding", + ); +} + +export function registerChanges(cli: CAC): void { + options( + cli.command( + "changes [consumer]", + "Private change-log operations: status, retry ", + ), + ) + .option("--json", "Print machine-readable status JSON") + .action( + async ( + action: string, + consumer: string | undefined, + commandOptions: ChangeCommandOptions, + ) => { + if (action !== "status" && action !== "retry") { + throw new Error("changes action must be 'status' or 'retry'"); + } + if (action === "retry" && !consumer) { + throw new Error("changes retry requires a consumer ID"); + } + if (action === "status" && consumer) { + throw new Error("changes status does not accept a consumer ID"); + } + + await withChangesDatabase(commandOptions, async (contrail, db) => { + if (action === "retry") { + await contrail.changes.retry(consumer!, undefined, db); + console.log(`change consumer ${consumer}: retry is now due`); + return; + } + + const status = await contrail.changes.status(db); + if (commandOptions.json) { + console.log(JSON.stringify(status, null, 2)); + return; + } + if (!status.enabled || !status.state) { + console.log("change log: disabled"); + return; + } + console.log( + `change log: generation=${status.state.generation} head=${status.state.head} ` + + `floor=${status.state.retainedFloor} rows=${status.rows} ` + + `changes=${status.changes} bytes=${status.bytes}`, + ); + for (const item of status.consumers) { + const retry = + item.nextAttemptAt === null + ? "due" + : new Date(item.nextAttemptAt).toISOString(); + console.log( + ` ${item.id}: state=${item.bootstrapState} ` + + `position=${item.position} backlog=${item.backlogBatches}/` + + `${item.backlogChanges} attempts=${item.attempts} ` + + `retry=${retry} leased=${item.leased ? "yes" : "no"}`, + ); + } + }); + }, + ); +} diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index bba7572..859d054 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -14,6 +14,7 @@ import { prepareRecordValidation, } from "./core/validation"; import { getIngestDiagnostics } from "./core/diagnostics"; +import { ChangeConsumers } from "./core/changes"; import { optimizeDatabase } from "./core/db/optimize"; import { assertServingSourceCompatibility, @@ -60,6 +61,8 @@ export interface ContrailOptions extends ContrailConfig { export class Contrail { readonly config: ResolvedContrailConfig; + /** Durable low-level claim/hydrate/ack consumer API. */ + readonly changes: ChangeConsumers; private _db?: Database; private _ingestState: IngestState = createIngestState(); @@ -72,6 +75,10 @@ export class Contrail { // Otherwise init/app binds the runtime's generated bundle first. if (lexicons) prepareRecordValidation(this.config); this._db = db; + this.changes = new ChangeConsumers( + this.config, + (database) => this.getDb(database), + ); } private getDb(db?: Database): Database { diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 6505ce9..5973925 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -102,6 +102,7 @@ export function buildChangeLogSchema( phase TEXT NOT NULL CHECK (phase IN ('historical', 'live')), changes_json TEXT NOT NULL, change_count INTEGER NOT NULL, + encoded_bytes INTEGER NOT NULL, created_at ${bigint} NOT NULL, PRIMARY KEY (generation_id, position) )`, @@ -235,8 +236,8 @@ export async function initializeChangeLog( configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, bootstrap_anchor_position, attempts, updated_at) - SELECT ?, generation_id, head_position, ?, ?, ?, ?, ?, - CASE WHEN ? = 'current' THEN head_position ELSE NULL END, + SELECT ?, generation_id, 0, ?, ?, ?, ?, ?, + CASE WHEN ? = 'current' THEN 0 ELSE NULL END, 0, ? FROM change_log_state WHERE id = 1 AND definitions_json = ? @@ -262,7 +263,7 @@ export async function initializeChangeLog( .prepare( `INSERT INTO change_log_coverage (generation_id, collection, phase, from_position, through_position) - SELECT generation_id, ?, ?, head_position, NULL + SELECT generation_id, ?, ?, 0, NULL FROM change_log_state WHERE id = 1 AND definitions_json = ? ON CONFLICT(generation_id, collection, phase, from_position) @@ -312,7 +313,6 @@ async function assertChangeLogDefinition( for (let index = 0; index < expectedConsumers.length; index++) { const [id, expected] = expectedConsumers[index]!; const actual = consumers.results[index]!; - const bootstrapState = expected.initial === "current" ? "pending" : "ready"; if ( actual.consumer_id !== id || actual.generation_id !== state.generation_id || @@ -320,12 +320,7 @@ async function assertChangeLogDefinition( actual.configured_phases_json !== canonicalPhases(changeConsumerPhases(expected)) || actual.initial_mode !== expected.initial || Number(actual.required_for_activation) !== - (expected.requiredForActivation === true ? 1 : 0) || - actual.bootstrap_state !== bootstrapState || - Number(actual.acknowledged_position) !== 0 || - (expected.initial === "current" - ? Number(actual.bootstrap_anchor_position) !== 0 - : actual.bootstrap_anchor_position !== null) + (expected.requiredForActivation === true ? 1 : 0) ) { throw new Error(`Durable change consumer ${id} is incompatible`); } @@ -499,11 +494,11 @@ export function appendChangeLogStatements( `INSERT INTO change_batches (generation_id, position, projection_transaction_id, source_id, source_epoch, source_cursor, phase, changes_json, change_count, - created_at) + encoded_bytes, created_at) VALUES ( (SELECT generation_id FROM change_log_state WHERE id = 1), (SELECT head_position FROM change_log_state WHERE id = 1), - ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ? )`, ) .bind( @@ -514,6 +509,7 @@ export function appendChangeLogStatements( phase, serialized, changes.length, + bytes, now, ), ]; diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts new file mode 100644 index 0000000..8482453 --- /dev/null +++ b/packages/contrail/src/core/changes.ts @@ -0,0 +1,981 @@ +import type { + ContrailConfig, + Database, + ProjectionPhase, +} from "./types"; +import { + changeConsumerPhases, + recordsTableName, + resolveCollectionKey, +} from "./types"; +import { + getChangeLogState, + type ChangeLogState, + type RecordChange, +} from "./change-log"; + +const DEFAULT_MAX_BATCHES = 20; +const DEFAULT_MAX_CHANGES = 500; +const DEFAULT_MAX_BYTES = 512_000; +const DEFAULT_LEASE_MS = 30_000; +const DEFAULT_AUTO_ADVANCE_RANGES = 8; +const MAX_CLAIM_BATCHES = 100; +const MAX_CLAIM_CHANGES = 5_000; +const MAX_CLAIM_BYTES = 4 * 1_024 * 1_024; +const MAX_LEASE_MS = 10 * 60_000; +const FAILURE_CODE = /^[a-zA-Z0-9_.:-]{1,64}$/; + +export class ChangeConsumerNotFoundError extends Error {} +export class ChangeClaimTooLargeError extends Error {} +export class ChangeLeaseLostError extends Error {} +export class ChangeHistoryGapError extends Error {} +export class ChangeGenerationMismatchError extends Error {} + +export interface ChangeClaimOptions { + maxBatches?: number; + maxChanges?: number; + maxBytes?: number; + leaseMs?: number; + /** Maximum irrelevant position ranges core may acknowledge without invoking + * a handler in one claim call. */ + maxAutoAdvanceRanges?: number; + /** @internal Deterministic clock seam for conformance tests. */ + now?: number; +} + +export interface ChangeClaim { + consumerId: string; + generation: string; + from: string; + through: string; + changes: RecordChange[]; + attempt: number; + leaseExpiresAt: number; + /** Opaque CAS capability. Do not pass this field to delivery handlers. */ + readonly leaseOwner: string; +} + +export interface CurrentRecord { + uri: string; + did: string; + collection: string; + rkey: string; + cid: string | null; + value: unknown; + timeUs: number; + indexedAt: number; +} + +export interface DeliveryBatch { + consumerId: string; + cursor: { + generation: string; + from: string; + through: string; + }; + changes: RecordChange[]; + currentRecords: CurrentRecord[]; + absentUris: string[]; +} + +export interface ChangeFailure { + /** Stable sanitized category. Raw destination errors are never persisted. */ + code: string; + /** Runtime-computed retry eligibility. Null makes the claim immediately due. */ + nextAttemptAt: number | null; +} + +export interface ChangeConsumerStatus { + id: string; + generation: string; + position: string; + bootstrapState: string; + initialMode: string; + requiredForActivation: boolean; + attempts: number; + nextAttemptAt: number | null; + lastSuccessAt: number | null; + lastErrorCode: string | null; + lastErrorAt: number | null; + leased: boolean; + leaseExpiresAt: number | null; + backlogBatches: number; + backlogChanges: number; + backlogBytes: number; + oldestPendingAt: number | null; + generationMatches: boolean; +} + +export interface ChangeLogStatus { + enabled: boolean; + state: ChangeLogState | null; + rows: number; + changes: number; + bytes: number; + oldestRetainedAt: number | null; + consumers: ChangeConsumerStatus[]; +} + +interface DurableConsumerRow { + consumer_id: string; + generation_id: string; + acknowledged_position: number | string; + configured_collections_json: string; + configured_phases_json: string; + initial_mode: string; + required_for_activation: number | string; + bootstrap_state: string; + lease_owner: string | null; + lease_expires_at: number | string | null; + attempts: number | string; + next_attempt_at: number | string | null; + last_success_at: number | string | null; + last_error_code: string | null; + last_error_at: number | string | null; +} + +interface BatchMetadataRow { + position: number | string; + phase: ProjectionPhase; + change_count: number | string; + encoded_bytes: number | string; +} + +interface StoredBatchRow extends BatchMetadataRow { + changes_json: string; +} + +interface BacklogRow { + backlog_batches: number | string; + backlog_changes: number | string | null; + backlog_bytes: number | string | null; + oldest_pending_at: number | string | null; +} + +function integer( + value: number | undefined, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new TypeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return result; +} + +function timestamp(value: number | undefined = Date.now()): number { + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError("now must be a non-negative safe integer"); + } + return value; +} + +function nullableNumber(value: number | string | null): number | null { + return value === null ? null : Number(value); +} + +function parseStringArray(value: string, label: string): string[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error(`${label} is not valid JSON`); + } + if ( + !Array.isArray(parsed) || + !parsed.every((item) => typeof item === "string") || + new Set(parsed).size !== parsed.length + ) { + throw new Error(`${label} is malformed`); + } + return parsed; +} + +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function decodeChange( + value: unknown, + generation: string, + position: string, + ordinal: number, +): RecordChange { + if ( + !object(value) || + value.kind !== "record" || + (value.operation !== "put" && value.operation !== "delete") || + typeof value.uri !== "string" || + typeof value.did !== "string" || + typeof value.collection !== "string" || + typeof value.rkey !== "string" || + (value.cid !== null && typeof value.cid !== "string") || + !object(value.version) || + typeof value.version.sourceId !== "string" || + (value.version.sourceEpoch !== null && + typeof value.version.sourceEpoch !== "string") || + (value.version.sourceRevision !== null && + typeof value.version.sourceRevision !== "string") || + !Number.isSafeInteger(value.version.sourceTimeUs) || + Number(value.version.sourceTimeUs) < 0 || + (value.version.sourceCursor !== null && + typeof value.version.sourceCursor !== "string") + ) { + throw new Error(`Change batch ${generation}/${position} is malformed`); + } + return { + ...(value as unknown as Omit), + id: `${generation}:${position}:${ordinal}`, + }; +} + +function decodeBatchChanges( + row: StoredBatchRow, + generation: string, +): RecordChange[] { + const position = String(row.position); + const encoded = new TextEncoder().encode(row.changes_json).byteLength; + if (encoded !== Number(row.encoded_bytes)) { + throw new Error(`Change batch ${generation}/${position} byte count is invalid`); + } + let parsed: unknown; + try { + parsed = JSON.parse(row.changes_json); + } catch { + throw new Error(`Change batch ${generation}/${position} is not valid JSON`); + } + if (!Array.isArray(parsed) || parsed.length !== Number(row.change_count)) { + throw new Error(`Change batch ${generation}/${position} count is invalid`); + } + return parsed.map((change, ordinal) => + decodeChange(change, generation, position, ordinal), + ); +} + +async function durableConsumer( + db: Database, + consumerId: string, +): Promise { + return db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, + initial_mode, required_for_activation, bootstrap_state, + lease_owner, lease_expires_at, attempts, next_attempt_at, + last_success_at, last_error_code, last_error_at + FROM change_consumers WHERE consumer_id = ?`, + ) + .bind(consumerId) + .first(); +} + +async function releaseLease( + db: Database, + consumerId: string, + generation: string, + from: string, + owner: string, + now: number, +): Promise { + await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ?`, + ) + .bind(now, consumerId, generation, from, owner) + .run(); +} + +function normalizedClaimOptions(options: ChangeClaimOptions = {}) { + return { + maxBatches: integer( + options.maxBatches, + DEFAULT_MAX_BATCHES, + 1, + MAX_CLAIM_BATCHES, + "maxBatches", + ), + maxChanges: integer( + options.maxChanges, + DEFAULT_MAX_CHANGES, + 1, + MAX_CLAIM_CHANGES, + "maxChanges", + ), + maxBytes: integer( + options.maxBytes, + DEFAULT_MAX_BYTES, + 1, + MAX_CLAIM_BYTES, + "maxBytes", + ), + leaseMs: integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ), + maxAutoAdvanceRanges: integer( + options.maxAutoAdvanceRanges, + DEFAULT_AUTO_ADVANCE_RANGES, + 0, + 32, + "maxAutoAdvanceRanges", + ), + now: timestamp(options.now), + }; +} + +/** Verify that schema initialization registered one configured consumer. */ +export async function registerChangeConsumer( + db: Database, + config: ContrailConfig, + consumerId: string, +): Promise { + const configured = config.changes?.consumers[consumerId]; + if (!configured) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not configured`, + ); + } + const state = await getChangeLogState(db); + if (!state) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const durable = await durableConsumer(db, consumerId); + if (!durable) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const collections = parseStringArray( + durable.configured_collections_json, + `Change consumer ${consumerId} collections`, + ); + const phases = parseStringArray( + durable.configured_phases_json, + `Change consumer ${consumerId} phases`, + ); + if ( + durable.generation_id !== state.generation || + JSON.stringify(collections) !== + JSON.stringify([...configured.collections].sort()) || + JSON.stringify(phases) !== + JSON.stringify([...changeConsumerPhases(configured)].sort()) || + durable.initial_mode !== configured.initial || + Number(durable.required_for_activation) !== + (configured.requiredForActivation === true ? 1 : 0) + ) { + throw new Error(`Change consumer ${consumerId} is incompatible`); + } +} + +/** Claim one bounded contiguous position range. Irrelevant ranges advance by + * CAS without invoking application code. */ +export async function claimChanges( + db: Database, + consumerId: string, + options: ChangeClaimOptions = {}, +): Promise { + const limits = normalizedClaimOptions(options); + if (!(await getChangeLogState(db))) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + + for ( + let autoAdvanced = 0; + autoAdvanced <= limits.maxAutoAdvanceRanges; + autoAdvanced++ + ) { + const owner = crypto.randomUUID(); + const leaseExpiresAt = limits.now + limits.leaseMs; + const leased = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? + AND bootstrap_state = 'ready' + AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) + AND acknowledged_position >= (SELECT retained_floor_position FROM change_log_state WHERE id = 1) + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, + initial_mode, required_for_activation, bootstrap_state, + lease_owner, lease_expires_at, attempts, next_attempt_at, + last_success_at, last_error_code, last_error_at`, + ) + .bind( + owner, + leaseExpiresAt, + limits.now, + consumerId, + limits.now, + limits.now, + ) + .first(); + if (!leased) { + const durable = await durableConsumer(db, consumerId); + if (!durable) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const state = await getChangeLogState(db); + if (state && durable.generation_id !== state.generation) { + throw new ChangeGenerationMismatchError( + `Change consumer ${consumerId} belongs to generation ${durable.generation_id}, not ${state.generation}`, + ); + } + if ( + state && + BigInt(String(durable.acknowledged_position)) < + BigInt(state.retainedFloor) + ) { + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} is behind retained floor ${state.retainedFloor}`, + ); + } + return null; + } + + const generation = leased.generation_id; + const from = String(leased.acknowledged_position); + const metadata = await db + .prepare( + `SELECT position, phase, change_count, encoded_bytes + FROM change_batches + WHERE generation_id = ? AND position > ? + ORDER BY position + LIMIT ?`, + ) + .bind(generation, from, limits.maxBatches) + .all(); + + if (metadata.results.length === 0) { + const state = await getChangeLogState(db); + await releaseLease(db, consumerId, generation, from, owner, limits.now); + if (state && state.generation === generation && state.head !== from) { + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} cannot resume from ${from}; log head is ${state.head}`, + ); + } + return null; + } + + const expectedFirst = BigInt(from) + 1n; + if (BigInt(String(metadata.results[0]!.position)) !== expectedFirst) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} has a gap after ${from}`, + ); + } + + for (let index = 0; index < metadata.results.length; index++) { + const expected = BigInt(from) + BigInt(index) + 1n; + if (BigInt(String(metadata.results[index]!.position)) !== expected) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} has a gap after ${from}`, + ); + } + } + + const selected: BatchMetadataRow[] = []; + let totalChanges = 0; + let totalBytes = 0; + for (const row of metadata.results) { + const count = Number(row.change_count); + const bytes = Number(row.encoded_bytes); + if ( + !Number.isSafeInteger(count) || + count < 1 || + !Number.isSafeInteger(bytes) || + bytes < 1 + ) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new Error(`Change batch ${generation}/${row.position} has invalid bounds`); + } + if ( + selected.length > 0 && + (totalChanges + count > limits.maxChanges || + totalBytes + bytes > limits.maxBytes) + ) { + break; + } + if (count > limits.maxChanges || bytes > limits.maxBytes) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeClaimTooLargeError( + `Change batch ${generation}/${row.position} exceeds this consumer's claim limits`, + ); + } + selected.push(row); + totalChanges += count; + totalBytes += bytes; + } + + const through = String(selected.at(-1)!.position); + const rows = await db + .prepare( + `SELECT position, phase, change_count, encoded_bytes, changes_json + FROM change_batches + WHERE generation_id = ? AND position > ? AND position <= ? + ORDER BY position`, + ) + .bind(generation, from, through) + .all(); + if (rows.results.length !== selected.length) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new ChangeHistoryGapError( + `Change consumer ${consumerId} lost claimed history`, + ); + } + + const collections = new Set( + parseStringArray( + leased.configured_collections_json, + `Change consumer ${consumerId} collections`, + ), + ); + const phases = new Set( + parseStringArray( + leased.configured_phases_json, + `Change consumer ${consumerId} phases`, + ), + ); + const coalesced = new Map(); + for (const row of rows.results) { + if (!phases.has(row.phase)) continue; + for (const change of decodeBatchChanges(row, generation)) { + if (!collections.has(change.collection)) continue; + coalesced.delete(change.uri); + coalesced.set(change.uri, change); + } + } + + const claim: ChangeClaim = { + consumerId, + generation, + from, + through, + changes: [...coalesced.values()], + attempt: Number(leased.attempts) + 1, + leaseExpiresAt, + leaseOwner: owner, + }; + if (claim.changes.length > 0) return claim; + + await acknowledgeChanges(db, claim, { now: limits.now }); + if (autoAdvanced === limits.maxAutoAdvanceRanges) return null; + } + return null; +} + +/** Hydrate the newest canonical state for each coalesced claimed URI. */ +export async function hydrateChanges( + db: Database, + config: ContrailConfig, + claim: ChangeClaim, +): Promise { + const byCollection = new Map(); + for (const change of claim.changes) { + const uris = byCollection.get(change.collection) ?? []; + uris.push(change.uri); + byCollection.set(change.collection, uris); + } + + const found = new Map(); + for (const [collection, uris] of byCollection) { + const short = resolveCollectionKey(config, collection); + if (!short) { + throw new Error(`Claim references unconfigured collection ${collection}`); + } + const table = recordsTableName(short); + for (let index = 0; index < uris.length; index += 50) { + const chunk = uris.slice(index, index + 50); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare( + `SELECT uri, did, rkey, cid, record, time_us, indexed_at + FROM ${table} WHERE uri IN (${placeholders})`, + ) + .bind(...chunk) + .all<{ + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }>(); + for (const row of rows.results) { + let value: unknown; + try { + value = JSON.parse(row.record); + } catch { + throw new Error(`Current record ${row.uri} is not valid JSON`); + } + found.set(row.uri, { + uri: row.uri, + did: row.did, + collection, + rkey: row.rkey, + cid: row.cid, + value, + timeUs: Number(row.time_us), + indexedAt: Number(row.indexed_at), + }); + } + } + } + + const currentRecords: CurrentRecord[] = []; + const absentUris: string[] = []; + for (const change of claim.changes) { + const current = found.get(change.uri); + if (current) currentRecords.push(current); + else absentUris.push(change.uri); + } + return { + consumerId: claim.consumerId, + cursor: { + generation: claim.generation, + from: claim.from, + through: claim.through, + }, + changes: claim.changes, + currentRecords, + absentUris, + }; +} + +export async function acknowledgeChanges( + db: Database, + claim: ChangeClaim, + options: { now?: number } = {}, +): Promise { + const now = timestamp(options.now); + const acknowledged = await db + .prepare( + `UPDATE change_consumers + SET acknowledged_position = ?, lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_success_at = ?, last_error_code = NULL, last_error_at = NULL, + updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + AND ? > acknowledged_position + AND ? <= ( + SELECT head_position FROM change_log_state + WHERE id = 1 AND generation_id = ? + ) + RETURNING consumer_id`, + ) + .bind( + claim.through, + now, + now, + claim.consumerId, + claim.generation, + claim.from, + claim.leaseOwner, + now, + claim.through, + claim.through, + claim.generation, + ) + .first<{ consumer_id: string }>(); + if (!acknowledged) { + throw new ChangeLeaseLostError( + `Change claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +export async function renewChangeClaim( + db: Database, + claim: ChangeClaim, + options: { leaseMs?: number; now?: number } = {}, +): Promise { + const now = timestamp(options.now); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const expires = now + leaseMs; + const renewed = await db + .prepare( + `UPDATE change_consumers + SET lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + expires, + now, + claim.consumerId, + claim.generation, + claim.from, + claim.leaseOwner, + now, + ) + .first<{ consumer_id: string }>(); + if (!renewed) { + throw new ChangeLeaseLostError( + `Change claim for ${claim.consumerId} cannot be renewed`, + ); + } + return { ...claim, leaseExpiresAt: expires }; +} + +export async function failChanges( + db: Database, + claim: ChangeClaim, + failure: ChangeFailure, + options: { now?: number } = {}, +): Promise<{ attempts: number; nextAttemptAt: number | null }> { + const now = timestamp(options.now); + if (!FAILURE_CODE.test(failure.code)) { + throw new TypeError( + "Change failure code must contain 1-64 safe identifier characters", + ); + } + if ( + failure.nextAttemptAt !== null && + (!Number.isSafeInteger(failure.nextAttemptAt) || + failure.nextAttemptAt < now) + ) { + throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); + } + const failed = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, + attempts = attempts + 1, next_attempt_at = ?, + last_error_code = ?, last_error_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING attempts, next_attempt_at`, + ) + .bind( + failure.nextAttemptAt, + failure.code, + now, + now, + claim.consumerId, + claim.generation, + claim.from, + claim.leaseOwner, + now, + ) + .first<{ attempts: number | string; next_attempt_at: number | string | null }>(); + if (!failed) { + throw new ChangeLeaseLostError( + `Change claim for ${claim.consumerId} is stale or expired`, + ); + } + return { + attempts: Number(failed.attempts), + nextAttemptAt: nullableNumber(failed.next_attempt_at), + }; +} + +export async function retryChangeConsumer( + db: Database, + consumerId: string, + options: { now?: number } = {}, +): Promise { + const now = timestamp(options.now); + if (!(await getChangeLogState(db))) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const retried = await db + .prepare( + `UPDATE change_consumers + SET next_attempt_at = NULL, last_error_code = NULL, + last_error_at = NULL, updated_at = ? + WHERE consumer_id = ? + AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id`, + ) + .bind(now, consumerId, now) + .first<{ consumer_id: string }>(); + if (!retried) { + const existing = await durableConsumer(db, consumerId); + if (!existing) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + throw new ChangeLeaseLostError( + `Change consumer ${consumerId} currently has an active lease`, + ); + } +} + +export async function getChangesStatus(db: Database): Promise { + const state = await getChangeLogState(db); + if (!state) { + return { + enabled: false, + state: null, + rows: 0, + changes: 0, + bytes: 0, + oldestRetainedAt: null, + consumers: [], + }; + } + const aggregate = await db + .prepare( + `SELECT COUNT(*) AS rows, COALESCE(SUM(change_count), 0) AS changes, + COALESCE(SUM(encoded_bytes), 0) AS bytes, + MIN(created_at) AS oldest_retained_at + FROM change_batches WHERE generation_id = ?`, + ) + .bind(state.generation) + .first<{ + rows: number | string; + changes: number | string; + bytes: number | string; + oldest_retained_at: number | string | null; + }>(); + const rows = await db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, + initial_mode, required_for_activation, bootstrap_state, + lease_owner, lease_expires_at, attempts, next_attempt_at, + last_success_at, last_error_code, last_error_at + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + const consumers: ChangeConsumerStatus[] = []; + for (const row of rows.results) { + const backlog = await db + .prepare( + `SELECT COUNT(*) AS backlog_batches, + COALESCE(SUM(change_count), 0) AS backlog_changes, + COALESCE(SUM(encoded_bytes), 0) AS backlog_bytes, + MIN(created_at) AS oldest_pending_at + FROM change_batches + WHERE generation_id = ? AND position > ?`, + ) + .bind(row.generation_id, row.acknowledged_position) + .first(); + const generationMatches = row.generation_id === state.generation; + consumers.push({ + id: row.consumer_id, + generation: row.generation_id, + position: String(row.acknowledged_position), + bootstrapState: generationMatches ? row.bootstrap_state : "reset-required", + initialMode: row.initial_mode, + requiredForActivation: Number(row.required_for_activation) === 1, + attempts: Number(row.attempts), + nextAttemptAt: nullableNumber(row.next_attempt_at), + lastSuccessAt: nullableNumber(row.last_success_at), + lastErrorCode: row.last_error_code, + lastErrorAt: nullableNumber(row.last_error_at), + leased: row.lease_owner !== null, + leaseExpiresAt: nullableNumber(row.lease_expires_at), + backlogBatches: Number(backlog?.backlog_batches ?? 0), + backlogChanges: Number(backlog?.backlog_changes ?? 0), + backlogBytes: Number(backlog?.backlog_bytes ?? 0), + oldestPendingAt: nullableNumber(backlog?.oldest_pending_at ?? null), + generationMatches, + }); + } + return { + enabled: true, + state, + rows: Number(aggregate?.rows ?? 0), + changes: Number(aggregate?.changes ?? 0), + bytes: Number(aggregate?.bytes ?? 0), + oldestRetainedAt: nullableNumber(aggregate?.oldest_retained_at ?? null), + consumers, + }; +} + +type DatabaseResolver = (db?: Database) => Database; + +/** Bound low-level API exposed as `contrail.changes`. */ +export class ChangeConsumers { + constructor( + private readonly config: ContrailConfig, + private readonly database: DatabaseResolver, + ) {} + + register(consumerId: string, db?: Database): Promise { + return registerChangeConsumer(this.database(db), this.config, consumerId); + } + + claim( + consumerId: string, + options?: ChangeClaimOptions, + db?: Database, + ): Promise { + return claimChanges(this.database(db), consumerId, options); + } + + hydrate(claim: ChangeClaim, db?: Database): Promise { + return hydrateChanges(this.database(db), this.config, claim); + } + + ack( + claim: ChangeClaim, + options?: { now?: number }, + db?: Database, + ): Promise { + return acknowledgeChanges(this.database(db), claim, options); + } + + renew( + claim: ChangeClaim, + options?: { leaseMs?: number; now?: number }, + db?: Database, + ): Promise { + return renewChangeClaim(this.database(db), claim, options); + } + + fail( + claim: ChangeClaim, + failure: ChangeFailure, + options?: { now?: number }, + db?: Database, + ): Promise<{ attempts: number; nextAttemptAt: number | null }> { + return failChanges(this.database(db), claim, failure, options); + } + + retry( + consumerId: string, + options?: { now?: number }, + db?: Database, + ): Promise { + return retryChangeConsumer(this.database(db), consumerId, options); + } + + status(db?: Database): Promise { + return getChangesStatus(this.database(db)); + } +} diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 51dfe2f..18b05b5 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -37,7 +37,7 @@ import { } from "../types"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 12; +export const CONTRAIL_SCHEMA_VERSION = 13; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -783,6 +783,17 @@ export async function initSchema( await assertFreshChangeLogGeneration(db); } for (const statement of changes) await runIdempotentDdl(db, statement); + await addColumnIfNotExists( + db, + "change_batches", + "encoded_bytes", + "INTEGER", + ); + await db + .prepare( + "UPDATE change_batches SET encoded_bytes = LENGTH(changes_json) WHERE encoded_bytes IS NULL", + ) + .run(); await initializeChangeLog(db, config); } diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 0e68f6c..47fbcc6 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -35,6 +35,7 @@ export { MAX_CHANGE_BATCH_CHANGES, } from "./core/change-log"; export type { ChangeLogState, RecordChange } from "./core/change-log"; +export * from "./core/changes"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/tests/built-changes.mjs b/packages/contrail/tests/built-changes.mjs new file mode 100644 index 0000000..1c3cf76 --- /dev/null +++ b/packages/contrail/tests/built-changes.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; + +const root = mkdtempSync(join(tmpdir(), "contrail-changes-")); +try { + writeFileSync( + join(root, "contrail.config.js"), + `export const config = { + namespace: "com.example", + profiles: [], + collections: { event: { collection: "com.example.event" } }, + changes: { consumers: { + webhook: { collections: ["com.example.event"], initial: "future" } + } } + };\n`, + ); + const cli = resolve("dist/cli.js"); + const database = join(root, "contrail.sqlite"); + const status = spawnSync( + process.execPath, + [cli, "changes", "status", "--root", root, "--sqlite", database, "--json"], + { cwd: resolve("."), encoding: "utf8" }, + ); + assert.equal(status.status, 0, status.stderr); + const parsed = JSON.parse(status.stdout); + assert.equal(parsed.enabled, true); + assert.equal(parsed.consumers[0].id, "webhook"); + + const retry = spawnSync( + process.execPath, + [cli, "changes", "retry", "webhook", "--root", root, "--sqlite", database], + { cwd: resolve("."), encoding: "utf8" }, + ); + assert.equal(retry.status, 0, retry.stderr); + assert.match(retry.stdout, /retry is now due/); + console.log("built change consumer CLI passed"); +} finally { + rmSync(root, { recursive: true, force: true }); +} diff --git a/packages/contrail/tests/built-sqlite.mjs b/packages/contrail/tests/built-sqlite.mjs index 4c11bb2..be2dbc9 100644 --- a/packages/contrail/tests/built-sqlite.mjs +++ b/packages/contrail/tests/built-sqlite.mjs @@ -12,6 +12,8 @@ assert.equal(row?.ok, 1); const contrail = new Contrail({ namespace: "smoke", collections: {}, db }); await contrail.init(); +assert.equal(typeof contrail.changes.claim, "function"); +assert.equal((await contrail.changes.status()).enabled, false); const statusResponse = await contrail .app() .fetch(new Request("http://localhost/status")); diff --git a/packages/contrail/tests/change-consumers.test.ts b/packages/contrail/tests/change-consumers.test.ts new file mode 100644 index 0000000..71b6184 --- /dev/null +++ b/packages/contrail/tests/change-consumers.test.ts @@ -0,0 +1,351 @@ +import { describe, expect, it } from "vitest"; +import { + ChangeClaimTooLargeError, + ChangeLeaseLostError, + acknowledgeChanges, + claimChanges, + createIngestEvent, + failChanges, + getChangesStatus, + hydrateChanges, + ingestRecords, + initSchema, + registerChangeConsumer, + renewChangeClaim, + resolveConfig, + retryChangeConsumer, + type ChangeClaim, + type ContrailConfig, + type Database, + type ProjectionPhase, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const EVENT = "com.example.event"; +const NOTE = "com.example.note"; +const logger = { log() {}, warn() {}, error() {} }; + +function consumerConfig( + consumers: NonNullable["consumers"], +) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { + event: { collection: EVENT }, + note: { collection: NOTE }, + }, + changes: { consumers }, + }); +} + +function mutation(options: { + collection?: string; + rkey: string; + sourceTime: number; + cid?: string; + name?: string; + operation?: "create" | "update" | "delete"; +}) { + const collection = options.collection ?? EVENT; + const operation = options.operation ?? "update"; + const did = "did:plc:alice"; + return createIngestEvent({ + uri: `at://${did}/${collection}/${options.rkey}`, + did, + collection, + rkey: options.rkey, + operation, + cid: operation === "delete" ? null : (options.cid ?? `cid-${options.sourceTime}`), + value: + operation === "delete" + ? undefined + : { name: options.name ?? `${options.rkey}-${options.sourceTime}` }, + timeUs: options.sourceTime, + indexedAt: options.sourceTime + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: options.sourceTime, + revision: String(options.sourceTime), + cursor: String(options.sourceTime), + }, + }); +} + +async function apply( + db: Database, + config: ReturnType, + events: ReturnType[], + phase: ProjectionPhase = "live", +) { + await ingestRecords(db, events, config, { phase }); +} + +function readyEventConsumer(id = "search") { + return consumerConfig({ + [id]: { + collections: [EVENT], + initial: "history", + }, + }); +} + +describe("durable change consumers", () => { + it("verifies static registration and keeps current consumers pending", async () => { + const db = createSqliteDatabase(":memory:"); + const config = consumerConfig({ + search: { collections: [EVENT], initial: "current" }, + webhook: { collections: [EVENT], phases: ["live"], initial: "future" }, + }); + await initSchema(db, config); + + await expect(registerChangeConsumer(db, config, "search")).resolves.toBeUndefined(); + await expect(registerChangeConsumer(db, config, "missing")).rejects.toThrow( + "not configured", + ); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + expect(await claimChanges(db, "search", { now: 100 })).toBeNull(); + expect(await claimChanges(db, "webhook", { now: 100 })).not.toBeNull(); + }); + + it("claims a bounded range, coalesces URIs, hydrates current state, and acks with CAS", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply( + db, + config, + [mutation({ rkey: "one", sourceTime: 1, cid: "cid-one", name: "one" })], + "historical", + ); + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 2, cid: "cid-two", name: "two" }), + ]); + + const claim = await claimChanges(db, "search", { now: 1_000 }); + expect(claim).toMatchObject({ + consumerId: "search", + from: "0", + through: "2", + attempt: 1, + }); + expect(claim!.changes).toHaveLength(1); + expect(claim!.changes[0]).toMatchObject({ + id: `${claim!.generation}:2:0`, + uri: `at://did:plc:alice/${EVENT}/one`, + cid: "cid-two", + }); + + const delivery = await hydrateChanges(db, config, claim!); + expect(delivery.cursor).toEqual({ + generation: claim!.generation, + from: "0", + through: "2", + }); + expect(delivery.currentRecords).toHaveLength(1); + expect(delivery.currentRecords[0]).toMatchObject({ + cid: "cid-two", + value: { name: "two" }, + }); + expect(delivery.absentUris).toEqual([]); + + await acknowledgeChanges(db, claim!, { now: 1_001 }); + await expect( + acknowledgeChanges(db, claim!, { now: 1_002 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + expect(await claimChanges(db, "search", { now: 1_003 })).toBeNull(); + + const status = await getChangesStatus(db); + expect(status).toMatchObject({ enabled: true, rows: 2, changes: 2 }); + expect(status.consumers[0]).toMatchObject({ + id: "search", + position: "2", + backlogBatches: 0, + attempts: 0, + leased: false, + }); + + // Unrelated schema migrations must preserve mutable consumer progress. + await db + .prepare( + "UPDATE _contrail_meta SET value = 'stale' WHERE key = 'schema_fingerprint'", + ) + .run(); + await initSchema(db, config); + expect((await getChangesStatus(db)).consumers[0].position).toBe("2"); + }); + + it("hydrates newest state when delete/recreate races a claimed change", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + const claim = await claimChanges(db, "search", { now: 100 }); + + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 2, operation: "delete" }), + ]); + let delivery = await hydrateChanges(db, config, claim!); + expect(delivery.currentRecords).toEqual([]); + expect(delivery.absentUris).toEqual([ + `at://did:plc:alice/${EVENT}/one`, + ]); + + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 3, cid: "cid-three", name: "three" }), + ]); + delivery = await hydrateChanges(db, config, claim!); + expect(delivery.absentUris).toEqual([]); + expect(delivery.currentRecords[0]).toMatchObject({ + cid: "cid-three", + value: { name: "three" }, + }); + }); + + it("leases independently, persists failure backoff, and manually retries", async () => { + const db = createSqliteDatabase(":memory:"); + const config = consumerConfig({ + search: { collections: [EVENT], initial: "history" }, + analytics: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + + const search = await claimChanges(db, "search", { now: 1_000 }); + const analytics = await claimChanges(db, "analytics", { now: 1_000 }); + expect(search?.leaseOwner).not.toBe(analytics?.leaseOwner); + + expect( + await failChanges( + db, + search!, + { code: "destination_unavailable", nextAttemptAt: 2_000 }, + { now: 1_001 }, + ), + ).toEqual({ attempts: 1, nextAttemptAt: 2_000 }); + await acknowledgeChanges(db, analytics!, { now: 1_001 }); + + expect(await claimChanges(db, "search", { now: 1_500 })).toBeNull(); + await retryChangeConsumer(db, "search", { now: 1_500 }); + const retry = await claimChanges(db, "search", { now: 1_500 }); + expect(retry?.attempt).toBe(2); + await acknowledgeChanges(db, retry!, { now: 1_501 }); + + const status = await getChangesStatus(db); + expect(status.consumers.find((item) => item.id === "analytics")).toMatchObject({ + position: "1", + attempts: 0, + }); + expect(status.consumers.find((item) => item.id === "search")).toMatchObject({ + position: "1", + attempts: 0, + nextAttemptAt: null, + lastErrorCode: null, + }); + }); + + it("allows only one concurrent lease and rejects an expired owner", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + + const claims = await Promise.all([ + claimChanges(db, "search", { now: 100, leaseMs: 50 }), + claimChanges(db, "search", { now: 100, leaseMs: 50 }), + ]); + expect(claims.filter(Boolean)).toHaveLength(1); + const stale = claims.find(Boolean)!; + const replacement = await claimChanges(db, "search", { + now: 151, + leaseMs: 50, + }); + expect(replacement?.leaseOwner).not.toBe(stale.leaseOwner); + expect(replacement?.changes.map((change) => change.id)).toEqual( + stale.changes.map((change) => change.id), + ); + await expect( + acknowledgeChanges(db, stale, { now: 152 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await acknowledgeChanges(db, replacement!, { now: 152 }); + }); + + it("renews a live lease without changing its CAS owner", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + const claim = await claimChanges(db, "search", { + now: 100, + leaseMs: 50, + }); + const renewed = await renewChangeClaim(db, claim!, { + now: 120, + leaseMs: 100, + }); + expect(renewed.leaseOwner).toBe(claim!.leaseOwner); + expect(renewed.leaseExpiresAt).toBe(220); + await acknowledgeChanges(db, renewed, { now: 200 }); + }); + + it("auto-advances irrelevant ranges while preserving independent progress", async () => { + const db = createSqliteDatabase(":memory:"); + const config = consumerConfig({ + events: { collections: [EVENT], phases: ["live"], initial: "future" }, + notes: { collections: [NOTE], phases: ["live"], initial: "future" }, + }); + await initSchema(db, config); + await apply(db, config, [ + mutation({ collection: NOTE, rkey: "note", sourceTime: 1 }), + ]); + await apply(db, config, [mutation({ rkey: "event", sourceTime: 2 })]); + + const eventClaim = await claimChanges(db, "events", { + now: 100, + maxBatches: 1, + }); + expect(eventClaim).toMatchObject({ from: "1", through: "2" }); + expect(eventClaim!.changes[0].collection).toBe(EVENT); + await acknowledgeChanges(db, eventClaim!, { now: 101 }); + + const noteClaim = await claimChanges(db, "notes", { now: 100 }); + expect(noteClaim).toMatchObject({ from: "0", through: "2" }); + expect(noteClaim!.changes[0].collection).toBe(NOTE); + await acknowledgeChanges(db, noteClaim!, { now: 101 }); + }); + + it("releases a lease when the first durable batch exceeds claim limits", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [ + mutation({ rkey: "one", sourceTime: 1 }), + mutation({ rkey: "two", sourceTime: 2 }), + ]); + + await expect( + claimChanges(db, "search", { now: 100, maxChanges: 1 }), + ).rejects.toBeInstanceOf(ChangeClaimTooLargeError); + const claim = await claimChanges(db, "search", { + now: 101, + maxChanges: 2, + }); + expect(claim?.changes).toHaveLength(2); + }); + + it("rejects forged generation cursors and never regresses a checkpoint", async () => { + const db = createSqliteDatabase(":memory:"); + const config = readyEventConsumer(); + await initSchema(db, config); + await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); + const claim = await claimChanges(db, "search", { now: 100 }); + const forged: ChangeClaim = { ...claim!, generation: crypto.randomUUID() }; + await expect( + acknowledgeChanges(db, forged, { now: 101 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await acknowledgeChanges(db, claim!, { now: 101 }); + expect((await getChangesStatus(db)).consumers[0].position).toBe("1"); + }); +}); diff --git a/packages/contrail/tests/postgres-e2e.test.ts b/packages/contrail/tests/postgres-e2e.test.ts index 0e5d499..2e199b8 100644 --- a/packages/contrail/tests/postgres-e2e.test.ts +++ b/packages/contrail/tests/postgres-e2e.test.ts @@ -12,6 +12,10 @@ import pg from "pg"; import { createPostgresDatabase } from "../src/adapters/postgres"; import { initSchema } from "../src/index"; import { + acknowledgeChanges, + claimChanges, + getChangesStatus, + hydrateChanges, ingestRecords, queryRecords, getLastCursor, @@ -55,6 +59,26 @@ const TEST_CONFIG = resolveConfig({ }, }); +const CHANGE_CONFIG = resolveConfig({ + namespace: "com.example", + profiles: [], + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + analytics: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, +}); + const PG_URL = process.env.TEST_DATABASE_URL; if (!PG_URL) { describe.skip("PostgreSQL e2e (TEST_DATABASE_URL not set)", () => { @@ -559,6 +583,44 @@ if (!PG_URL) { }); }); + describe("durable change consumers", () => { + it("claims, hydrates, and acknowledges independently", async () => { + await initSchema(db, CHANGE_CONFIG); + await ingestRecords( + db, + [ + makeEvent({ + uri: "at://did:plc:test/community.lexicon.calendar.event/change", + rkey: "change", + cid: "cid-change", + record: { name: "Change", mode: "online" }, + time_us: 100, + indexed_at: 100, + }), + ], + CHANGE_CONFIG, + ); + + const search = await claimChanges(db, "search", { now: 1_000 }); + const analytics = await claimChanges(db, "analytics", { now: 1_000 }); + expect(search?.through).toBe("1"); + expect(analytics?.through).toBe("1"); + const delivery = await hydrateChanges(db, CHANGE_CONFIG, search!); + expect(delivery.currentRecords[0]).toMatchObject({ + cid: "cid-change", + value: { name: "Change", mode: "online" }, + }); + await acknowledgeChanges(db, search!, { now: 1_001 }); + expect((await getChangesStatus(db)).consumers).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "search", position: "1" }), + expect.objectContaining({ id: "analytics", position: "0" }), + ]), + ); + await acknowledgeChanges(db, analytics!, { now: 1_001 }); + }); + }); + // --- Cursor --- describe("cursor persistence", () => { -- 2.51.2 From 1d27c86f3eb0bd5e6a39ce3ece471bd33ee5ba9e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:21:03 +0200 Subject: [PATCH 03/10] Add current-state consumer bootstrap and retention --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 4 +- packages/contrail/src/cli/commands/changes.ts | 61 +- .../contrail/src/core/change-bootstrap.ts | 663 ++++++++++++++++++ packages/contrail/src/core/change-log.ts | 145 +++- packages/contrail/src/core/changes.ts | 488 ++++++++++++- packages/contrail/src/core/db/schema.ts | 2 +- packages/contrail/src/index.ts | 1 + packages/contrail/tests/built-changes.mjs | 8 + .../contrail/tests/change-bootstrap.test.ts | 354 ++++++++++ packages/contrail/tests/change-log.test.ts | 2 +- .../tests/postgres-concurrent-init.test.ts | 2 +- packages/contrail/tests/postgres-e2e.test.ts | 39 +- packages/contrail/tests/postgres.test.ts | 2 +- 14 files changed, 1730 insertions(+), 43 deletions(-) create mode 100644 packages/contrail/src/core/change-bootstrap.ts create mode 100644 packages/contrail/tests/change-bootstrap.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index b76f0f1..27dd926 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs plus `contrail changes status/retry` commands. Enabling or changing log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 33d0b54..09f1052 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -121,7 +121,9 @@ if (claim) { Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and resolve delete/recreate races from newest canonical state. Consumers lease and progress independently; irrelevant position ranges advance without invoking a handler. Delivery is intentionally at least once—a destination success followed by an acknowledgement crash causes duplicate delivery. Handlers must be idempotent by stable record/document key. -`initial: "future"` and `initial: "history"` consumers can use this API now. `initial: "current"` remains pending until the snapshot-plus-tail bootstrap coordinator lands in the next milestone. `contrail changes status` and `contrail changes retry ` expose private status and manual retry for SQLite or Wrangler D1 deployments. Enabling logging on a populated database, changing coverage, or disabling an existing log fails closed until explicit quiet-boundary migration tooling lands. With no configured consumers, no change-log tables or append writes exist. +`initial: "current"` uses a durable snapshot-plus-tail coordinator. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. + +A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. ## Local development diff --git a/packages/contrail/src/cli/commands/changes.ts b/packages/contrail/src/cli/commands/changes.ts index d035b55..1dfd205 100644 --- a/packages/contrail/src/cli/commands/changes.ts +++ b/packages/contrail/src/cli/commands/changes.ts @@ -13,6 +13,11 @@ interface ChangeCommandOptions { binding: string; sqlite?: string; json?: boolean; + through?: string; + reason?: string; + yes?: boolean; + maxBatches?: number; + olderThan?: number; } async function withChangesDatabase( @@ -68,24 +73,32 @@ export function registerChanges(cli: CAC): void { options( cli.command( "changes [consumer]", - "Private change-log operations: status, retry ", + "Private change-log operations: status, retry, prune, skip", ), ) .option("--json", "Print machine-readable status JSON") + .option("--through ", "Required position for changes skip") + .option("--reason ", "Required operator reason for changes skip") + .option("--yes", "Confirm the data loss caused by changes skip") + .option("--max-batches ", "Maximum rows for one prune slice", { + default: 500, + }) + .option("--older-than ", "Prune only rows older than milliseconds") .action( async ( action: string, consumer: string | undefined, commandOptions: ChangeCommandOptions, ) => { - if (action !== "status" && action !== "retry") { - throw new Error("changes action must be 'status' or 'retry'"); + const actions = ["status", "retry", "prune", "skip"]; + if (!actions.includes(action)) { + throw new Error(`changes action must be one of: ${actions.join(", ")}`); } - if (action === "retry" && !consumer) { - throw new Error("changes retry requires a consumer ID"); + if ((action === "retry" || action === "skip") && !consumer) { + throw new Error(`changes ${action} requires a consumer ID`); } - if (action === "status" && consumer) { - throw new Error("changes status does not accept a consumer ID"); + if ((action === "status" || action === "prune") && consumer) { + throw new Error(`changes ${action} does not accept a consumer ID`); } await withChangesDatabase(commandOptions, async (contrail, db) => { @@ -94,6 +107,40 @@ export function registerChanges(cli: CAC): void { console.log(`change consumer ${consumer}: retry is now due`); return; } + if (action === "skip") { + if (!commandOptions.through || !commandOptions.reason) { + throw new Error("changes skip requires --through and --reason"); + } + await contrail.changes.skip( + consumer!, + { + through: commandOptions.through, + reason: commandOptions.reason, + confirm: commandOptions.yes === true, + }, + db, + ); + console.log( + `change consumer ${consumer}: explicitly skipped through ${commandOptions.through}`, + ); + return; + } + if (action === "prune") { + const result = await contrail.changes.prune( + { + maxBatches: Number(commandOptions.maxBatches), + ...(commandOptions.olderThan === undefined + ? {} + : { olderThan: Number(commandOptions.olderThan) }), + }, + db, + ); + console.log( + `change log: pruned=${result.pruned} floor=${result.retainedFloor} ` + + `safeThrough=${result.safeThrough} done=${result.done}`, + ); + return; + } const status = await contrail.changes.status(db); if (commandOptions.json) { diff --git a/packages/contrail/src/core/change-bootstrap.ts b/packages/contrail/src/core/change-bootstrap.ts new file mode 100644 index 0000000..0cc6a97 --- /dev/null +++ b/packages/contrail/src/core/change-bootstrap.ts @@ -0,0 +1,663 @@ +import type { ContrailConfig, Database } from "./types"; +import { recordsTableName, resolveCollectionKey } from "./types"; +import type { ChangeFailure, CurrentRecord } from "./changes"; +import { + ChangeConsumerNotFoundError, + ChangeGenerationMismatchError, + ChangeLeaseLostError, +} from "./changes"; +import { getChangeLogState } from "./change-log"; + +const DEFAULT_PAGE_SIZE = 100; +const MAX_PAGE_SIZE = 500; +const DEFAULT_LEASE_MS = 30_000; +const MAX_LEASE_MS = 10 * 60_000; +const FAILURE_CODE = /^[a-zA-Z0-9_.:-]{1,64}$/; + +export interface CurrentBootstrapClaimOptions { + pageSize?: number; + leaseMs?: number; + /** @internal Deterministic clock seam for conformance tests. */ + now?: number; +} + +export interface CurrentSnapshotClaim { + kind: "snapshot"; + consumerId: string; + generation: string; + bootstrapToken: string; + collection: string; + fromUri: string | null; + throughUri: string; + pageId: string; + records: CurrentRecord[]; + attempt: number; + leaseExpiresAt: number; + readonly leaseOwner: string; +} + +export interface CurrentActivationClaim { + kind: "activation"; + consumerId: string; + generation: string; + bootstrapToken: string; + target: string; + attempt: number; + leaseExpiresAt: number; + readonly leaseOwner: string; +} + +export interface CurrentBootstrapStatus { + consumerId: string; + generation: string; + state: string; + anchor: string | null; + scanCollection: string | null; + scanCursor: string | null; + target: string | null; + token: string | null; + position: string; +} + +interface BootstrapRow { + consumer_id: string; + generation_id: string; + acknowledged_position: number | string; + configured_collections_json: string; + initial_mode: string; + bootstrap_state: string; + bootstrap_anchor_position: number | string | null; + bootstrap_scan_collection: string | null; + bootstrap_scan_cursor: string | null; + bootstrap_target_position: number | string | null; + bootstrap_token: string | null; + lease_owner: string | null; + lease_expires_at: number | string | null; + attempts: number | string; +} + +function integer( + value: number | undefined, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new TypeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return result; +} + +function now(value: number | undefined): number { + const result = value ?? Date.now(); + if (!Number.isSafeInteger(result) || result < 0) { + throw new TypeError("now must be a non-negative safe integer"); + } + return result; +} + +function parseCollections(value: string, consumerId: string): string[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error(`Current consumer ${consumerId} collections are not valid JSON`); + } + if ( + !Array.isArray(parsed) || + parsed.length === 0 || + !parsed.every((item) => typeof item === "string") + ) { + throw new Error(`Current consumer ${consumerId} collections are malformed`); + } + return [...parsed].sort(); +} + +async function bootstrapRow( + db: Database, + consumerId: string, +): Promise { + return db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, initial_mode, bootstrap_state, + bootstrap_anchor_position, bootstrap_scan_collection, + bootstrap_scan_cursor, bootstrap_target_position, + bootstrap_token, lease_owner, lease_expires_at, attempts + FROM change_consumers WHERE consumer_id = ?`, + ) + .bind(consumerId) + .first(); +} + +async function release( + db: Database, + consumerId: string, + generation: string, + owner: string, + timestamp: number, +): Promise { + await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? AND lease_owner = ?`, + ) + .bind(timestamp, consumerId, generation, owner) + .run(); +} + +function currentRecord( + row: { + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }, + collection: string, +): CurrentRecord { + let value: unknown; + try { + value = JSON.parse(row.record); + } catch { + throw new Error(`Current record ${row.uri} is not valid JSON`); + } + return { + uri: row.uri, + did: row.did, + collection, + rkey: row.rkey, + cid: row.cid, + value, + timeUs: Number(row.time_us), + indexedAt: Number(row.indexed_at), + }; +} + +/** Claim one stable URI-keyset page. Empty collections advance internally; + * exhausting the final collection atomically pins the catch-up target. */ +export async function claimCurrentSnapshotPage( + db: Database, + config: ContrailConfig, + consumerId: string, + options: CurrentBootstrapClaimOptions = {}, +): Promise { + const pageSize = integer( + options.pageSize, + DEFAULT_PAGE_SIZE, + 1, + MAX_PAGE_SIZE, + "pageSize", + ); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const timestamp = now(options.now); + + const configured = config.changes?.consumers[consumerId]; + if (!configured || configured.initial !== "current") { + throw new ChangeConsumerNotFoundError( + `Current-state change consumer ${consumerId} is not configured`, + ); + } + const expectedCollections = [...configured.collections].sort(); + + for (let step = 0; step <= expectedCollections.length; step++) { + const owner = crypto.randomUUID(); + const expires = timestamp + leaseMs; + const leased = await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'scanning', + bootstrap_scan_collection = CASE + WHEN bootstrap_state = 'pending' THEN ? + ELSE bootstrap_scan_collection END, + bootstrap_scan_cursor = CASE + WHEN bootstrap_state = 'pending' THEN NULL + ELSE bootstrap_scan_cursor END, + lease_owner = ?, lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND initial_mode = 'current' + AND bootstrap_state IN ('pending', 'scanning') + AND generation_id = ( + SELECT generation_id FROM change_log_state WHERE id = 1 + ) + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id, generation_id, acknowledged_position, + configured_collections_json, initial_mode, bootstrap_state, + bootstrap_anchor_position, bootstrap_scan_collection, + bootstrap_scan_cursor, bootstrap_target_position, + bootstrap_token, lease_owner, lease_expires_at, attempts`, + ) + .bind( + expectedCollections[0], + owner, + expires, + timestamp, + consumerId, + timestamp, + timestamp, + ) + .first(); + if (!leased) { + const durable = await bootstrapRow(db, consumerId); + if (!durable) { + throw new ChangeConsumerNotFoundError( + `Current-state change consumer ${consumerId} is not initialized`, + ); + } + const state = await getChangeLogState(db); + if (state && durable.generation_id !== state.generation) { + throw new ChangeGenerationMismatchError( + `Current-state consumer ${consumerId} belongs to another generation`, + ); + } + return null; + } + if (!leased.bootstrap_token || !leased.bootstrap_scan_collection) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state change consumer ${consumerId} is malformed`); + } + const durableCollections = parseCollections( + leased.configured_collections_json, + consumerId, + ); + if (JSON.stringify(durableCollections) !== JSON.stringify(expectedCollections)) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state change consumer ${consumerId} changed collections`); + } + const collectionIndex = expectedCollections.indexOf( + leased.bootstrap_scan_collection, + ); + if (collectionIndex < 0) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state change consumer ${consumerId} has an invalid scan collection`); + } + const short = resolveCollectionKey(config, leased.bootstrap_scan_collection); + if (!short) { + await release(db, consumerId, leased.generation_id, owner, timestamp); + throw new Error(`Current-state scan collection is not configured`); + } + const table = recordsTableName(short); + const result = leased.bootstrap_scan_cursor === null + ? await db + .prepare( + `SELECT uri, did, rkey, cid, record, time_us, indexed_at + FROM ${table} ORDER BY uri LIMIT ?`, + ) + .bind(pageSize) + .all<{ + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }>() + : await db + .prepare( + `SELECT uri, did, rkey, cid, record, time_us, indexed_at + FROM ${table} WHERE uri > ? ORDER BY uri LIMIT ?`, + ) + .bind(leased.bootstrap_scan_cursor, pageSize) + .all<{ + uri: string; + did: string; + rkey: string; + cid: string | null; + record: string; + time_us: number | string; + indexed_at: number | string; + }>(); + + if (result.results.length > 0) { + const records = result.results.map((row) => + currentRecord(row, leased.bootstrap_scan_collection!), + ); + const throughUri = records.at(-1)!.uri; + return { + kind: "snapshot", + consumerId, + generation: leased.generation_id, + bootstrapToken: leased.bootstrap_token, + collection: leased.bootstrap_scan_collection, + fromUri: leased.bootstrap_scan_cursor, + throughUri, + pageId: + `${leased.bootstrap_token}:snapshot:` + + `${leased.bootstrap_scan_collection}:${throughUri}`, + records, + attempt: Number(leased.attempts) + 1, + leaseExpiresAt: expires, + leaseOwner: owner, + }; + } + + const nextCollection = expectedCollections[collectionIndex + 1] ?? null; + if (nextCollection !== null) { + await db + .prepare( + `UPDATE change_consumers + SET bootstrap_scan_collection = ?, bootstrap_scan_cursor = NULL, + lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'scanning' AND lease_owner = ?`, + ) + .bind( + nextCollection, + timestamp, + consumerId, + leased.generation_id, + owner, + ) + .run(); + continue; + } + + await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'catching-up', + bootstrap_target_position = ( + SELECT head_position FROM change_log_state WHERE id = 1 + ), + bootstrap_scan_collection = NULL, bootstrap_scan_cursor = NULL, + lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'scanning' AND lease_owner = ?`, + ) + .bind(timestamp, consumerId, leased.generation_id, owner) + .run(); + return null; + } + return null; +} + +export async function acknowledgeCurrentSnapshotPage( + db: Database, + claim: CurrentSnapshotClaim, + options: { now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const acknowledged = await db + .prepare( + `UPDATE change_consumers + SET bootstrap_scan_cursor = ?, lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_success_at = ?, last_error_code = NULL, last_error_at = NULL, + updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'scanning' AND bootstrap_token = ? + AND bootstrap_scan_collection = ? + AND ((bootstrap_scan_cursor = ?) OR + (bootstrap_scan_cursor IS NULL AND CAST(? AS TEXT) IS NULL)) + AND lease_owner = ? AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + claim.throughUri, + timestamp, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.collection, + claim.fromUri, + claim.fromUri, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!acknowledged) { + throw new ChangeLeaseLostError( + `Current snapshot claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +async function failBootstrapLease( + db: Database, + claim: CurrentSnapshotClaim | CurrentActivationClaim, + failure: ChangeFailure, + timestamp: number, +): Promise { + if (!FAILURE_CODE.test(failure.code)) { + throw new TypeError( + "Change failure code must contain 1-64 safe identifier characters", + ); + } + if ( + failure.nextAttemptAt !== null && + (!Number.isSafeInteger(failure.nextAttemptAt) || + failure.nextAttemptAt < timestamp) + ) { + throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); + } + const failed = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = NULL, lease_expires_at = NULL, + attempts = attempts + 1, next_attempt_at = ?, + last_error_code = ?, last_error_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_token = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + failure.nextAttemptAt, + failure.code, + timestamp, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!failed) { + throw new ChangeLeaseLostError( + `Current bootstrap claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +export function failCurrentSnapshotPage( + db: Database, + claim: CurrentSnapshotClaim, + failure: ChangeFailure, + options: { now?: number } = {}, +): Promise { + return failBootstrapLease(db, claim, failure, now(options.now)); +} + +export async function renewCurrentBootstrapClaim< + T extends CurrentSnapshotClaim | CurrentActivationClaim, +>( + db: Database, + claim: T, + options: { leaseMs?: number; now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const expires = timestamp + leaseMs; + const renewed = await db + .prepare( + `UPDATE change_consumers + SET lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_token = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + expires, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!renewed) { + throw new ChangeLeaseLostError( + `Current bootstrap claim for ${claim.consumerId} cannot be renewed`, + ); + } + return { ...claim, leaseExpiresAt: expires }; +} + +/** Claim the idempotent destination-activation step after fixed-target catch-up. */ +export async function claimCurrentActivation( + db: Database, + consumerId: string, + options: { leaseMs?: number; now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const leaseMs = integer( + options.leaseMs, + DEFAULT_LEASE_MS, + 1, + MAX_LEASE_MS, + "leaseMs", + ); + const owner = crypto.randomUUID(); + const expires = timestamp + leaseMs; + const row = await db + .prepare( + `UPDATE change_consumers + SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + WHERE consumer_id = ? AND initial_mode = 'current' + AND bootstrap_state = 'activating' + AND generation_id = ( + SELECT generation_id FROM change_log_state WHERE id = 1 + ) + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND (lease_owner IS NULL OR lease_expires_at <= ?) + RETURNING consumer_id, generation_id, acknowledged_position, + configured_collections_json, initial_mode, bootstrap_state, + bootstrap_anchor_position, bootstrap_scan_collection, + bootstrap_scan_cursor, bootstrap_target_position, + bootstrap_token, lease_owner, lease_expires_at, attempts`, + ) + .bind(owner, expires, timestamp, consumerId, timestamp, timestamp) + .first(); + if (!row) { + const durable = await bootstrapRow(db, consumerId); + const state = await getChangeLogState(db); + if (durable && state && durable.generation_id !== state.generation) { + throw new ChangeGenerationMismatchError( + `Current-state consumer ${consumerId} belongs to another generation`, + ); + } + return null; + } + if (!row.bootstrap_token || row.bootstrap_target_position === null) { + await release(db, consumerId, row.generation_id, owner, timestamp); + throw new Error(`Current activation ${consumerId} is malformed`); + } + return { + kind: "activation", + consumerId, + generation: row.generation_id, + bootstrapToken: row.bootstrap_token, + target: String(row.bootstrap_target_position), + attempt: Number(row.attempts) + 1, + leaseExpiresAt: expires, + leaseOwner: owner, + }; +} + +export async function completeCurrentActivation( + db: Database, + claim: CurrentActivationClaim, + options: { now?: number } = {}, +): Promise { + const timestamp = now(options.now); + const completed = await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'ready', lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_success_at = ?, last_error_code = NULL, last_error_at = NULL, + updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'activating' AND bootstrap_token = ? + AND bootstrap_target_position = ? + AND acknowledged_position = bootstrap_target_position + AND lease_owner = ? AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + timestamp, + timestamp, + claim.consumerId, + claim.generation, + claim.bootstrapToken, + claim.target, + claim.leaseOwner, + timestamp, + ) + .first<{ consumer_id: string }>(); + if (!completed) { + throw new ChangeLeaseLostError( + `Current activation claim for ${claim.consumerId} is stale or expired`, + ); + } +} + +export function failCurrentActivation( + db: Database, + claim: CurrentActivationClaim, + failure: ChangeFailure, + options: { now?: number } = {}, +): Promise { + return failBootstrapLease(db, claim, failure, now(options.now)); +} + +export async function getCurrentBootstrapStatus( + db: Database, + consumerId: string, +): Promise { + const row = await bootstrapRow(db, consumerId); + if (!row || row.initial_mode !== "current") { + throw new ChangeConsumerNotFoundError( + `Current-state change consumer ${consumerId} is not initialized`, + ); + } + return { + consumerId, + generation: row.generation_id, + state: row.bootstrap_state, + anchor: + row.bootstrap_anchor_position === null + ? null + : String(row.bootstrap_anchor_position), + scanCollection: row.bootstrap_scan_collection, + scanCursor: row.bootstrap_scan_cursor, + target: + row.bootstrap_target_position === null + ? null + : String(row.bootstrap_target_position), + token: row.bootstrap_token, + position: String(row.acknowledged_position), + }; +} diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 5973925..b50a551 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -139,6 +139,17 @@ export function buildChangeLogSchema( through_position ${bigint}, PRIMARY KEY (generation_id, collection, phase, from_position) )`, + `CREATE TABLE IF NOT EXISTS change_consumer_actions ( + action_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + consumer_id TEXT NOT NULL, + action TEXT NOT NULL CHECK (action IN ('skip', 'remove')), + from_position ${bigint} NOT NULL, + through_position ${bigint} NOT NULL, + reason TEXT NOT NULL, + created_at ${bigint} NOT NULL + )`, + "CREATE INDEX IF NOT EXISTS idx_change_consumer_actions_consumer ON change_consumer_actions(generation_id, consumer_id, created_at)", ]; } @@ -201,8 +212,88 @@ function canonicalPhases(phases: ProjectionPhase[]): string { return JSON.stringify([...phases].sort()); } -/** Initialize one immutable milestone-1 logging definition. Later milestones - * add explicit quiet-boundary operations for changing this durable definition. */ +function sameStaticConsumer( + actual: ChangeConsumerRow, + expected: NonNullable["consumers"][string], +): boolean { + return ( + actual.configured_collections_json === + canonicalCollections(expected.collections) && + actual.configured_phases_json === + canonicalPhases(changeConsumerPhases(expected)) && + actual.initial_mode === expected.initial && + Number(actual.required_for_activation) === + (expected.requiredForActivation === true ? 1 : 0) + ); +} + +/** Reconcile only additive consumers whose collection/phase coverage is already + * durable. Expanding coverage still requires a fresh generation or an explicit + * old-writer quiet boundary. */ +async function reconcileAdditiveDefinitions( + db: Database, + config: ContrailConfig, + state: ChangeLogStateRow, + definitions: string, +): Promise { + const durableConsumers = await db + .prepare( + `SELECT consumer_id, generation_id, acknowledged_position, + configured_collections_json, configured_phases_json, initial_mode, + required_for_activation, bootstrap_state, + bootstrap_anchor_position + FROM change_consumers ORDER BY consumer_id`, + ) + .all(); + const configured = config.changes?.consumers ?? {}; + for (const durable of durableConsumers.results) { + const expected = configured[durable.consumer_id]; + if (!expected || !sameStaticConsumer(durable, expected)) { + throw new Error( + "Durable change consumers cannot be removed or modified during ordinary initialization", + ); + } + } + + const coverage = await db + .prepare( + `SELECT generation_id, collection, phase, from_position, through_position + FROM change_log_coverage + WHERE generation_id = ? AND through_position IS NULL + ORDER BY collection, phase, from_position`, + ) + .bind(state.generation_id) + .all(); + const durablePairs = new Set( + coverage.results.map((item) => `${item.collection}\0${item.phase}`), + ); + const expectedPairs = changeLogCoverage(config); + if ( + durablePairs.size !== expectedPairs.length || + expectedPairs.some( + (item) => !durablePairs.has(`${item.collection}\0${item.phase}`), + ) + ) { + throw new Error( + "Adding this change consumer expands collection/phase coverage; use a fresh generation or an explicit quiet-boundary migration", + ); + } + + await db + .prepare( + `UPDATE change_log_state SET definitions_json = ? + WHERE id = 1 AND generation_id = ? AND definitions_json = ?`, + ) + .bind(definitions, state.generation_id, state.definitions_json) + .run(); + const updated = (await probeChangeLogSchema(db)).state; + if (!updated || updated.definitions_json !== definitions) { + throw new Error("Change consumer definitions changed concurrently"); + } +} + +/** Initialize the fresh log or safely add consumers over already-logged + * collection/phase coverage. */ export async function initializeChangeLog( db: Database, config: ContrailConfig, @@ -211,19 +302,26 @@ export async function initializeChangeLog( const definitions = canonicalChangeDefinitions(config); const now = Date.now(); - const candidateGeneration = crypto.randomUUID(); - const statements: Statement[] = [ - db - .prepare( - `INSERT INTO change_log_state - (id, generation_id, head_position, retained_floor_position, - definitions_json, created_at) - VALUES (1, ?, 0, 0, ?, ?) - ON CONFLICT(id) DO NOTHING`, - ) - .bind(candidateGeneration, definitions, now), - ]; + await db + .prepare( + `INSERT INTO change_log_state + (id, generation_id, head_position, retained_floor_position, + definitions_json, created_at) + VALUES (1, ?, 0, 0, ?, ?) + ON CONFLICT(id) DO NOTHING`, + ) + .bind(crypto.randomUUID(), definitions, now) + .run(); + + let state = (await probeChangeLogSchema(db)).state; + if (!state) throw new Error("Could not initialize change-log state"); + if (state.definitions_json !== definitions) { + await reconcileAdditiveDefinitions(db, config, state, definitions); + state = (await probeChangeLogSchema(db)).state; + if (!state) throw new Error("Could not reload change-log state"); + } + const statements: Statement[] = []; for (const [consumerId, consumer] of Object.entries( config.changes?.consumers ?? {}, ).sort(([left], [right]) => left.localeCompare(right))) { @@ -235,9 +333,13 @@ export async function initializeChangeLog( (consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, - bootstrap_anchor_position, attempts, updated_at) - SELECT ?, generation_id, 0, ?, ?, ?, ?, ?, - CASE WHEN ? = 'current' THEN 0 ELSE NULL END, + bootstrap_anchor_position, bootstrap_token, attempts, updated_at) + SELECT ?, generation_id, + CASE WHEN ? = 'history' THEN retained_floor_position + ELSE head_position END, + ?, ?, ?, ?, ?, + CASE WHEN ? = 'current' THEN head_position ELSE NULL END, + CASE WHEN ? = 'current' THEN ? ELSE NULL END, 0, ? FROM change_log_state WHERE id = 1 AND definitions_json = ? @@ -245,12 +347,15 @@ export async function initializeChangeLog( ) .bind( consumerId, + consumer.initial, canonicalCollections(consumer.collections), canonicalPhases(changeConsumerPhases(consumer)), consumer.initial, consumer.requiredForActivation === true ? 1 : 0, initialReady, consumer.initial, + consumer.initial, + crypto.randomUUID(), now, definitions, ), @@ -273,11 +378,7 @@ export async function initializeChangeLog( ); } - // Keep initialization under conservative D1 statement limits. The durable - // definitions_json winner makes these resumable chunks safe under concurrent - // initialization; init does not return until the complete set verifies. - await db.batch(statements.slice(0, 1)); - for (let index = 1; index < statements.length; index += 50) { + for (let index = 0; index < statements.length; index += 50) { await db.batch(statements.slice(index, index + 50)); } await assertChangeLogDefinition(db, config); diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 8482453..4875240 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -13,6 +13,20 @@ import { type ChangeLogState, type RecordChange, } from "./change-log"; +import { + acknowledgeCurrentSnapshotPage, + claimCurrentActivation, + claimCurrentSnapshotPage, + completeCurrentActivation, + failCurrentActivation, + failCurrentSnapshotPage, + getCurrentBootstrapStatus, + renewCurrentBootstrapClaim, + type CurrentActivationClaim, + type CurrentBootstrapClaimOptions, + type CurrentBootstrapStatus, + type CurrentSnapshotClaim, +} from "./change-bootstrap"; const DEFAULT_MAX_BATCHES = 20; const DEFAULT_MAX_CHANGES = 500; @@ -51,6 +65,10 @@ export interface ChangeClaim { changes: RecordChange[]; attempt: number; leaseExpiresAt: number; + /** Generation-scoped destination token during current-state catch-up. */ + readonly bootstrapToken?: string; + /** Fixed catch-up target for a current-state bootstrap claim. */ + readonly bootstrapTarget?: string; /** Opaque CAS capability. Do not pass this field to delivery handlers. */ readonly leaseOwner: string; } @@ -125,6 +143,8 @@ interface DurableConsumerRow { initial_mode: string; required_for_activation: number | string; bootstrap_state: string; + bootstrap_target_position: number | string | null; + bootstrap_token: string | null; lease_owner: string | null; lease_expires_at: number | string | null; attempts: number | string; @@ -264,6 +284,7 @@ async function durableConsumer( `SELECT consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, + bootstrap_target_position, bootstrap_token, lease_owner, lease_expires_at, attempts, next_attempt_at, last_success_at, last_error_code, last_error_at FROM change_consumers WHERE consumer_id = ?`, @@ -380,12 +401,14 @@ export async function registerChangeConsumer( /** Claim one bounded contiguous position range. Irrelevant ranges advance by * CAS without invoking application code. */ -export async function claimChanges( +async function claimChangeRange( db: Database, consumerId: string, options: ChangeClaimOptions = {}, + bootstrap = false, ): Promise { const limits = normalizedClaimOptions(options); + const requiredState = bootstrap ? "catching-up" : "ready"; if (!(await getChangeLogState(db))) { throw new ChangeConsumerNotFoundError( `Change consumer ${consumerId} is not initialized`, @@ -404,7 +427,7 @@ export async function claimChanges( `UPDATE change_consumers SET lease_owner = ?, lease_expires_at = ?, updated_at = ? WHERE consumer_id = ? - AND bootstrap_state = 'ready' + AND bootstrap_state = '${requiredState}' AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) AND acknowledged_position >= (SELECT retained_floor_position FROM change_log_state WHERE id = 1) AND (next_attempt_at IS NULL OR next_attempt_at <= ?) @@ -412,6 +435,7 @@ export async function claimChanges( RETURNING consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, + bootstrap_target_position, bootstrap_token, lease_owner, lease_expires_at, attempts, next_attempt_at, last_success_at, last_error_code, last_error_at`, ) @@ -451,23 +475,57 @@ export async function claimChanges( const generation = leased.generation_id; const from = String(leased.acknowledged_position); + const bootstrapTarget = bootstrap + ? String(leased.bootstrap_target_position) + : null; + if (bootstrap && leased.bootstrap_target_position === null) { + await releaseLease(db, consumerId, generation, from, owner, limits.now); + throw new Error(`Change consumer ${consumerId} has no bootstrap target`); + } const metadata = await db .prepare( `SELECT position, phase, change_count, encoded_bytes FROM change_batches WHERE generation_id = ? AND position > ? + AND (? IS NULL OR position <= ?) ORDER BY position LIMIT ?`, ) - .bind(generation, from, limits.maxBatches) + .bind( + generation, + from, + bootstrapTarget, + bootstrapTarget, + limits.maxBatches, + ) .all(); if (metadata.results.length === 0) { + if (bootstrap && from === bootstrapTarget) { + await db + .prepare( + `UPDATE change_consumers + SET bootstrap_state = 'activating', lease_owner = NULL, + lease_expires_at = NULL, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND bootstrap_state = 'catching-up' + AND acknowledged_position = bootstrap_target_position + AND acknowledged_position = ? AND lease_owner = ?`, + ) + .bind(limits.now, consumerId, generation, from, owner) + .run(); + return null; + } const state = await getChangeLogState(db); await releaseLease(db, consumerId, generation, from, owner, limits.now); - if (state && state.generation === generation && state.head !== from) { + const expectedHead = bootstrap ? bootstrapTarget : state?.head; + if ( + expectedHead !== null && + expectedHead !== undefined && + expectedHead !== from + ) { throw new ChangeHistoryGapError( - `Change consumer ${consumerId} cannot resume from ${from}; log head is ${state.head}`, + `Change consumer ${consumerId} cannot resume from ${from}; target is ${expectedHead}`, ); } return null; @@ -571,6 +629,10 @@ export async function claimChanges( changes: [...coalesced.values()], attempt: Number(leased.attempts) + 1, leaseExpiresAt, + ...(leased.bootstrap_token === null + ? {} + : { bootstrapToken: leased.bootstrap_token }), + ...(bootstrapTarget === null ? {} : { bootstrapTarget }), leaseOwner: owner, }; if (claim.changes.length > 0) return claim; @@ -581,6 +643,23 @@ export async function claimChanges( return null; } +export function claimChanges( + db: Database, + consumerId: string, + options: ChangeClaimOptions = {}, +): Promise { + return claimChangeRange(db, consumerId, options, false); +} + +/** Claim changes only through the fixed target of a current-state bootstrap. */ +export function claimCurrentBootstrapChanges( + db: Database, + consumerId: string, + options: ChangeClaimOptions = {}, +): Promise { + return claimChangeRange(db, consumerId, options, true); +} + /** Hydrate the newest canonical state for each coalesced claimed URI. */ export async function hydrateChanges( db: Database, @@ -669,8 +748,13 @@ export async function acknowledgeChanges( const acknowledged = await db .prepare( `UPDATE change_consumers - SET acknowledged_position = ?, lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + SET acknowledged_position = ?, + bootstrap_state = CASE + WHEN bootstrap_state = 'catching-up' + AND ? = bootstrap_target_position THEN 'activating' + ELSE bootstrap_state END, + lease_owner = NULL, lease_expires_at = NULL, + attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? @@ -684,6 +768,7 @@ export async function acknowledgeChanges( RETURNING consumer_id`, ) .bind( + claim.through, claim.through, now, now, @@ -866,6 +951,7 @@ export async function getChangesStatus(db: Database): Promise { `SELECT consumer_id, generation_id, acknowledged_position, configured_collections_json, configured_phases_json, initial_mode, required_for_activation, bootstrap_state, + bootstrap_target_position, bootstrap_token, lease_owner, lease_expires_at, attempts, next_attempt_at, last_success_at, last_error_code, last_error_at FROM change_consumers ORDER BY consumer_id`, @@ -917,6 +1003,280 @@ export async function getChangesStatus(db: Database): Promise { }; } +export interface RequiredChangeConsumerReadiness { + ready: boolean; + through: string; + pending: Array<{ + id: string; + state: string; + position: string; + }>; +} + +/** Check activation-gating consumers against one fixed log target. */ +export async function getRequiredChangeConsumerReadiness( + db: Database, + through?: string, +): Promise { + const state = await getChangeLogState(db); + if (!state) return { ready: true, through: "0", pending: [] }; + const target = through ?? state.head; + if (!/^\d+$/.test(target) || BigInt(target) > BigInt(state.head)) { + throw new TypeError("Required-consumer target must be a retained log position at or below head"); + } + const rows = await db + .prepare( + `SELECT consumer_id, bootstrap_state, acknowledged_position + FROM change_consumers + WHERE generation_id = ? AND required_for_activation = 1 + ORDER BY consumer_id`, + ) + .bind(state.generation) + .all<{ + consumer_id: string; + bootstrap_state: string; + acknowledged_position: number | string; + }>(); + const pending = rows.results + .filter( + (row) => + row.bootstrap_state !== "ready" || + BigInt(String(row.acknowledged_position)) < BigInt(target), + ) + .map((row) => ({ + id: row.consumer_id, + state: row.bootstrap_state, + position: String(row.acknowledged_position), + })); + return { ready: pending.length === 0, through: target, pending }; +} + +export async function assertRequiredChangeConsumersReady( + db: Database, + through?: string, +): Promise { + const readiness = await getRequiredChangeConsumerReadiness(db, through); + if (!readiness.ready) { + throw new Error( + `Required change consumers are not ready through ${readiness.through}: ` + + readiness.pending.map((item) => item.id).join(", "), + ); + } +} + +export interface SkipChangeConsumerOptions { + through: string; + reason: string; + confirm: boolean; + /** @internal Deterministic clock seam for conformance tests. */ + now?: number; +} + +/** Explicit audited data-loss operation for one ready consumer. */ +export async function skipChangeConsumer( + db: Database, + consumerId: string, + options: SkipChangeConsumerOptions, +): Promise { + if (options.confirm !== true) { + throw new Error("Skipping change delivery requires confirm: true"); + } + const reason = options.reason.trim(); + if (reason.length === 0 || reason.length > 256) { + throw new TypeError("Skip reason must contain 1-256 characters"); + } + if (!/^\d+$/.test(options.through)) { + throw new TypeError("Skip position must be an opaque decimal string"); + } + const timestampValue = timestamp(options.now); + const state = await getChangeLogState(db); + const consumer = state ? await durableConsumer(db, consumerId) : null; + if (!state || !consumer) { + throw new ChangeConsumerNotFoundError( + `Change consumer ${consumerId} is not initialized`, + ); + } + const from = String(consumer.acknowledged_position); + if ( + consumer.generation_id !== state.generation || + consumer.bootstrap_state !== "ready" || + BigInt(options.through) <= BigInt(from) || + BigInt(options.through) > BigInt(state.head) + ) { + throw new Error("Skip position is outside this ready consumer's pending range"); + } + const actionId = crypto.randomUUID(); + const actionMarker = `operator_skip:${actionId}`; + const results = await db.batch([ + db + .prepare( + `UPDATE change_consumers + SET acknowledged_position = ?, lease_owner = NULL, + lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + last_error_code = ?, last_error_at = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? + AND (lease_owner IS NULL OR lease_expires_at <= ?)`, + ) + .bind( + options.through, + actionMarker, + timestampValue, + timestampValue, + consumerId, + state.generation, + from, + timestampValue, + ), + db + .prepare( + `INSERT INTO change_consumer_actions + (action_id, generation_id, consumer_id, action, from_position, + through_position, reason, created_at) + SELECT ?, generation_id, consumer_id, 'skip', ?, + acknowledged_position, ?, ? + FROM change_consumers + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND last_error_code = ?`, + ) + .bind( + actionId, + from, + reason, + timestampValue, + consumerId, + state.generation, + options.through, + actionMarker, + ), + ]); + if (affectedRows(results[0]) !== 1 || affectedRows(results[1]) !== 1) { + throw new ChangeLeaseLostError( + `Change consumer ${consumerId} changed during skip`, + ); + } +} + +export interface PruneChangesOptions { + maxBatches?: number; + /** Delete only batches older than this timestamp. */ + olderThan?: number; +} + +export interface PruneChangesResult { + pruned: number; + retainedFloor: string; + safeThrough: string; + done: boolean; +} + +function affectedRows(result: unknown): number { + if (!result || typeof result !== "object") return 0; + const value = result as { + changes?: unknown; + rowsAffected?: unknown; + meta?: { changes?: unknown }; + }; + return Number(value.changes ?? value.rowsAffected ?? value.meta?.changes ?? 0); +} + +/** Consumer-aware bounded pruning. The slowest durable position, including a + * current bootstrap anchor, is the hard safety boundary. */ +export async function pruneChanges( + db: Database, + options: PruneChangesOptions = {}, +): Promise { + const maxBatches = integer( + options.maxBatches, + 500, + 1, + 5_000, + "maxBatches", + ); + if ( + options.olderThan !== undefined && + (!Number.isSafeInteger(options.olderThan) || options.olderThan < 0) + ) { + throw new TypeError("olderThan must be a non-negative safe timestamp"); + } + const state = await getChangeLogState(db); + if (!state) { + return { + pruned: 0, + retainedFloor: "0", + safeThrough: "0", + done: true, + }; + } + const slowest = await db + .prepare( + `SELECT MIN(acknowledged_position) AS safe_through + FROM change_consumers WHERE generation_id = ?`, + ) + .bind(state.generation) + .first<{ safe_through: number | string | null }>(); + const safeThrough = String(slowest?.safe_through ?? state.retainedFloor); + if (BigInt(safeThrough) <= BigInt(state.retainedFloor)) { + return { + pruned: 0, + retainedFloor: state.retainedFloor, + safeThrough, + done: true, + }; + } + + const results = await db.batch([ + db + .prepare( + `DELETE FROM change_batches + WHERE generation_id = ? AND position IN ( + SELECT position FROM change_batches + WHERE generation_id = ? AND position > ? AND position <= ? + AND (? IS NULL OR created_at < ?) + ORDER BY position LIMIT ? + )`, + ) + .bind( + state.generation, + state.generation, + state.retainedFloor, + safeThrough, + options.olderThan ?? null, + options.olderThan ?? null, + maxBatches, + ), + db + .prepare( + `UPDATE change_log_state + SET retained_floor_position = CASE + WHEN EXISTS ( + SELECT 1 FROM change_batches WHERE generation_id = ? + ) THEN ( + SELECT MIN(position) - 1 FROM change_batches + WHERE generation_id = ? + ) + ELSE head_position END + WHERE id = 1 AND generation_id = ?`, + ) + .bind(state.generation, state.generation, state.generation), + ]); + const updated = await getChangeLogState(db); + if (!updated || updated.generation !== state.generation) { + throw new ChangeGenerationMismatchError( + "Change-log generation changed during pruning", + ); + } + const pruned = affectedRows(results[0]); + return { + pruned, + retainedFloor: updated.retainedFloor, + safeThrough, + done: + pruned < maxBatches || + BigInt(updated.retainedFloor) >= BigInt(safeThrough), + }; +} + type DatabaseResolver = (db?: Database) => Database; /** Bound low-level API exposed as `contrail.changes`. */ @@ -938,6 +1298,98 @@ export class ChangeConsumers { return claimChanges(this.database(db), consumerId, options); } + claimBootstrapChanges( + consumerId: string, + options?: ChangeClaimOptions, + db?: Database, + ): Promise { + return claimCurrentBootstrapChanges( + this.database(db), + consumerId, + options, + ); + } + + claimSnapshotPage( + consumerId: string, + options?: CurrentBootstrapClaimOptions, + db?: Database, + ): Promise { + return claimCurrentSnapshotPage( + this.database(db), + this.config, + consumerId, + options, + ); + } + + ackSnapshotPage( + claim: CurrentSnapshotClaim, + options?: { now?: number }, + db?: Database, + ): Promise { + return acknowledgeCurrentSnapshotPage(this.database(db), claim, options); + } + + failSnapshotPage( + claim: CurrentSnapshotClaim, + failure: ChangeFailure, + options?: { now?: number }, + db?: Database, + ): Promise { + return failCurrentSnapshotPage( + this.database(db), + claim, + failure, + options, + ); + } + + claimActivation( + consumerId: string, + options?: { leaseMs?: number; now?: number }, + db?: Database, + ): Promise { + return claimCurrentActivation(this.database(db), consumerId, options); + } + + completeActivation( + claim: CurrentActivationClaim, + options?: { now?: number }, + db?: Database, + ): Promise { + return completeCurrentActivation(this.database(db), claim, options); + } + + renewBootstrap( + claim: T, + options?: { leaseMs?: number; now?: number }, + db?: Database, + ): Promise { + return renewCurrentBootstrapClaim(this.database(db), claim, options); + } + + failActivation( + claim: CurrentActivationClaim, + failure: ChangeFailure, + options?: { now?: number }, + db?: Database, + ): Promise { + return failCurrentActivation( + this.database(db), + claim, + failure, + options, + ); + } + + bootstrapStatus( + consumerId: string, + db?: Database, + ): Promise { + return getCurrentBootstrapStatus(this.database(db), consumerId); + } + hydrate(claim: ChangeClaim, db?: Database): Promise { return hydrateChanges(this.database(db), this.config, claim); } @@ -978,4 +1430,26 @@ export class ChangeConsumers { status(db?: Database): Promise { return getChangesStatus(this.database(db)); } + + readiness( + through?: string, + db?: Database, + ): Promise { + return getRequiredChangeConsumerReadiness(this.database(db), through); + } + + skip( + consumerId: string, + options: SkipChangeConsumerOptions, + db?: Database, + ): Promise { + return skipChangeConsumer(this.database(db), consumerId, options); + } + + prune( + options?: PruneChangesOptions, + db?: Database, + ): Promise { + return pruneChanges(this.database(db), options); + } } diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 18b05b5..d412723 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -37,7 +37,7 @@ import { } from "../types"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 13; +export const CONTRAIL_SCHEMA_VERSION = 14; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 47fbcc6..36b6e5d 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -36,6 +36,7 @@ export { } from "./core/change-log"; export type { ChangeLogState, RecordChange } from "./core/change-log"; export * from "./core/changes"; +export * from "./core/change-bootstrap"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/tests/built-changes.mjs b/packages/contrail/tests/built-changes.mjs index 1c3cf76..e73a1e8 100644 --- a/packages/contrail/tests/built-changes.mjs +++ b/packages/contrail/tests/built-changes.mjs @@ -36,6 +36,14 @@ try { ); assert.equal(retry.status, 0, retry.stderr); assert.match(retry.stdout, /retry is now due/); + + const prune = spawnSync( + process.execPath, + [cli, "changes", "prune", "--root", root, "--sqlite", database], + { cwd: resolve("."), encoding: "utf8" }, + ); + assert.equal(prune.status, 0, prune.stderr); + assert.match(prune.stdout, /pruned=0/); console.log("built change consumer CLI passed"); } finally { rmSync(root, { recursive: true, force: true }); diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts new file mode 100644 index 0000000..9571bfe --- /dev/null +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -0,0 +1,354 @@ +import { describe, expect, it } from "vitest"; +import { + acknowledgeChanges, + acknowledgeCurrentSnapshotPage, + claimChanges, + claimCurrentActivation, + claimCurrentBootstrapChanges, + claimCurrentSnapshotPage, + completeCurrentActivation, + createIngestEvent, + failCurrentSnapshotPage, + getChangesStatus, + getCurrentBootstrapStatus, + getRequiredChangeConsumerReadiness, + hydrateChanges, + ingestRecords, + initSchema, + pruneChanges, + resolveConfig, + retryChangeConsumer, + skipChangeConsumer, + type ChangeConsumerConfig, + type Database, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const EVENT = "com.example.event"; +const NOTE = "com.example.note"; +const logger = { log() {}, warn() {}, error() {} }; + +function config(consumers: Record) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { + event: { collection: EVENT }, + note: { collection: NOTE }, + }, + changes: { consumers }, + }); +} + +function event(options: { + rkey: string; + time: number; + operation?: "create" | "update" | "delete"; + collection?: string; + name?: string; +}) { + const collection = options.collection ?? EVENT; + const operation = options.operation ?? "update"; + const did = "did:plc:alice"; + return createIngestEvent({ + uri: `at://${did}/${collection}/${options.rkey}`, + did, + collection, + rkey: options.rkey, + operation, + cid: operation === "delete" ? null : `cid-${options.rkey}-${options.time}`, + value: + operation === "delete" + ? undefined + : { name: options.name ?? `${options.rkey}-${options.time}` }, + timeUs: options.time, + indexedAt: options.time + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: options.time, + revision: String(options.time), + cursor: String(options.time), + }, + }); +} + +async function apply(db: Database, resolved: ReturnType, ...events: ReturnType[]) { + await ingestRecords(db, events, resolved, { phase: "live" }); +} + +const keeper: ChangeConsumerConfig = { + collections: [EVENT, NOTE], + initial: "history", +}; + +describe("current-state change consumer bootstrap", () => { + it("adds a current consumer over existing coverage and converges snapshot plus racing tail", async () => { + const db = createSqliteDatabase(":memory:"); + const original = config({ keeper }); + await initSchema(db, original); + await apply( + db, + original, + event({ rkey: "a", time: 1, name: "a-one" }), + event({ rkey: "b", time: 2, name: "b-one" }), + ); + + const withSearch = config({ + keeper, + search: { + collections: [EVENT], + initial: "current", + requiredForActivation: true, + }, + }); + await initSchema(db, withSearch); + let status = await getCurrentBootstrapStatus(db, "search"); + expect(status).toMatchObject({ state: "pending", anchor: "1", position: "1" }); + expect(status.token).toMatch(/^[0-9a-f-]{36}$/); + expect(await getRequiredChangeConsumerReadiness(db, "1")).toMatchObject({ + ready: false, + pending: [expect.objectContaining({ id: "search", state: "pending" })], + }); + + const first = await claimCurrentSnapshotPage(db, withSearch, "search", { + pageSize: 1, + leaseMs: 10, + now: 100, + }); + expect(first?.records[0]).toMatchObject({ + rkey: "a", + value: { name: "a-one" }, + }); + + // Destination success followed by an ack crash replays the same stable page. + const replay = await claimCurrentSnapshotPage(db, withSearch, "search", { + pageSize: 1, + leaseMs: 10, + now: 111, + }); + expect(replay?.pageId).toBe(first?.pageId); + expect(replay?.bootstrapToken).toBe(first?.bootstrapToken); + + // Mutations after anchor M race the scan and must be corrected by the tail. + await apply( + db, + withSearch, + event({ rkey: "a", time: 3, name: "a-two" }), + event({ rkey: "b", time: 4, operation: "delete" }), + event({ rkey: "c", time: 5, name: "c-one" }), + ); + await acknowledgeCurrentSnapshotPage(db, replay!, { now: 112 }); + + let clock = 120; + for (;;) { + const page = await claimCurrentSnapshotPage(db, withSearch, "search", { + pageSize: 1, + now: clock++, + }); + if (!page) break; + await acknowledgeCurrentSnapshotPage(db, page, { now: clock++ }); + } + status = await getCurrentBootstrapStatus(db, "search"); + expect(status).toMatchObject({ + state: "catching-up", + anchor: "1", + target: "2", + position: "1", + }); + + const tail = await claimCurrentBootstrapChanges(db, "search", { + now: 200, + }); + expect(tail).toMatchObject({ + from: "1", + through: "2", + bootstrapTarget: "2", + bootstrapToken: status.token, + }); + const delivery = await hydrateChanges(db, withSearch, tail!); + expect(delivery.currentRecords.map((record) => record.rkey).sort()).toEqual([ + "a", + "c", + ]); + expect(delivery.absentUris).toEqual([ + `at://did:plc:alice/${EVENT}/b`, + ]); + await acknowledgeChanges(db, tail!, { now: 201 }); + expect((await getCurrentBootstrapStatus(db, "search")).state).toBe( + "activating", + ); + + const activation = await claimCurrentActivation(db, "search", { + now: 300, + leaseMs: 10, + }); + const repeatedActivation = await claimCurrentActivation(db, "search", { + now: 311, + leaseMs: 10, + }); + expect(repeatedActivation?.bootstrapToken).toBe( + activation?.bootstrapToken, + ); + await completeCurrentActivation(db, repeatedActivation!, { now: 312 }); + expect(await getCurrentBootstrapStatus(db, "search")).toMatchObject({ + state: "ready", + position: "2", + target: "2", + }); + expect(await getRequiredChangeConsumerReadiness(db, "2")).toMatchObject({ + ready: true, + pending: [], + }); + + await apply(db, withSearch, event({ rkey: "d", time: 6 })); + const ordinary = await claimChanges(db, "search", { now: 400 }); + expect(ordinary).toMatchObject({ from: "2", through: "3" }); + }); + + it("persists snapshot failure backoff and resumes the same page", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + keeper, + search: { collections: [EVENT], initial: "current" }, + }); + await initSchema(db, resolved); + await apply(db, resolved, event({ rkey: "a", time: 1 })); + const page = await claimCurrentSnapshotPage(db, resolved, "search", { + now: 100, + }); + await failCurrentSnapshotPage( + db, + page!, + { code: "destination_unavailable", nextAttemptAt: 200 }, + { now: 101 }, + ); + expect( + await claimCurrentSnapshotPage(db, resolved, "search", { now: 150 }), + ).toBeNull(); + await retryChangeConsumer(db, "search", { now: 150 }); + const retry = await claimCurrentSnapshotPage(db, resolved, "search", { + now: 150, + }); + expect(retry?.pageId).toBe(page?.pageId); + }); + + it("rejects additive consumers that require unlogged coverage", async () => { + const db = createSqliteDatabase(":memory:"); + const eventOnly = config({ + keeper: { collections: [EVENT], phases: ["live"], initial: "history" }, + }); + await initSchema(db, eventOnly); + const expanded = config({ + keeper: { collections: [EVENT], phases: ["live"], initial: "history" }, + notes: { collections: [NOTE], phases: ["live"], initial: "current" }, + }); + await expect(initSchema(db, expanded)).rejects.toThrow( + "expands collection/phase coverage", + ); + }); +}); + +describe("consumer-aware change pruning", () => { + it("requires explicit confirmation and audits an operator skip", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + blocked: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, resolved); + for (let position = 1; position <= 3; position++) { + await apply(db, resolved, event({ rkey: String(position), time: position })); + } + + await expect( + skipChangeConsumer(db, "blocked", { + through: "2", + reason: "destination was rebuilt out of band", + confirm: false, + now: 100, + }), + ).rejects.toThrow("confirm: true"); + await skipChangeConsumer(db, "blocked", { + through: "2", + reason: "destination was rebuilt out of band", + confirm: true, + now: 100, + }); + expect((await getChangesStatus(db)).consumers[0]).toMatchObject({ + position: "2", + lastErrorCode: expect.stringMatching(/^operator_skip:/), + }); + expect( + await db + .prepare( + `SELECT action, from_position, through_position, reason + FROM change_consumer_actions`, + ) + .first(), + ).toEqual({ + action: "skip", + from_position: 0, + through_position: 2, + reason: "destination was rebuilt out of band", + }); + }); + + it("prunes only through the slowest consumer in bounded resumable slices", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + fast: { collections: [EVENT], initial: "history" }, + slow: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, resolved); + for (let position = 1; position <= 5; position++) { + await apply(db, resolved, event({ rkey: String(position), time: position })); + } + + const fast = await claimChanges(db, "fast", { now: 100 }); + await acknowledgeChanges(db, fast!, { now: 101 }); + const slowFirst = await claimChanges(db, "slow", { + now: 100, + maxBatches: 2, + }); + await acknowledgeChanges(db, slowFirst!, { now: 101 }); + + let pruned = await pruneChanges(db, { maxBatches: 10 }); + expect(pruned).toEqual({ + pruned: 2, + retainedFloor: "2", + safeThrough: "2", + done: true, + }); + expect((await getChangesStatus(db)).rows).toBe(3); + + const slowRest = await claimChanges(db, "slow", { now: 102 }); + await acknowledgeChanges(db, slowRest!, { now: 103 }); + pruned = await pruneChanges(db, { maxBatches: 2 }); + expect(pruned).toMatchObject({ + pruned: 2, + retainedFloor: "4", + safeThrough: "5", + done: false, + }); + pruned = await pruneChanges(db, { maxBatches: 2 }); + expect(pruned).toMatchObject({ + pruned: 1, + retainedFloor: "5", + safeThrough: "5", + done: true, + }); + expect((await getChangesStatus(db)).rows).toBe(0); + + // A later history consumer starts at the retained floor, never a deleted row. + const withLate = config({ + fast: { collections: [EVENT], initial: "history" }, + slow: { collections: [EVENT], initial: "history" }, + late: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, withLate); + expect( + (await getChangesStatus(db)).consumers.find((item) => item.id === "late"), + ).toMatchObject({ position: "5", backlogBatches: 0 }); + }); +}); diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index ead13a1..00ae632 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -420,7 +420,7 @@ describe("transactional projection change log", () => { }, }); await expect(initSchema(initialized, changed)).rejects.toThrow( - "definitions differ", + "cannot be removed or modified", ); }); diff --git a/packages/contrail/tests/postgres-concurrent-init.test.ts b/packages/contrail/tests/postgres-concurrent-init.test.ts index dd6fdae..f6238de 100644 --- a/packages/contrail/tests/postgres-concurrent-init.test.ts +++ b/packages/contrail/tests/postgres-concurrent-init.test.ts @@ -71,7 +71,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage', 'change_consumer_actions'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); diff --git a/packages/contrail/tests/postgres-e2e.test.ts b/packages/contrail/tests/postgres-e2e.test.ts index 2e199b8..e71c6c2 100644 --- a/packages/contrail/tests/postgres-e2e.test.ts +++ b/packages/contrail/tests/postgres-e2e.test.ts @@ -13,7 +13,12 @@ import { createPostgresDatabase } from "../src/adapters/postgres"; import { initSchema } from "../src/index"; import { acknowledgeChanges, + acknowledgeCurrentSnapshotPage, claimChanges, + claimCurrentActivation, + claimCurrentBootstrapChanges, + claimCurrentSnapshotPage, + completeCurrentActivation, getChangesStatus, hydrateChanges, ingestRecords, @@ -75,6 +80,10 @@ const CHANGE_CONFIG = resolveConfig({ collections: ["community.lexicon.calendar.event"], initial: "history", }, + cache: { + collections: ["community.lexicon.calendar.event"], + initial: "current", + }, }, }, }); @@ -102,7 +111,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage', 'change_consumer_actions'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); @@ -618,6 +627,34 @@ if (!PG_URL) { ]), ); await acknowledgeChanges(db, analytics!, { now: 1_001 }); + + const snapshot = await claimCurrentSnapshotPage( + db, + CHANGE_CONFIG, + "cache", + { now: 2_000 }, + ); + expect(snapshot?.records).toHaveLength(1); + await acknowledgeCurrentSnapshotPage(db, snapshot!, { now: 2_001 }); + expect( + await claimCurrentSnapshotPage(db, CHANGE_CONFIG, "cache", { + now: 2_002, + }), + ).toBeNull(); + const tail = await claimCurrentBootstrapChanges(db, "cache", { + now: 2_003, + }); + expect(tail).toMatchObject({ from: "0", through: "1" }); + await acknowledgeChanges(db, tail!, { now: 2_004 }); + const activation = await claimCurrentActivation(db, "cache", { + now: 2_005, + }); + await completeCurrentActivation(db, activation!, { now: 2_006 }); + expect( + (await getChangesStatus(db)).consumers.find( + (consumer) => consumer.id === "cache", + ), + ).toMatchObject({ bootstrapState: "ready", position: "1" }); }); }); diff --git a/packages/contrail/tests/postgres.test.ts b/packages/contrail/tests/postgres.test.ts index 0b7b530..3e8bc8a 100644 --- a/packages/contrail/tests/postgres.test.ts +++ b/packages/contrail/tests/postgres.test.ts @@ -57,7 +57,7 @@ if (!PG_URL) { const tables = await pool.query( `SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' - OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage'))` + OR tablename IN ('_contrail_meta', '_contrail_projection_state', 'backfills', 'backfill_state', 'discovery', 'cursor', 'source_position', 'bootstrap_state', 'bootstrap_snapshot_progress', 'identities', 'record_versions', 'ingest_diagnostics', 'feed_items', 'feed_prune_cursor', 'feed_backfills', 'change_log_state', 'change_batches', 'change_consumers', 'change_log_coverage', 'change_consumer_actions'))` ); for (const { tablename } of tables.rows) { await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); -- 2.51.2 From 21d8ed5302107d796ecc860c32118ad362fabcb1 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:31:06 +0200 Subject: [PATCH 04/10] Add change delivery runtimes --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 28 + packages/contrail/src/contrail.ts | 25 + packages/contrail/src/core/delivery.ts | 503 ++++++++++++++++++ .../contrail/src/core/router/diagnostics.ts | 11 +- packages/contrail/src/core/router/index.ts | 6 + packages/contrail/src/index.ts | 1 + packages/contrail/src/worker/index.ts | 113 +++- packages/contrail/tests/delivery.test.ts | 333 ++++++++++++ packages/contrail/tests/worker.test.ts | 171 +++++- 10 files changed, 1178 insertions(+), 15 deletions(-) create mode 100644 packages/contrail/src/core/delivery.ts create mode 100644 packages/contrail/tests/delivery.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 27dd926..619eb38 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 09f1052..6eac71f 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -96,6 +96,34 @@ const config = { Static definitions contain no handlers, URLs, clients, credentials, or secrets. Contrail registers them with a random database-generation ID and collection/phase coverage ledger. A winning logical put/delete appends one compact URI/version reference in the same transaction as canonical and derived state plus the source checkpoint. Duplicate, stale, same-CID, absent-delete, rejected, and rolled-back mutations append nothing. Record bodies are hydrated from current state by the later delivery layer rather than copied into the log. +Cloudflare Workers bind handlers and runtime-only secrets separately from static policy: + +```ts +export default createWorker(config, { + deliveries: { + search: async (batch, { env, signal }) => { + await updateSearch(env.SEARCH_KEY, batch, signal); + }, + }, + changeBootstraps: { + search: { + snapshot: async (page, { env, signal }) => { + await writeCandidateIndex(env.SEARCH_KEY, page, signal); + }, + activate: async (activation, { env, signal }) => { + await activateCandidateIdempotently( + env.SEARCH_KEY, + activation.bootstrapToken, + signal, + ); + }, + }, + }, +}); +``` + +Scheduled execution runs ingestion, due historical retries, then bounded fair delivery rounds. One consumer failure is persisted with backoff and does not stop ingestion or another consumer. A successful notify request schedules a best-effort one-round wake through `ExecutionContext`; projection success never depends on it. Persistent deployments run `contrail.runPersistentDeliveries()` alongside `runPersistent()`. + Low-level delivery uses bounded leases and compare-and-swap acknowledgement: ```ts diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index 859d054..8cccb88 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -15,6 +15,12 @@ import { } from "./core/validation"; import { getIngestDiagnostics } from "./core/diagnostics"; import { ChangeConsumers } from "./core/changes"; +import { + runPersistentChangeDeliveries, + type CurrentBootstrapRuntimeHandlers, + type DeliveryHandlers, + type DeliveryRuntimeOptions, +} from "./core/delivery"; import { optimizeDatabase } from "./core/db/optimize"; import { assertServingSourceCompatibility, @@ -160,6 +166,25 @@ export class Contrail { await Promise.all(tasks); } + /** Run a persistent fair change-delivery supervisor. Run this alongside + * `runPersistent()`; destination failures never stop source ingestion. */ + async runPersistentDeliveries( + options: { + env: Env; + deliveries: DeliveryHandlers; + bootstraps?: CurrentBootstrapRuntimeHandlers; + runtime?: DeliveryRuntimeOptions & { idleMs?: number }; + }, + db?: Database, + ): Promise { + await runPersistentChangeDeliveries({ + changes: this.changes, + config: this.config, + db: this.getDb(db), + ...options, + }); + } + /** Run *only* the labeler ingestion cycle. Escape hatch for callers who * want to run record and label ingestion in separate processes / workers. * `ingest()` already covers the typical case. */ diff --git a/packages/contrail/src/core/delivery.ts b/packages/contrail/src/core/delivery.ts new file mode 100644 index 0000000..65a2283 --- /dev/null +++ b/packages/contrail/src/core/delivery.ts @@ -0,0 +1,503 @@ +import type { ContrailConfig, Database, Logger } from "./types"; +import type { + ChangeClaimOptions, + ChangeConsumers, + DeliveryBatch, +} from "./changes"; +import type { + CurrentActivationClaim, + CurrentSnapshotClaim, +} from "./change-bootstrap"; + +export interface DeliveryContext { + env: Env; + signal: AbortSignal; + attempt: number; +} + +export type DeliveryHandler = ( + batch: DeliveryBatch, + context: DeliveryContext, +) => Promise; + +export type SnapshotDeliveryPage = Omit; +export type ActivationDelivery = Omit; + +export interface CurrentBootstrapRuntimeHandler { + snapshot: ( + page: SnapshotDeliveryPage, + context: DeliveryContext, + ) => Promise; + activate: ( + activation: ActivationDelivery, + context: DeliveryContext, + ) => Promise; +} + +export type DeliveryHandlers = Record>; +export type CurrentBootstrapRuntimeHandlers = Record< + string, + CurrentBootstrapRuntimeHandler +>; + +export interface DeliveryRuntimeOptions { + maxRounds?: number; + maxDurationMs?: number; + claim?: ChangeClaimOptions; + baseRetryMs?: number; + maxRetryMs?: number; + jitter?: number; + signal?: AbortSignal; + logger?: Logger; + /** @internal Deterministic runtime seams. */ + clock?: () => number; + random?: () => number; +} + +export interface DeliverySliceResult { + steps: number; + delivered: number; + snapshotPages: number; + activations: number; + failures: number; + consumerErrors: Record; + deadlineReached: boolean; +} + +function boundedInteger( + value: number | undefined, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw new TypeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return result; +} + +export function validateDeliveryHandlers( + config: ContrailConfig, + deliveries: DeliveryHandlers, + bootstraps: CurrentBootstrapRuntimeHandlers = {}, +): void { + const consumers = config.changes?.consumers ?? {}; + for (const id of Object.keys(consumers)) { + if (typeof deliveries[id] !== "function") { + throw new Error(`Missing runtime delivery handler for change consumer ${id}`); + } + if ( + consumers[id]!.initial === "current" && + (!bootstraps[id] || + typeof bootstraps[id].snapshot !== "function" || + typeof bootstraps[id].activate !== "function") + ) { + throw new Error( + `Missing current-state bootstrap handlers for change consumer ${id}`, + ); + } + } + for (const id of Object.keys(deliveries)) { + if (!consumers[id]) { + throw new Error(`Runtime delivery handler ${id} has no static consumer definition`); + } + } + for (const id of Object.keys(bootstraps)) { + if (!consumers[id] || consumers[id]!.initial !== "current") { + throw new Error(`Runtime bootstrap handler ${id} has no current consumer definition`); + } + } +} + +function publicSnapshot(claim: CurrentSnapshotClaim): SnapshotDeliveryPage { + const { leaseOwner: _leaseOwner, ...page } = claim; + return page; +} + +function publicActivation(claim: CurrentActivationClaim): ActivationDelivery { + const { leaseOwner: _leaseOwner, ...activation } = claim; + return activation; +} + +async function withDeadline( + parent: AbortSignal | undefined, + deadline: number, + clock: () => number, + callback: (signal: AbortSignal) => Promise, +): Promise { + const controller = new AbortController(); + const abort = () => controller.abort(parent?.reason ?? new Error("Delivery cancelled")); + if (parent?.aborted) abort(); + else parent?.addEventListener("abort", abort, { once: true }); + const remaining = Math.max(0, deadline - clock()); + const timer = setTimeout( + () => controller.abort(new Error("Delivery deadline reached")), + remaining, + ); + try { + return await callback(controller.signal); + } finally { + clearTimeout(timer); + parent?.removeEventListener("abort", abort); + } +} + +function retryAt( + attempt: number, + now: number, + base: number, + maximum: number, + jitter: number, + random: () => number, +): number { + const exponential = Math.min(maximum, base * 2 ** Math.min(20, attempt - 1)); + const factor = 1 + (random() * 2 - 1) * jitter; + return now + Math.max(1, Math.round(exponential * factor)); +} + +interface RuntimeState { + changes: ChangeConsumers; + config: ContrailConfig; + db: Database; + env: Env; + deliveries: DeliveryHandlers; + bootstraps: CurrentBootstrapRuntimeHandlers; + claim: ChangeClaimOptions; + deadline: number; + baseRetryMs: number; + maxRetryMs: number; + jitter: number; + signal?: AbortSignal; + logger: Logger; + clock: () => number; + random: () => number; +} + +async function persistFailure( + state: RuntimeState, + consumerId: string, + claim: Parameters[0], + attempt: number, +): Promise { + const now = state.clock(); + try { + await state.changes.fail( + claim, + { + code: "handler_error", + nextAttemptAt: retryAt( + attempt, + now, + state.baseRetryMs, + state.maxRetryMs, + state.jitter, + state.random, + ), + }, + { now }, + state.db, + ); + } catch (error) { + state.logger.warn( + `[changes] consumer=${consumerId} could not persist failure: ${error}`, + ); + } +} + +async function runNormal( + state: RuntimeState, + consumerId: string, + bootstrap: boolean, +): Promise<"empty" | "delivered" | "failed"> { + const now = state.clock(); + const claimOptions = { ...state.claim, now }; + const claim = bootstrap + ? await state.changes.claimBootstrapChanges( + consumerId, + claimOptions, + state.db, + ) + : await state.changes.claim(consumerId, claimOptions, state.db); + if (!claim) return "empty"; + try { + const batch = await state.changes.hydrate(claim, state.db); + await withDeadline(state.signal, state.deadline, state.clock, (signal) => + state.deliveries[consumerId]!(batch, { + env: state.env, + signal, + attempt: claim.attempt, + }), + ); + await state.changes.ack( + claim, + { now: state.clock() }, + state.db, + ); + return "delivered"; + } catch (error) { + state.logger.warn(`[changes] consumer=${consumerId} delivery failed: ${error}`); + await persistFailure( + state as RuntimeState, + consumerId, + claim, + claim.attempt, + ); + return "failed"; + } +} + +async function runCurrent( + state: RuntimeState, + consumerId: string, +): Promise<"empty" | "delivered" | "snapshot" | "activation" | "failed" | "progressed"> { + const before = await state.changes.bootstrapStatus(consumerId, state.db); + if (before.state === "ready") { + return runNormal(state, consumerId, false); + } + if (before.state === "pending" || before.state === "scanning") { + const claim = await state.changes.claimSnapshotPage( + consumerId, + { ...state.claim, now: state.clock() }, + state.db, + ); + if (!claim) { + const after = await state.changes.bootstrapStatus(consumerId, state.db); + return after.state !== before.state ? "progressed" : "empty"; + } + try { + await withDeadline(state.signal, state.deadline, state.clock, (signal) => + state.bootstraps[consumerId]!.snapshot(publicSnapshot(claim), { + env: state.env, + signal, + attempt: claim.attempt, + }), + ); + await state.changes.ackSnapshotPage( + claim, + { now: state.clock() }, + state.db, + ); + return "snapshot"; + } catch (error) { + state.logger.warn(`[changes] consumer=${consumerId} snapshot failed: ${error}`); + try { + await state.changes.failSnapshotPage( + claim, + { + code: "snapshot_handler_error", + nextAttemptAt: retryAt( + claim.attempt, + state.clock(), + state.baseRetryMs, + state.maxRetryMs, + state.jitter, + state.random, + ), + }, + { now: state.clock() }, + state.db, + ); + } catch (failureError) { + state.logger.warn(`[changes] consumer=${consumerId} snapshot failure state lost: ${failureError}`); + } + return "failed"; + } + } + if (before.state === "catching-up") { + const result = await runNormal(state, consumerId, true); + if (result !== "empty") return result; + const after = await state.changes.bootstrapStatus(consumerId, state.db); + return after.state !== before.state ? "progressed" : "empty"; + } + if (before.state === "activating") { + const claim = await state.changes.claimActivation( + consumerId, + { now: state.clock() }, + state.db, + ); + if (!claim) return "empty"; + try { + await withDeadline(state.signal, state.deadline, state.clock, (signal) => + state.bootstraps[consumerId]!.activate(publicActivation(claim), { + env: state.env, + signal, + attempt: claim.attempt, + }), + ); + await state.changes.completeActivation( + claim, + { now: state.clock() }, + state.db, + ); + return "activation"; + } catch (error) { + state.logger.warn(`[changes] consumer=${consumerId} activation failed: ${error}`); + try { + await state.changes.failActivation( + claim, + { + code: "activation_handler_error", + nextAttemptAt: retryAt( + claim.attempt, + state.clock(), + state.baseRetryMs, + state.maxRetryMs, + state.jitter, + state.random, + ), + }, + { now: state.clock() }, + state.db, + ); + } catch (failureError) { + state.logger.warn(`[changes] consumer=${consumerId} activation failure state lost: ${failureError}`); + } + return "failed"; + } + } + return "empty"; +} + +/** Run fair round-robin delivery work without coupling failures to ingestion. */ +export async function runChangeDeliverySlice(options: { + changes: ChangeConsumers; + config: ContrailConfig; + db: Database; + env: Env; + deliveries: DeliveryHandlers; + bootstraps?: CurrentBootstrapRuntimeHandlers; + runtime?: DeliveryRuntimeOptions; +}): Promise { + const bootstraps = options.bootstraps ?? {}; + validateDeliveryHandlers(options.config, options.deliveries, bootstraps); + const runtime = options.runtime ?? {}; + const clock = runtime.clock ?? Date.now; + const maxRounds = boundedInteger(runtime.maxRounds, 4, 1, 100, "maxRounds"); + const maxDurationMs = boundedInteger( + runtime.maxDurationMs, + 15_000, + 1, + 10 * 60_000, + "maxDurationMs", + ); + const baseRetryMs = boundedInteger( + runtime.baseRetryMs, + 1_000, + 1, + 60 * 60_000, + "baseRetryMs", + ); + const maxRetryMs = boundedInteger( + runtime.maxRetryMs, + 60 * 60_000, + baseRetryMs, + 48 * 60 * 60_000, + "maxRetryMs", + ); + const jitter = runtime.jitter ?? 0.2; + if (!Number.isFinite(jitter) || jitter < 0 || jitter > 1) { + throw new TypeError("jitter must be between 0 and 1"); + } + const deadline = clock() + maxDurationMs; + const state: RuntimeState = { + changes: options.changes, + config: options.config, + db: options.db, + env: options.env, + deliveries: options.deliveries, + bootstraps, + claim: runtime.claim ?? {}, + deadline, + baseRetryMs, + maxRetryMs, + jitter, + signal: runtime.signal, + logger: runtime.logger ?? options.config.logger ?? console, + clock, + random: runtime.random ?? Math.random, + }; + const result: DeliverySliceResult = { + steps: 0, + delivered: 0, + snapshotPages: 0, + activations: 0, + failures: 0, + consumerErrors: {}, + deadlineReached: false, + }; + const consumers = Object.entries(options.config.changes?.consumers ?? {}).sort( + ([left], [right]) => left.localeCompare(right), + ); + + for (let round = 0; round < maxRounds; round++) { + let progressed = false; + for (const [consumerId, consumer] of consumers) { + if (runtime.signal?.aborted || clock() >= deadline) { + result.deadlineReached = true; + return result; + } + try { + const outcome = consumer.initial === "current" + ? await runCurrent(state, consumerId) + : await runNormal(state, consumerId, false); + if (outcome !== "empty") { + progressed = true; + result.steps++; + } + if (outcome === "delivered") result.delivered++; + else if (outcome === "snapshot") result.snapshotPages++; + else if (outcome === "activation") result.activations++; + else if (outcome === "failed") result.failures++; + } catch (error) { + result.failures++; + result.consumerErrors[consumerId] = + error instanceof Error ? error.message : String(error); + state.logger.error(`[changes] consumer=${consumerId} runtime failed: ${error}`); + } + } + if (!progressed) break; + } + result.deadlineReached = clock() >= deadline; + return result; +} + +function waitFor(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(done, ms); + function done() { + clearTimeout(timer); + signal?.removeEventListener("abort", done); + resolve(); + } + signal?.addEventListener("abort", done, { once: true }); + }); +} + +/** Persistent fair supervisor. Source ingestion remains a separate task and is + * never cancelled by a destination outage. */ +export async function runPersistentChangeDeliveries(options: { + changes: ChangeConsumers; + config: ContrailConfig; + db: Database; + env: Env; + deliveries: DeliveryHandlers; + bootstraps?: CurrentBootstrapRuntimeHandlers; + runtime?: DeliveryRuntimeOptions & { idleMs?: number }; +}): Promise { + const idleMs = boundedInteger( + options.runtime?.idleMs, + 1_000, + 1, + 60_000, + "idleMs", + ); + while (!options.runtime?.signal?.aborted) { + const result = await runChangeDeliverySlice(options); + if (result.steps === 0) { + await waitFor(idleMs, options.runtime?.signal); + } + } +} diff --git a/packages/contrail/src/core/router/diagnostics.ts b/packages/contrail/src/core/router/diagnostics.ts index 5a7927d..a080011 100644 --- a/packages/contrail/src/core/router/diagnostics.ts +++ b/packages/contrail/src/core/router/diagnostics.ts @@ -3,6 +3,7 @@ import type { ContrailConfig, Database } from "../types"; import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; import { getLastCursor, getServingSourcePosition } from "../db"; import { getBackfillStatus } from "../status"; +import { getRequiredChangeConsumerReadiness } from "../changes"; export interface CursorStatus { cursor: number | null; @@ -48,9 +49,12 @@ export async function getStatusOverview(db: Database, config: ContrailConfig) { } } - const [ingestion, backfill] = await Promise.all([ + const [ingestion, backfill, requiredDelivery] = await Promise.all([ getCursorStatus(db), getBackfillStatus(db, config), + config.changes && Object.keys(config.changes.consumers).length > 0 + ? getRequiredChangeConsumerReadiness(db) + : Promise.resolve({ ready: true, through: "0", pending: [] }), ]); return { @@ -59,6 +63,11 @@ export async function getStatusOverview(db: Database, config: ContrailConfig) { collections, ingestion, backfill, + delivery: { + required: requiredDelivery.ready ? "ready" as const : "catching_up" as const, + pending: requiredDelivery.pending.length, + through: requiredDelivery.through, + }, }; } diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index a4e9720..8f66072 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -53,6 +53,9 @@ export function createApp( const overview = await getStatusOverview(db, config); if (!options.publicService) return c.json(overview); c.header("cache-control", "public, max-age=15, stale-while-revalidate=45"); + const hasRequiredDelivery = Object.values( + config.changes?.consumers ?? {}, + ).some((consumer) => consumer.requiredForActivation === true); return c.json({ status: overview.status, serving: "ready", @@ -63,6 +66,9 @@ export function createApp( seconds_ago: overview.ingestion.seconds_ago, }, backfill: overview.backfill, + ...(hasRequiredDelivery + ? { required_delivery: overview.delivery.required } + : {}), }); }); app.get("/health", (c) => c.json({ status: "ok" })); diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 36b6e5d..dfd17b6 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -37,6 +37,7 @@ export { export type { ChangeLogState, RecordChange } from "./core/change-log"; export * from "./core/changes"; export * from "./core/change-bootstrap"; +export * from "./core/delivery"; export * from "./core/validation"; export * from "./core/search"; export * from "./core/constellation"; diff --git a/packages/contrail/src/worker/index.ts b/packages/contrail/src/worker/index.ts index a75f0a0..d38ba04 100644 --- a/packages/contrail/src/worker/index.ts +++ b/packages/contrail/src/worker/index.ts @@ -18,6 +18,13 @@ import { Contrail } from "../contrail.js"; import { createHandler } from "../server.js"; import type { ContrailConfig, Database } from "../core/types.js"; import type { BackfillRetryOptions } from "../core/backfill.js"; +import { + runChangeDeliverySlice, + validateDeliveryHandlers, + type CurrentBootstrapRuntimeHandlers, + type DeliveryHandlers, + type DeliveryRuntimeOptions, +} from "../core/delivery.js"; import { normalizePublicServiceEndpoint, validatePublicServiceAuthEndpoint, @@ -25,7 +32,9 @@ import { type PublicServiceOptions, } from "../public-service.js"; -export interface CreateWorkerOptions { +type WorkerEnv = Record; + +export interface CreateWorkerOptions { /** D1 binding name in wrangler env. Default: `"DB"`. */ binding?: string; /** Exact generated/pinned bundle exposed for type generation and used by @@ -36,18 +45,33 @@ export interface CreateWorkerOptions { /** Bounded pending-account retry slice after each scheduled ingest. Enabled * by default; pass `false` to disable or options to tune its budget. */ backfillRetries?: BackfillRetryOptions | false; + /** Runtime delivery handlers, kept separate from static consumer policy. */ + deliveries?: DeliveryHandlers; + /** Snapshot and activation handlers for `initial: "current"` consumers. */ + changeBootstraps?: CurrentBootstrapRuntimeHandlers; + /** Bounded scheduled delivery policy. Set false only when another runtime + * owns all configured consumers. */ + delivery?: DeliveryRuntimeOptions | false; /** Runs once per isolate, after schema init, before handling the first * request. Use for app-specific setup that needs a live DB handle. */ - onInit?: (env: Record, db: Database) => void | Promise; + onInit?: (env: Env, db: Database) => void | Promise; } -type WorkerEnv = Record; - -export function createWorker( +export function createWorker( config: ContrailConfig, - options: CreateWorkerOptions = {} + options: CreateWorkerOptions = {} ) { const binding = options.binding ?? "DB"; + const deliveryEnabled = + options.delivery !== false && + Object.keys(config.changes?.consumers ?? {}).length > 0; + if (deliveryEnabled) { + validateDeliveryHandlers( + config, + options.deliveries ?? {}, + options.changeBootstraps ?? {}, + ); + } if (options.publicService) { normalizePublicServiceEndpoint(options.publicService.endpoint); validatePublicServiceLexicons(config, options.lexicons ?? []); @@ -60,7 +84,7 @@ export function createWorker( }); let ready = false; - const ensureReady = async (env: WorkerEnv, db: Database): Promise => { + const ensureReady = async (env: Env, db: Database): Promise => { if (ready) return; await contrail.init(db); await options.onInit?.(env, db); @@ -68,14 +92,52 @@ export function createWorker( }; return { - async fetch(request: Request, env: WorkerEnv): Promise { + async fetch( + request: Request, + env: Env, + ctx?: ExecutionContext, + ): Promise { const db = env[binding] as Database; await ensureReady(env, db); - return (await handle(request, db)) as Response; + const response = (await handle(request, db)) as Response; + const notifyPath = `/xrpc/${contrail.config.namespace}.notifyOfUpdate`; + if ( + deliveryEnabled && + ctx && + response.ok && + request.method === "POST" && + new URL(request.url).pathname === notifyPath + ) { + ctx.waitUntil( + runChangeDeliverySlice({ + changes: contrail.changes, + config: contrail.config, + db, + env, + deliveries: options.deliveries!, + bootstraps: options.changeBootstraps, + runtime: { + ...(options.delivery || {}), + maxRounds: 1, + maxDurationMs: Math.min( + options.delivery && options.delivery.maxDurationMs + ? options.delivery.maxDurationMs + : 5_000, + 5_000, + ), + }, + }).catch((error) => { + contrail.config.logger?.error( + `[changes] immediate delivery wake failed: ${error}`, + ); + }), + ); + } + return response; }, async scheduled( _event: ScheduledEvent, - env: WorkerEnv, + env: Env, ctx: ExecutionContext ): Promise { const db = env[binding] as Database; @@ -84,9 +146,36 @@ export function createWorker( // failures. A database lease prevents overlap with a manual backfill. ctx.waitUntil( (async () => { - await contrail.ingest({}, db); + try { + await contrail.ingest({}, db); + } catch (error) { + contrail.config.logger?.error(`[ingest] scheduled cycle failed: ${error}`); + } if (options.backfillRetries !== false) { - await contrail.retryBackfill(options.backfillRetries, db); + try { + await contrail.retryBackfill(options.backfillRetries, db); + } catch (error) { + contrail.config.logger?.error( + `[backfill] scheduled retry slice failed: ${error}`, + ); + } + } + if (deliveryEnabled) { + try { + await runChangeDeliverySlice({ + changes: contrail.changes, + config: contrail.config, + db, + env, + deliveries: options.deliveries!, + bootstraps: options.changeBootstraps, + runtime: options.delivery || undefined, + }); + } catch (error) { + contrail.config.logger?.error( + `[changes] scheduled delivery slice failed: ${error}`, + ); + } } })() ); diff --git a/packages/contrail/tests/delivery.test.ts b/packages/contrail/tests/delivery.test.ts new file mode 100644 index 0000000..2835fe5 --- /dev/null +++ b/packages/contrail/tests/delivery.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; +import { + createIngestEvent, + getChangesStatus, + ingestRecords, + initSchema, + resolveConfig, + runChangeDeliverySlice, + runPersistentChangeDeliveries, + validateDeliveryHandlers, + type ChangeConsumerConfig, + type Database, + type DeliveryHandlers, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { Contrail } from "../src/contrail"; + +const EVENT = "com.example.event"; +const logger = { log() {}, warn() {}, error() {} }; + +function config(consumers: Record) { + return resolveConfig({ + namespace: "com.example", + profiles: [], + logger, + collections: { event: { collection: EVENT } }, + changes: { consumers }, + }); +} + +function event(rkey: string, time: number) { + const did = "did:plc:alice"; + return createIngestEvent({ + uri: `at://${did}/${EVENT}/${rkey}`, + did, + collection: EVENT, + rkey, + operation: "update", + cid: `cid-${rkey}-${time}`, + value: { name: rkey }, + timeUs: time, + indexedAt: time + 10_000, + source: { + id: "source", + epoch: "epoch", + time_us: time, + revision: String(time), + cursor: String(time), + }, + }); +} + +async function append(db: Database, resolved: ReturnType, rkey: string, time: number) { + await ingestRecords(db, [event(rkey, time)], resolved); +} + +describe("change delivery runtime", () => { + it("fails startup for missing, extra, or incomplete runtime handlers", () => { + const resolved = config({ + search: { collections: [EVENT], initial: "current" }, + }); + expect(() => validateDeliveryHandlers(resolved, {})).toThrow( + "Missing runtime delivery handler", + ); + expect(() => + validateDeliveryHandlers(resolved, { search: async () => {} }), + ).toThrow("Missing current-state bootstrap handlers"); + expect(() => + validateDeliveryHandlers( + resolved, + { search: async () => {}, extra: async () => {} }, + { + search: { + snapshot: async () => {}, + activate: async () => {}, + }, + }, + ), + ).toThrow("extra has no static consumer definition"); + }); + + it("runs one bounded claim per consumer per fair round", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + alpha: { collections: [EVENT], initial: "history" }, + beta: { collections: [EVENT], initial: "history" }, + gamma: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + await append(db, resolved, "two", 2); + + const order: string[] = []; + const handlers: DeliveryHandlers<{}> = Object.fromEntries( + ["alpha", "beta", "gamma"].map((id) => [ + id, + async () => { + order.push(id); + }, + ]), + ); + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: handlers, + runtime: { + maxRounds: 2, + claim: { maxBatches: 1 }, + jitter: 0, + }, + }); + expect(order).toEqual([ + "alpha", + "beta", + "gamma", + "alpha", + "beta", + "gamma", + ]); + expect(result).toMatchObject({ + delivered: 6, + failures: 0, + steps: 6, + }); + }); + + it("isolates one failing consumer and resumes it after persisted backoff", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + broken: { collections: [EVENT], initial: "history" }, + healthy: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + + let fail = true; + const handlers = { + broken: async () => { + if (fail) throw new Error("destination down"); + }, + healthy: async () => {}, + }; + let current = 1_000; + const first = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: handlers, + runtime: { + maxRounds: 1, + baseRetryMs: 100, + maxRetryMs: 100, + jitter: 0, + clock: () => current, + }, + }); + expect(first).toMatchObject({ delivered: 1, failures: 1 }); + let status = await getChangesStatus(db); + expect(status.consumers.find((item) => item.id === "broken")).toMatchObject({ + position: "0", + attempts: 1, + nextAttemptAt: 1_100, + }); + expect(status.consumers.find((item) => item.id === "healthy")).toMatchObject({ + position: "1", + }); + + fail = false; + current = 1_101; + const resumed = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: handlers, + runtime: { + maxRounds: 1, + jitter: 0, + clock: () => current, + }, + }); + expect(resumed.delivered).toBe(1); + status = await getChangesStatus(db); + expect(status.consumers.find((item) => item.id === "broken")).toMatchObject({ + position: "1", + attempts: 0, + }); + }); + + it("drives current snapshot, catch-up, and idempotent activation", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + search: { + collections: [EVENT], + initial: "current", + requiredForActivation: true, + }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + expect( + await ( + await contrail.app().fetch(new Request("http://localhost/status")) + ).json(), + ).toMatchObject({ + delivery: { required: "catching_up", pending: 1 }, + }); + + const calls: string[] = []; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: { secret: "runtime-only" }, + deliveries: { + search: async (batch, context) => { + calls.push(`tail:${batch.cursor.through}:${context.env.secret}`); + }, + }, + bootstraps: { + search: { + snapshot: async (page) => { + expect(page).not.toHaveProperty("leaseOwner"); + calls.push(`snapshot:${page.records.length}`); + }, + activate: async (activation) => { + expect(activation).not.toHaveProperty("leaseOwner"); + calls.push(`activate:${activation.target}`); + }, + }, + }, + runtime: { maxRounds: 6, jitter: 0 }, + }); + expect(calls).toEqual([ + "snapshot:1", + "tail:1:runtime-only", + "activate:1", + ]); + expect(result).toMatchObject({ + snapshotPages: 1, + delivered: 1, + activations: 1, + failures: 0, + }); + expect(await contrail.changes.bootstrapStatus("search")).toMatchObject({ + state: "ready", + position: "1", + }); + expect( + await ( + await contrail.app().fetch(new Request("http://localhost/status")) + ).json(), + ).toMatchObject({ + delivery: { required: "ready", pending: 0 }, + }); + }); + + it("aborts destination work at the runtime deadline", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + webhook: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + let aborted = false; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { + webhook: async (_batch, { signal }) => { + await new Promise((resolve) => { + if (signal.aborted) { + aborted = true; + resolve(); + return; + } + signal.addEventListener( + "abort", + () => { + aborted = true; + resolve(); + }, + { once: true }, + ); + }); + }, + }, + runtime: { maxRounds: 1, maxDurationMs: 5 }, + }); + expect(aborted).toBe(true); + expect(result.deadlineReached).toBe(true); + }); + + it("runs a persistent supervisor until cancellation", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + webhook: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + const controller = new AbortController(); + let deliveries = 0; + await runPersistentChangeDeliveries({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { + webhook: async () => { + deliveries++; + controller.abort(); + }, + }, + runtime: { + signal: controller.signal, + idleMs: 1, + maxRounds: 1, + }, + }); + expect(deliveries).toBe(1); + expect((await getChangesStatus(db)).consumers[0].position).toBe("1"); + }); +}); diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts index bd69244..49cedd6 100644 --- a/packages/contrail/tests/worker.test.ts +++ b/packages/contrail/tests/worker.test.ts @@ -2,7 +2,13 @@ import { describe, it, expect, vi } from "vitest"; import { createWorker } from "../src/worker"; import { Contrail } from "../src/contrail"; import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { saveCursor, type ContrailConfig } from "../src/index"; +import { + createIngestEvent, + ingestRecords, + resolveConfig, + saveCursor, + type ContrailConfig, +} from "../src/index"; const MINIMAL_CONFIG: ContrailConfig = { namespace: "com.example", @@ -71,6 +77,24 @@ describe("createWorker", () => { ).not.toThrow(); }); + it("rejects configured consumers without matching runtime handlers", () => { + const config: ContrailConfig = { + ...MINIMAL_CONFIG, + changes: { + consumers: { + webhook: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, + }; + expect(() => createWorker(config)).toThrow( + "Missing runtime delivery handler", + ); + expect(() => createWorker(config, { delivery: false })).not.toThrow(); + }); + it("returns an object with fetch + scheduled handlers", () => { const worker = createWorker(MINIMAL_CONFIG); expect(typeof worker.fetch).toBe("function"); @@ -497,6 +521,151 @@ describe("createWorker", () => { ).toThrow("public method requires a matching query Lexicon"); }); + it("scheduled handler isolates ingest failure before retry and delivery", async () => { + const order: string[] = []; + const ingest = vi + .spyOn(Contrail.prototype, "ingest") + .mockImplementation(async () => { + order.push("ingest"); + throw new Error("source unavailable"); + }); + const retry = vi + .spyOn(Contrail.prototype, "retryBackfill") + .mockImplementation(async () => { + order.push("retry"); + return { + attempted: 0, + completed: 0, + failed: 0, + records: 0, + skipped: false, + }; + }); + const config: ContrailConfig = { + namespace: "com.example", + profiles: [], + logger: { log() {}, warn() {}, error() {} }, + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + webhook: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, + }; + + try { + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(config, { + deliveries: { + webhook: async () => { + order.push("delivery"); + }, + }, + }); + const env = { DB: db }; + await worker.fetch(new Request("http://localhost/health"), env); + const resolved = resolveConfig(config); + await ingestRecords( + db, + [ + createIngestEvent({ + did: "did:plc:alice", + collection: "community.lexicon.calendar.event", + rkey: "one", + operation: "create", + cid: "cid-one", + value: { name: "one" }, + timeUs: 1, + }), + ], + resolved, + ); + const waitUntil = vi.fn(); + const ctx = { + waitUntil, + passThroughOnException: vi.fn(), + } as unknown as ExecutionContext; + + await worker.scheduled({} as ScheduledEvent, env, ctx); + await waitUntil.mock.calls[0][0]; + expect(order).toEqual(["ingest", "retry", "delivery"]); + } finally { + ingest.mockRestore(); + retry.mockRestore(); + } + }); + + it("schedules a best-effort delivery wake after successful notify", async () => { + const config: ContrailConfig = { + namespace: "com.example", + profiles: [], + notify: true, + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + webhook: { + collections: ["community.lexicon.calendar.event"], + initial: "history", + }, + }, + }, + }; + const delivered = vi.fn(async () => {}); + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(config, { + deliveries: { webhook: delivered }, + }); + const env = { DB: db }; + await worker.fetch(new Request("http://localhost/health"), env); + await ingestRecords( + db, + [ + createIngestEvent({ + did: "did:plc:alice", + collection: "community.lexicon.calendar.event", + rkey: "one", + operation: "create", + cid: "cid-one", + value: { name: "one" }, + timeUs: 1, + }), + ], + resolveConfig(config), + ); + + const handler = vi + .spyOn(Contrail.prototype, "handler") + .mockReturnValue(async () => Response.json({ ok: true })); + try { + const waitUntil = vi.fn(); + const ctx = { + waitUntil, + passThroughOnException: vi.fn(), + } as unknown as ExecutionContext; + const response = await worker.fetch( + new Request("http://localhost/xrpc/com.example.notifyOfUpdate", { + method: "POST", + body: JSON.stringify({ uri: "at://did:plc:alice/community.lexicon.calendar.event/one" }), + }), + env, + ctx, + ); + expect(response.status).toBe(200); + expect(waitUntil).toHaveBeenCalledTimes(1); + await waitUntil.mock.calls[0][0]; + expect(delivered).toHaveBeenCalledTimes(1); + } finally { + handler.mockRestore(); + } + }); + it("scheduled handler runs live ingest then a bounded backfill retry slice", async () => { const order: string[] = []; const ingest = vi -- 2.51.2 From 9820a04fc2272e5deb455007a8124145eb49200e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:37:51 +0200 Subject: [PATCH 05/10] Add atmo Meilisearch reference consumer --- .changeset/durable-projection-log.md | 2 +- apps/atmo-rsvp/README.md | 16 + apps/atmo-rsvp/src/contrail.config.ts | 17 + apps/atmo-rsvp/src/meilisearch.ts | 446 ++++++++++++++++++ apps/atmo-rsvp/src/search-worker.ts | 18 + apps/atmo-rsvp/tests/meilisearch.test.ts | 274 +++++++++++ packages/contrail/src/core/changes.ts | 5 + .../contrail/tests/change-bootstrap.test.ts | 1 + 8 files changed, 778 insertions(+), 1 deletion(-) create mode 100644 apps/atmo-rsvp/src/meilisearch.ts create mode 100644 apps/atmo-rsvp/src/search-worker.ts create mode 100644 apps/atmo-rsvp/tests/meilisearch.test.ts diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 619eb38..08d7e18 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/apps/atmo-rsvp/README.md b/apps/atmo-rsvp/README.md index fbc10e8..bdb8cf5 100644 --- a/apps/atmo-rsvp/README.md +++ b/apps/atmo-rsvp/README.md @@ -45,6 +45,22 @@ Event, RSVP, and feed reads can hydrate indexed actor profiles. Follows are an i The service has no user sessions and never signs or publishes records. Applications authenticate users and write through their PDSes. +## Meilisearch candidate generation + +`src/meilisearch.ts` is the application-owned reference consumer for the transactional Contrail change log. It normalizes event documents, excludes cancelled events, restores direct geo/FSQ coordinates (with an injection seam for geocode-cache/H3 lookup), performs idempotent upsert/delete batches, and acknowledges only after every Meilisearch task reports `succeeded`. + +`searchGenerationConfig` and `src/search-worker.ts` are intentionally separate from the active Worker configuration. Use them only with a fresh D1 generation and these runtime secrets: + +```text +MEILI_URL +MEILI_KEY +MEILI_EVENTS_INDEX # optional; defaults to atmo_events +``` + +Current-state bootstrap writes a token-scoped candidate index, catches up a fixed retained tail, then swaps it into the stable UID. A reserved generation marker makes activation idempotent even if the Worker dies after Meilisearch succeeds but before Contrail acknowledges activation. Search clients must filter `kind = event`, which excludes that control marker. The previous stable contents remain under the candidate UID for rollback handling. + +Do not point the candidate config at the active populated D1 database: first-time logging fails closed there by design. Build/import a fresh candidate database, drain and verify the required `search` consumer, then switch the Worker/D1/index tuple together. + ## Development Build the workspace package before running the app directly: diff --git a/apps/atmo-rsvp/src/contrail.config.ts b/apps/atmo-rsvp/src/contrail.config.ts index b907d21..37d894e 100644 --- a/apps/atmo-rsvp/src/contrail.config.ts +++ b/apps/atmo-rsvp/src/contrail.config.ts @@ -72,3 +72,20 @@ export const config: ContrailConfig = { }, }, }; + +/** Candidate-generation configuration for the Meilisearch reference consumer. + * The active Worker keeps `config` until a fresh D1 + candidate index are built + * and activated together. */ +export const searchGenerationConfig: ContrailConfig = { + ...config, + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "current", + requiredForActivation: true, + }, + }, + }, +}; diff --git a/apps/atmo-rsvp/src/meilisearch.ts b/apps/atmo-rsvp/src/meilisearch.ts new file mode 100644 index 0000000..d483926 --- /dev/null +++ b/apps/atmo-rsvp/src/meilisearch.ts @@ -0,0 +1,446 @@ +import type { + ActivationDelivery, + CurrentRecord, + DeliveryBatch, + DeliveryHandlers, + CurrentBootstrapRuntimeHandlers, + SnapshotDeliveryPage, +} from "@atmo-dev/contrail"; + +export const EVENT_COLLECTION = "community.lexicon.calendar.event"; +export const SEARCH_CONSUMER_ID = "search"; +const CONTROL_ID = "__contrail_generation__"; +const EVENT_KIND = "event"; +const CONTROL_KIND = "contrail-control"; + +export interface AtmoMeilisearchEnv { + MEILI_URL: string; + MEILI_KEY: string; + /** Stable serving index UID. Default: `atmo_events`. */ + MEILI_EVENTS_INDEX?: string; +} + +export interface GeoPoint { + lat: number; + lng: number; +} + +export interface AtmoEventDocument extends Record { + id: string; + kind: typeof EVENT_KIND; + uri: string; + did: string; + rkey: string; + cid: string | null; + name: string; + description?: string; + mode?: string; + status?: string; + startsAt?: string; + endsAt?: string; + createdAt?: string; + locations?: unknown[]; + _geo?: GeoPoint; +} + +export interface AtmoMeilisearchOptions { + fetch?: typeof fetch; + taskPollMs?: number; + maxTaskPolls?: number; + /** Application-owned geocode cache/H3 seam. Direct geo and FSQ coordinates + * are extracted before this fallback is called. */ + resolveGeo?: ( + record: CurrentRecord, + env: Env, + signal: AbortSignal, + ) => Promise; + /** Default excludes explicitly cancelled events. */ + isDiscoverable?: (record: CurrentRecord) => boolean; +} + +interface MeiliTaskResponse { + taskUid?: number; + uid?: number; + status?: string; + error?: { code?: string; message?: string }; +} + +function base64Url(value: string): string { + const bytes = new TextEncoder().encode(value); + let binary = ""; + for (let index = 0; index < bytes.length; index += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000)); + } + return btoa(binary) + .replaceAll("+", "-") + .replaceAll("/", "_") + .replace(/=+$/, ""); +} + +export function eventDocumentId(uri: string): string { + return base64Url(uri); +} + +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function string(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function coordinate(value: unknown, minimum: number, maximum: number): number | null { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) && parsed >= minimum && parsed <= maximum + ? parsed + : null; +} + +function directGeo(value: unknown): GeoPoint | null { + if (!object(value)) return null; + const type = string(value.$type); + if ( + type !== "community.lexicon.location.geo" && + type !== "community.lexicon.location.fsq" + ) { + return null; + } + const lat = coordinate(value.latitude, -90, 90); + const lng = coordinate(value.longitude, -180, 180); + return lat === null || lng === null ? null : { lat, lng }; +} + +export function eventGeo(value: unknown): GeoPoint | null { + if (!object(value) || !Array.isArray(value.locations)) return null; + for (const location of value.locations) { + const geo = directGeo(location); + if (geo) return geo; + } + return null; +} + +export function defaultEventDiscoverability(record: CurrentRecord): boolean { + if (!object(record.value) || typeof record.value.name !== "string") return false; + return record.value.status !== "community.lexicon.calendar.event#cancelled"; +} + +export function eventDocument( + record: CurrentRecord, + geo: GeoPoint | null = eventGeo(record.value), +): AtmoEventDocument { + if (!object(record.value) || typeof record.value.name !== "string") { + throw new Error(`Event ${record.uri} has no valid name`); + } + const value = record.value; + const name = value.name as string; + return { + id: eventDocumentId(record.uri), + kind: EVENT_KIND, + uri: record.uri, + did: record.did, + rkey: record.rkey, + cid: record.cid, + name, + ...(string(value.description) ? { description: string(value.description) } : {}), + ...(string(value.mode) ? { mode: string(value.mode) } : {}), + ...(string(value.status) ? { status: string(value.status) } : {}), + ...(string(value.startsAt) ? { startsAt: string(value.startsAt) } : {}), + ...(string(value.endsAt) ? { endsAt: string(value.endsAt) } : {}), + ...(string(value.createdAt) ? { createdAt: string(value.createdAt) } : {}), + ...(Array.isArray(value.locations) ? { locations: value.locations } : {}), + ...(geo ? { _geo: geo } : {}), + }; +} + +function endpoint(env: AtmoMeilisearchEnv): URL { + if (typeof env.MEILI_KEY !== "string" || env.MEILI_KEY.length === 0) { + throw new Error("MEILI_KEY is required"); + } + if (typeof env.MEILI_URL !== "string" || env.MEILI_URL.length === 0) { + throw new Error("MEILI_URL is required"); + } + const url = new URL(env.MEILI_URL); + if (url.username || url.password || url.search || url.hash) { + throw new Error("MEILI_URL cannot contain credentials, query, or fragment"); + } + return url; +} + +function stableIndex(env: AtmoMeilisearchEnv): string { + return env.MEILI_EVENTS_INDEX?.trim() || "atmo_events"; +} + +export function candidateIndex(stable: string, token: string): string { + const suffix = token.replace(/[^a-zA-Z0-9_-]/g, "_"); + if (!suffix || suffix.length > 128) throw new Error("Invalid bootstrap token"); + return `${stable}__candidate__${suffix}`; +} + +class MeiliClient { + private readonly opened = new Set(); + + constructor( + private readonly env: AtmoMeilisearchEnv, + private readonly request: typeof fetch, + private readonly pollMs: number, + private readonly maxPolls: number, + ) {} + + private async fetch( + path: string, + init: RequestInit & { signal: AbortSignal }, + ): Promise { + const url = new URL(path, endpoint(this.env)); + return this.request(url, { + ...init, + headers: { + authorization: `Bearer ${this.env.MEILI_KEY}`, + "content-type": "application/json", + ...init.headers, + }, + }); + } + + private async json(response: Response, label: string): Promise { + if (!response.ok) { + throw new Error(`${label} failed with HTTP ${response.status}`); + } + const text = await response.text(); + if (text.length > 64_000) throw new Error(`${label} response is too large`); + try { + return text ? JSON.parse(text) : {}; + } catch { + throw new Error(`${label} returned malformed JSON`); + } + } + + private async task(response: Response, signal: AbortSignal): Promise { + const accepted = (await this.json(response, "Meilisearch mutation")) as MeiliTaskResponse; + const uid = accepted.taskUid ?? accepted.uid; + if (!Number.isSafeInteger(uid)) { + throw new Error("Meilisearch mutation returned no task UID"); + } + for (let poll = 0; poll < this.maxPolls; poll++) { + if (signal.aborted) throw signal.reason ?? new Error("Meilisearch task cancelled"); + const task = (await this.json( + await this.fetch(`/tasks/${uid}`, { method: "GET", signal }), + "Meilisearch task", + )) as MeiliTaskResponse; + if (task.status === "succeeded") return; + if (task.status === "failed" || task.status === "canceled") { + throw new Error( + `Meilisearch task ${uid} ${task.status}: ${task.error?.code ?? "unknown"}`, + ); + } + await new Promise((resolve, reject) => { + const timer = setTimeout(done, this.pollMs); + const abort = () => { + clearTimeout(timer); + reject(signal.reason ?? new Error("Meilisearch task cancelled")); + }; + function done() { + signal.removeEventListener("abort", abort); + resolve(); + } + signal.addEventListener("abort", abort, { once: true }); + }); + } + throw new Error("Meilisearch task did not finish within its polling budget"); + } + + async ensureIndex(index: string, token: string | null, signal: AbortSignal): Promise { + const cacheKey = `${index}:${token ?? "stable"}`; + if (this.opened.has(cacheKey)) return; + const indexResponse = await this.fetch(`/indexes/${encodeURIComponent(index)}`, { + method: "GET", + signal, + }); + if (indexResponse.status === 404) { + await this.task( + await this.fetch("/indexes", { + method: "POST", + body: JSON.stringify({ uid: index, primaryKey: "id" }), + signal, + }), + signal, + ); + } else if (!indexResponse.ok) { + throw new Error(`Meilisearch index lookup failed with HTTP ${indexResponse.status}`); + } + await this.task( + await this.fetch(`/indexes/${encodeURIComponent(index)}/settings`, { + method: "PATCH", + body: JSON.stringify({ + filterableAttributes: ["kind", "did", "mode", "status", "startsAt", "endsAt"], + sortableAttributes: ["startsAt", "endsAt", "createdAt"], + searchableAttributes: ["name", "description", "locations"], + }), + signal, + }), + signal, + ); + if (token) { + await this.upsert( + index, + [{ id: CONTROL_ID, kind: CONTROL_KIND, generation: token }], + signal, + ); + } + this.opened.add(cacheKey); + } + + async upsert(index: string, documents: unknown[], signal: AbortSignal): Promise { + if (documents.length === 0) return; + await this.task( + await this.fetch( + `/indexes/${encodeURIComponent(index)}/documents?primaryKey=id`, + { method: "POST", body: JSON.stringify(documents), signal }, + ), + signal, + ); + } + + async delete(index: string, ids: string[], signal: AbortSignal): Promise { + if (ids.length === 0) return; + await this.task( + await this.fetch(`/indexes/${encodeURIComponent(index)}/documents/delete-batch`, { + method: "POST", + body: JSON.stringify(ids), + signal, + }), + signal, + ); + } + + async marker(index: string, signal: AbortSignal): Promise { + const response = await this.fetch( + `/indexes/${encodeURIComponent(index)}/documents/${encodeURIComponent(CONTROL_ID)}`, + { method: "GET", signal }, + ); + if (response.status === 404) return null; + const value = await this.json(response, "Meilisearch generation marker"); + return typeof value.generation === "string" ? value.generation : null; + } + + async activate(stable: string, candidate: string, token: string, signal: AbortSignal): Promise { + await this.ensureIndex(candidate, token, signal); + if ((await this.marker(stable, signal)) === token) return; + await this.ensureIndex(stable, "contrail-empty", signal); + if ((await this.marker(stable, signal)) === token) return; + if ((await this.marker(candidate, signal)) !== token) { + throw new Error("Meilisearch candidate has the wrong generation marker"); + } + await this.task( + await this.fetch("/swap-indexes", { + method: "POST", + body: JSON.stringify([{ indexes: [stable, candidate] }]), + signal, + }), + signal, + ); + if ((await this.marker(stable, signal)) !== token) { + throw new Error("Meilisearch activation did not expose the candidate generation"); + } + } +} + +async function documentsFor( + records: CurrentRecord[], + env: Env, + signal: AbortSignal, + options: AtmoMeilisearchOptions, +): Promise<{ documents: AtmoEventDocument[]; hidden: string[] }> { + const discoverable = options.isDiscoverable ?? defaultEventDiscoverability; + const documents: AtmoEventDocument[] = []; + const hidden: string[] = []; + for (const record of records) { + if (!discoverable(record)) { + hidden.push(eventDocumentId(record.uri)); + continue; + } + const geo = eventGeo(record.value) ?? + (options.resolveGeo ? await options.resolveGeo(record, env, signal) : null); + documents.push(eventDocument(record, geo)); + } + return { documents, hidden }; +} + +export function createAtmoMeilisearchRuntime( + options: AtmoMeilisearchOptions = {}, +): { + deliveries: DeliveryHandlers; + changeBootstraps: CurrentBootstrapRuntimeHandlers; +} { + const clients = new WeakMap(); + const client = (env: Env) => { + const key = env as object; + let value = clients.get(key); + if (!value) { + value = new MeiliClient( + env, + options.fetch ?? fetch, + options.taskPollMs ?? 100, + options.maxTaskPolls ?? 300, + ); + clients.set(key, value); + } + return value; + }; + + const deliver = async ( + batch: DeliveryBatch, + env: Env, + signal: AbortSignal, + ) => { + const meili = client(env); + const stable = stableIndex(env); + const index = batch.destinationToken + ? candidateIndex(stable, batch.destinationToken) + : stable; + await meili.ensureIndex(index, batch.destinationToken ?? null, signal); + const { documents, hidden } = await documentsFor( + batch.currentRecords, + env, + signal, + options, + ); + const absent = batch.absentUris.map(eventDocumentId); + await meili.upsert(index, documents, signal); + await meili.delete(index, [...new Set([...hidden, ...absent])], signal); + }; + + return { + deliveries: { + [SEARCH_CONSUMER_ID]: async (batch, { env, signal }) => { + await deliver(batch, env, signal); + }, + }, + changeBootstraps: { + [SEARCH_CONSUMER_ID]: { + snapshot: async (page: SnapshotDeliveryPage, { env, signal }) => { + const meili = client(env); + const stable = stableIndex(env); + const index = candidateIndex(stable, page.bootstrapToken); + await meili.ensureIndex(index, page.bootstrapToken, signal); + const { documents, hidden } = await documentsFor( + page.records, + env, + signal, + options, + ); + await meili.upsert(index, documents, signal); + await meili.delete(index, hidden, signal); + }, + activate: async (activation: ActivationDelivery, { env, signal }) => { + const stable = stableIndex(env); + await client(env).activate( + stable, + candidateIndex(stable, activation.bootstrapToken), + activation.bootstrapToken, + signal, + ); + }, + }, + }, + }; +} diff --git a/apps/atmo-rsvp/src/search-worker.ts b/apps/atmo-rsvp/src/search-worker.ts new file mode 100644 index 0000000..65bb714 --- /dev/null +++ b/apps/atmo-rsvp/src/search-worker.ts @@ -0,0 +1,18 @@ +/** Candidate-generation Worker entrypoint. Switch wrangler `main` to this file + * only while activating a fresh D1 + Meilisearch index generation. */ +import { createWorker } from "@atmo-dev/contrail/worker"; +import { lexicons } from "../lexicons/generated"; +import { searchGenerationConfig } from "./contrail.config"; +import { + createAtmoMeilisearchRuntime, + type AtmoMeilisearchEnv, +} from "./meilisearch"; + +type Env = AtmoMeilisearchEnv & Record; +const search = createAtmoMeilisearchRuntime(); + +export default createWorker(searchGenerationConfig, { + lexicons, + publicService: { endpoint: "https://api.atmo.rsvp" }, + ...search, +}); diff --git a/apps/atmo-rsvp/tests/meilisearch.test.ts b/apps/atmo-rsvp/tests/meilisearch.test.ts new file mode 100644 index 0000000..e4f8f0f --- /dev/null +++ b/apps/atmo-rsvp/tests/meilisearch.test.ts @@ -0,0 +1,274 @@ +import { describe, expect, it } from "vitest"; +import type { + ActivationDelivery, + CurrentRecord, + DeliveryBatch, + SnapshotDeliveryPage, +} from "@atmo-dev/contrail"; +import { + candidateIndex, + createAtmoMeilisearchRuntime, + defaultEventDiscoverability, + eventDocument, + eventDocumentId, + eventGeo, + type AtmoMeilisearchEnv, +} from "../src/meilisearch"; + +class FakeMeili { + indexes = new Map>(); + tasks = new Map(); + nextTask = 1; + failNext = false; + swaps = 0; + + task(apply?: () => void): Response { + const uid = this.nextTask++; + const status = this.failNext ? "failed" : "succeeded"; + this.failNext = false; + this.tasks.set(uid, status); + if (status === "succeeded") apply?.(); + return Response.json({ taskUid: uid }, { status: 202 }); + } + + fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = new URL(String(input)); + const method = init?.method ?? "GET"; + const parts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent); + + if (parts[0] === "tasks" && method === "GET") { + const status = this.tasks.get(Number(parts[1])); + return status + ? Response.json({ + uid: Number(parts[1]), + status, + ...(status === "failed" ? { error: { code: "fake_failure" } } : {}), + }) + : new Response(null, { status: 404 }); + } + if (url.pathname === "/indexes" && method === "POST") { + const body = JSON.parse(String(init?.body)); + return this.task(() => this.indexes.set(body.uid, new Map())); + } + if (url.pathname === "/swap-indexes" && method === "POST") { + const [{ indexes }] = JSON.parse(String(init?.body)); + return this.task(() => { + const first = this.indexes.get(indexes[0]) ?? new Map(); + const second = this.indexes.get(indexes[1]) ?? new Map(); + this.indexes.set(indexes[0], second); + this.indexes.set(indexes[1], first); + this.swaps++; + }); + } + if (parts[0] !== "indexes" || !parts[1]) { + return new Response(null, { status: 404 }); + } + const index = parts[1]; + if (parts.length === 2 && method === "GET") { + return this.indexes.has(index) + ? Response.json({ uid: index, primaryKey: "id" }) + : new Response(null, { status: 404 }); + } + if (parts[2] === "settings" && method === "PATCH") { + return this.task(); + } + if (parts[2] === "documents" && parts.length === 3 && method === "POST") { + const documents = JSON.parse(String(init?.body)); + return this.task(() => { + const target = this.indexes.get(index)!; + for (const document of documents) target.set(document.id, document); + }); + } + if ( + parts[2] === "documents" && + parts[3] === "delete-batch" && + method === "POST" + ) { + const ids = JSON.parse(String(init?.body)); + return this.task(() => { + const target = this.indexes.get(index)!; + for (const id of ids) target.delete(id); + }); + } + if (parts[2] === "documents" && parts[3] && method === "GET") { + const document = this.indexes.get(index)?.get(parts[3]); + return document + ? Response.json(document) + : new Response(null, { status: 404 }); + } + return new Response(null, { status: 404 }); + }; +} + +const env: AtmoMeilisearchEnv = { + MEILI_URL: "https://meili.example/", + MEILI_KEY: "secret", + MEILI_EVENTS_INDEX: "events", +}; + +function record(options: { + uri?: string; + name?: string; + status?: string; + locations?: unknown[]; +} = {}): CurrentRecord { + const uri = options.uri ?? "at://did:plc:alice/community.lexicon.calendar.event/one"; + return { + uri, + did: "did:plc:alice", + collection: "community.lexicon.calendar.event", + rkey: uri.split("/").at(-1)!, + cid: "cid-one", + value: { + name: options.name ?? "One", + createdAt: "2026-01-01T00:00:00Z", + startsAt: "2026-02-01T00:00:00Z", + ...(options.status ? { status: options.status } : {}), + ...(options.locations ? { locations: options.locations } : {}), + }, + timeUs: 1, + indexedAt: 2, + }; +} + +const context = { + env, + signal: new AbortController().signal, + attempt: 1, +}; + +describe("atmo Meilisearch reference consumer", () => { + it("normalizes stable document IDs, discoverability, and direct geo", () => { + const current = record({ + locations: [ + { + $type: "community.lexicon.location.geo", + latitude: "52.52", + longitude: "13.405", + }, + ], + }); + expect(eventDocumentId(current.uri)).toMatch(/^[a-zA-Z0-9_-]+$/); + expect(eventGeo(current.value)).toEqual({ lat: 52.52, lng: 13.405 }); + expect(eventDocument(current)).toMatchObject({ + id: eventDocumentId(current.uri), + kind: "event", + name: "One", + _geo: { lat: 52.52, lng: 13.405 }, + }); + expect( + defaultEventDiscoverability( + record({ + status: "community.lexicon.calendar.event#cancelled", + }), + ), + ).toBe(false); + }); + + it("writes snapshot/tail state, removes hidden and absent rows, and activates idempotently", async () => { + const fake = new FakeMeili(); + const runtime = createAtmoMeilisearchRuntime({ + fetch: fake.fetch as typeof fetch, + taskPollMs: 1, + maxTaskPolls: 3, + }); + const token = "generation-one"; + const snapshot: SnapshotDeliveryPage = { + kind: "snapshot", + consumerId: "search", + generation: "log-generation", + bootstrapToken: token, + collection: "community.lexicon.calendar.event", + fromUri: null, + throughUri: record().uri, + pageId: "page-one", + records: [record()], + attempt: 1, + leaseExpiresAt: Date.now() + 1_000, + }; + await runtime.changeBootstraps.search!.snapshot(snapshot, context); + const candidate = candidateIndex("events", token); + expect(fake.indexes.get(candidate)?.get(eventDocumentId(record().uri))).toMatchObject({ + name: "One", + }); + + const hidden = record({ + uri: "at://did:plc:alice/community.lexicon.calendar.event/hidden", + status: "community.lexicon.calendar.event#cancelled", + }); + fake.indexes.get(candidate)!.set(eventDocumentId(hidden.uri), { + id: eventDocumentId(hidden.uri), + kind: "event", + }); + const absentUri = "at://did:plc:alice/community.lexicon.calendar.event/absent"; + fake.indexes.get(candidate)!.set(eventDocumentId(absentUri), { + id: eventDocumentId(absentUri), + kind: "event", + }); + const batch: DeliveryBatch = { + consumerId: "search", + cursor: { generation: "log-generation", from: "0", through: "1" }, + changes: [], + currentRecords: [hidden], + absentUris: [absentUri], + destinationToken: token, + }; + await runtime.deliveries.search!(batch, context); + expect(fake.indexes.get(candidate)?.has(eventDocumentId(hidden.uri))).toBe(false); + expect(fake.indexes.get(candidate)?.has(eventDocumentId(absentUri))).toBe(false); + + const activation: ActivationDelivery = { + kind: "activation", + consumerId: "search", + generation: "log-generation", + bootstrapToken: token, + target: "1", + attempt: 1, + leaseExpiresAt: Date.now() + 1_000, + }; + await runtime.changeBootstraps.search!.activate(activation, context); + await runtime.changeBootstraps.search!.activate(activation, context); + expect(fake.swaps).toBe(1); + expect(fake.indexes.get("events")?.get("__contrail_generation__")).toMatchObject({ + generation: token, + }); + }); + + it("waits for task success and rejects a failed asynchronous task", async () => { + const fake = new FakeMeili(); + const runtime = createAtmoMeilisearchRuntime({ + fetch: fake.fetch as typeof fetch, + taskPollMs: 1, + maxTaskPolls: 3, + }); + const token = "generation-failure"; + const snapshot: SnapshotDeliveryPage = { + kind: "snapshot", + consumerId: "search", + generation: "log-generation", + bootstrapToken: token, + collection: "community.lexicon.calendar.event", + fromUri: null, + throughUri: record().uri, + pageId: "page-one", + records: [], + attempt: 1, + leaseExpiresAt: Date.now() + 1_000, + }; + await runtime.changeBootstraps.search!.snapshot(snapshot, context); + fake.failNext = true; + await expect( + runtime.deliveries.search!( + { + consumerId: "search", + cursor: { generation: "log-generation", from: "0", through: "1" }, + changes: [], + currentRecords: [record()], + absentUris: [], + destinationToken: token, + }, + context, + ), + ).rejects.toThrow("fake_failure"); + }); +}); diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 4875240..636684f 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -94,6 +94,8 @@ export interface DeliveryBatch { changes: RecordChange[]; currentRecords: CurrentRecord[]; absentUris: string[]; + /** Generation-scoped destination token during current bootstrap catch-up. */ + destinationToken?: string; } export interface ChangeFailure { @@ -736,6 +738,9 @@ export async function hydrateChanges( changes: claim.changes, currentRecords, absentUris, + ...(claim.bootstrapToken + ? { destinationToken: claim.bootstrapToken } + : {}), }; } diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index 9571bfe..435ede2 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -168,6 +168,7 @@ describe("current-state change consumer bootstrap", () => { bootstrapToken: status.token, }); const delivery = await hydrateChanges(db, withSearch, tail!); + expect(delivery.destinationToken).toBe(status.token); expect(delivery.currentRecords.map((record) => record.rkey).sort()).toEqual([ "a", "c", -- 2.51.2 From add067ae764f95d605335a1b6698e6b2b3d53d85 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:42:06 +0200 Subject: [PATCH 06/10] Stabilize change log operations and upgrades --- apps/atmo-rsvp/tests/contract.test.ts | 23 +++++++++- packages/contrail/README.md | 4 +- packages/contrail/src/cli/commands/changes.ts | 9 +++- packages/contrail/src/core/change-log.ts | 39 ++++++++++++++++ packages/contrail/src/index.ts | 7 ++- packages/contrail/tests/change-log.test.ts | 45 +++++++++++++++++++ 6 files changed, 123 insertions(+), 4 deletions(-) diff --git a/apps/atmo-rsvp/tests/contract.test.ts b/apps/atmo-rsvp/tests/contract.test.ts index bfea7e3..9f0030f 100644 --- a/apps/atmo-rsvp/tests/contract.test.ts +++ b/apps/atmo-rsvp/tests/contract.test.ts @@ -3,7 +3,11 @@ import { describePublicService } from "@atmo-dev/contrail"; import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createWorker } from "@atmo-dev/contrail/worker"; import { lexicons } from "../lexicons/generated"; -import { config } from "../src/contrail.config"; +import { config, searchGenerationConfig } from "../src/contrail.config"; +import { + createAtmoMeilisearchRuntime, + type AtmoMeilisearchEnv, +} from "../src/meilisearch"; const EXPECTED_METHODS = [ "rsvp.atmo.event.getRecord", @@ -71,6 +75,23 @@ describe("api.atmo.rsvp public contract", () => { publicService: { endpoint: "https://api.atmo.rsvp" }, }), ).not.toThrow(); + expect(searchGenerationConfig.changes?.consumers.search).toEqual({ + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "current", + requiredForActivation: true, + }); + const search = createAtmoMeilisearchRuntime(); + expect(() => + createWorker>( + searchGenerationConfig, + { + lexicons, + publicService: { endpoint: "https://api.atmo.rsvp" }, + ...search, + }, + ), + ).not.toThrow(); }); it("publishes its service DID and permits browser auth headers", async () => { diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 6eac71f..22ac887 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -151,7 +151,9 @@ Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and r `initial: "current"` uses a durable snapshot-plus-tail coordinator. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. -A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. +A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Status includes a conservative projection/head/batch/ack write plan. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. + +Backups retain the database's change-log generation ID, positions, leases, bootstrap token, and consumer progress. Restoring that backup as the same generation is resumable and may redeliver an in-flight lease after expiry. Do not clone it as a new deployment generation while reusing destination cursors: build a fresh projection database and run current-state bootstrap. Restoring projection tables without the matching change tables is an unrecoverable delivery gap and must enter reset/bootstrap rather than silently continuing numeric positions. ## Local development diff --git a/packages/contrail/src/cli/commands/changes.ts b/packages/contrail/src/cli/commands/changes.ts index 1dfd205..0d608d0 100644 --- a/packages/contrail/src/cli/commands/changes.ts +++ b/packages/contrail/src/cli/commands/changes.ts @@ -1,5 +1,6 @@ import type { CAC } from "cac"; import { Contrail } from "../../contrail.js"; +import { getChangeLogCostPlan } from "../../core/change-log.js"; import type { Database } from "../../core/types.js"; import { resolveAndLoadConfig, @@ -143,8 +144,9 @@ export function registerChanges(cli: CAC): void { } const status = await contrail.changes.status(db); + const cost = getChangeLogCostPlan(contrail.config); if (commandOptions.json) { - console.log(JSON.stringify(status, null, 2)); + console.log(JSON.stringify({ ...status, cost }, null, 2)); return; } if (!status.enabled || !status.state) { @@ -156,6 +158,11 @@ export function registerChanges(cli: CAC): void { `floor=${status.state.retainedFloor} rows=${status.rows} ` + `changes=${status.changes} bytes=${status.bytes}`, ); + console.log( + ` write plan: projection=${cost.projectionStateWrites} ` + + `head=${cost.changeHeadWrites} batch=${cost.changeBatchWrites} ` + + `ack=${cost.acknowledgementWrites}`, + ); for (const item of status.consumers) { const retry = item.nextAttemptAt === null diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index b50a551..7db6713 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -16,6 +16,45 @@ import { export const MAX_CHANGE_BATCH_CHANGES = 500; export const MAX_CHANGE_BATCH_BYTES = 512_000; +export interface ChangeLogCostPlan { + enabled: boolean; + consumers: number; + coveragePairs: number; + projectionStateWrites: number; + changeHeadWrites: number; + changeBatchWrites: number; + acknowledgementWrites: number; + /** Total expected rows written by one relevant projection transaction. */ + relevantProjectionWrites: number; +} + +/** Conservative write-amplification report for one bounded projection batch. */ +export function getChangeLogCostPlan( + config: ContrailConfig, + mutationUris = 50, +): ChangeLogCostPlan { + if (!Number.isSafeInteger(mutationUris) || mutationUris < 1 || mutationUris > 500) { + throw new TypeError("mutationUris must be an integer between 1 and 500"); + } + const enabled = changesEnabled(config); + // One serialized revision update plus one predecessor-check update per forty + // unique URIs. The idempotent singleton INSERT normally writes no row. + const projectionStateWrites = 1 + Math.ceil(mutationUris / 40); + const changeHeadWrites = enabled ? 1 : 0; + const changeBatchWrites = enabled ? 1 : 0; + return { + enabled, + consumers: Object.keys(config.changes?.consumers ?? {}).length, + coveragePairs: changeLogCoverage(config).length, + projectionStateWrites, + changeHeadWrites, + changeBatchWrites, + acknowledgementWrites: enabled ? 1 : 0, + relevantProjectionWrites: + projectionStateWrites + changeHeadWrites + changeBatchWrites, + }; +} + export interface RecordChange { id: string; kind: "record"; diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index dfd17b6..cd07755 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -30,11 +30,16 @@ export * from "./core/backfill"; export * from "./core/status"; export * from "./core/diagnostics"; export { + getChangeLogCostPlan, getChangeLogState, MAX_CHANGE_BATCH_BYTES, MAX_CHANGE_BATCH_CHANGES, } from "./core/change-log"; -export type { ChangeLogState, RecordChange } from "./core/change-log"; +export type { + ChangeLogCostPlan, + ChangeLogState, + RecordChange, +} from "./core/change-log"; export * from "./core/changes"; export * from "./core/change-bootstrap"; export * from "./core/delivery"; diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index 00ae632..bb5a3c3 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { Contrail, createIngestEvent, + getChangeLogCostPlan, getChangeLogState, ingestRecords, initSchema, @@ -146,6 +147,16 @@ describe("transactional projection change log", () => { it("has no change-log schema or writes when disabled", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config(); + expect(getChangeLogCostPlan(resolved, 50)).toEqual({ + enabled: false, + consumers: 0, + coveragePairs: 0, + projectionStateWrites: 3, + changeHeadWrites: 0, + changeBatchWrites: 0, + acknowledgementWrites: 0, + relevantProjectionWrites: 3, + }); await initSchema(db, resolved); const tables = await db @@ -169,6 +180,16 @@ describe("transactional projection change log", () => { it("initializes a fresh generation, registrations, and coverage ledger", async () => { const db = createSqliteDatabase(":memory:"); const resolved = loggedConfig(); + expect(getChangeLogCostPlan(resolved, 50)).toMatchObject({ + enabled: true, + consumers: 2, + coveragePairs: 2, + projectionStateWrites: 3, + changeHeadWrites: 1, + changeBatchWrites: 1, + acknowledgementWrites: 1, + relevantProjectionWrites: 5, + }); await initSchema(db, resolved); const state = await getChangeLogState(db); @@ -424,6 +445,30 @@ describe("transactional projection change log", () => { ); }); + it("upgrades retained pre-byte-count change batches without resetting consumers", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = loggedConfig(); + await initSchema(db, resolved); + await ingestRecords(db, [mutation({ sourceTime: 1 })], resolved); + await db + .prepare("ALTER TABLE change_batches DROP COLUMN encoded_bytes") + .run(); + await db + .prepare( + "UPDATE _contrail_meta SET value = 'old-change-schema' WHERE key = 'schema_fingerprint'", + ) + .run(); + + await initSchema(db, resolved); + const row = await db + .prepare("SELECT encoded_bytes, changes_json FROM change_batches") + .first<{ encoded_bytes: number; changes_json: string }>(); + expect(row?.encoded_bytes).toBe( + new TextEncoder().encode(row!.changes_json).byteLength, + ); + expect((await getChangeLogState(db))?.head).toBe("1"); + }); + it("retries a losing overlapping writer from fresh durable state", async () => { const real = createSqliteDatabase(":memory:"); const resolved = loggedConfig(); -- 2.51.2 From e70b0af5a16c083226888fe64a1d556c33575e41 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:56:30 +0200 Subject: [PATCH 07/10] Fix change bootstrap delivery invariants --- .changeset/durable-projection-log.md | 2 +- packages/contrail/README.md | 2 +- packages/contrail/src/core/change-log.ts | 27 ++++++++------- packages/contrail/src/core/changes.ts | 6 ++-- packages/contrail/src/core/types.ts | 21 +++++++++--- .../contrail/tests/change-bootstrap.test.ts | 5 ++- packages/contrail/tests/change-log.test.ts | 34 +++++++++++++++++++ 7 files changed, 74 insertions(+), 23 deletions(-) diff --git a/.changeset/durable-projection-log.md b/.changeset/durable-projection-log.md index 08d7e18..7318ebc 100644 --- a/.changeset/durable-projection-log.md +++ b/.changeset/durable-projection-log.md @@ -4,4 +4,4 @@ Add the first transactional projection change-log milestone. Optional static consumer definitions now create a fresh-generation log, durable registrations, and collection/phase coverage ledger. Winning logical URI changes append compact references atomically with canonical records, derived projections, tombstones, and source checkpoints; disabled configurations create no log tables or append writes. -Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. +Harden all projection writers with transaction-time predecessor guards and bounded conflict retries so overlapping cron, persistent, notify, and backfill work cannot commit stale canonical or derived state. Add independent bounded consumer leases, filtered/coalesced claims, set-oriented current-state hydration, CAS acknowledgement, failure backoff, lease renewal, private status, and manual retry APIs. Add crash-safe current-state snapshot/tail/activation bootstrap, safe additive consumers over existing coverage, required-consumer readiness gates, consumer-aware bounded pruning, and audited explicit skip operations. Current-state consumers require both projection phases, and candidate destination tokens are scoped strictly to bootstrap deliveries. Private `contrail changes` commands cover status, retry, prune, and skip. Add fair bounded Worker delivery after ingestion/retries, best-effort immediate notify wakes, runtime handler validation, deadline cancellation, isolated retry scheduling, and a persistent delivery supervisor. Include an app-owned atmo.rsvp Meilisearch reference consumer with task-success acknowledgement, hidden/delete convergence, candidate-index snapshot/tail bootstrap, and idempotent generation-marker activation. Enabling or expanding log coverage on a populated generation fails closed pending explicit quiet-boundary migration tooling. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 22ac887..947a24d 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -149,7 +149,7 @@ if (claim) { Claims coalesce repeated URIs, hydrate in set-oriented collection queries, and resolve delete/recreate races from newest canonical state. Consumers lease and progress independently; irrelevant position ranges advance without invoking a handler. Delivery is intentionally at least once—a destination success followed by an acknowledgement crash causes duplicate delivery. Handlers must be idempotent by stable record/document key. -`initial: "current"` uses a durable snapshot-plus-tail coordinator. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. +`initial: "current"` uses a durable snapshot-plus-tail coordinator and must observe both `historical` and `live` phases so every mutation racing the keyset scan is retained for reconciliation. Repeatedly claim and idempotently acknowledge `contrail.changes.claimSnapshotPage()`, then drain `claimBootstrapChanges()` through its fixed target using ordinary hydrate/ack. Snapshot and bootstrap-tail deliveries carry the candidate destination token; ordinary deliveries after activation do not. Finally claim the stable generation-scoped activation token with `claimActivation()`, perform an idempotent destination swap, and call `completeActivation()`. A crash replays the same URI page, tail range, or activation token. Records updated or deleted while the keyset scan races are corrected by the anchored tail. A consumer can be added to a populated log when all of its collection/phase pairs were already covered; its current/future anchor is the atomic current head, while history starts at the retained floor. Expanding coverage still fails closed without a fresh generation or explicit old-writer quiet boundary. `contrail changes status`, `retry`, `prune`, and explicitly confirmed `skip` expose private operations for SQLite or Wrangler D1 deployments. Status includes a conservative projection/head/batch/ack write plan. Pruning is bounded by the slowest durable consumer/bootstrap anchor. Skip records a bounded private audit reason and never occurs implicitly. Disabling or removing an existing log remains fail-closed. With no configured consumers, no change-log tables or append writes exist. diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 7db6713..1220a3a 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -363,7 +363,7 @@ export async function initializeChangeLog( const statements: Statement[] = []; for (const [consumerId, consumer] of Object.entries( config.changes?.consumers ?? {}, - ).sort(([left], [right]) => left.localeCompare(right))) { + ).sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))) { const initialReady = consumer.initial === "current" ? "pending" : "ready"; statements.push( db @@ -444,17 +444,17 @@ async function assertChangeLogDefinition( FROM change_consumers ORDER BY consumer_id`, ) .all(); - const expectedConsumers = Object.entries(config.changes?.consumers ?? {}).sort( - ([left], [right]) => left.localeCompare(right), - ); + const expectedConsumers = Object.entries(config.changes?.consumers ?? {}); if (consumers.results.length !== expectedConsumers.length) { throw new Error("Durable change consumer registration is incomplete"); } - for (let index = 0; index < expectedConsumers.length; index++) { - const [id, expected] = expectedConsumers[index]!; - const actual = consumers.results[index]!; + const consumersById = new Map( + consumers.results.map((consumer) => [consumer.consumer_id, consumer]), + ); + for (const [id, expected] of expectedConsumers) { + const actual = consumersById.get(id); if ( - actual.consumer_id !== id || + !actual || actual.generation_id !== state.generation_id || actual.configured_collections_json !== canonicalCollections(expected.collections) || actual.configured_phases_json !== canonicalPhases(changeConsumerPhases(expected)) || @@ -477,13 +477,14 @@ async function assertChangeLogDefinition( if (coverage.results.length !== expectedCoverage.length) { throw new Error("Durable change-log coverage is incomplete"); } - for (let index = 0; index < expectedCoverage.length; index++) { - const actual = coverage.results[index]!; - const expected = expectedCoverage[index]!; + const coverageByPair = new Map( + coverage.results.map((item) => [`${item.collection}\0${item.phase}`, item]), + ); + for (const expected of expectedCoverage) { + const actual = coverageByPair.get(`${expected.collection}\0${expected.phase}`); if ( + !actual || actual.generation_id !== state.generation_id || - actual.collection !== expected.collection || - actual.phase !== expected.phase || Number(actual.from_position) !== 0 || actual.through_position !== null ) { diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 636684f..3627147 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -631,9 +631,9 @@ async function claimChangeRange( changes: [...coalesced.values()], attempt: Number(leased.attempts) + 1, leaseExpiresAt, - ...(leased.bootstrap_token === null - ? {} - : { bootstrapToken: leased.bootstrap_token }), + ...(bootstrap && leased.bootstrap_token !== null + ? { bootstrapToken: leased.bootstrap_token } + : {}), ...(bootstrapTarget === null ? {} : { bootstrapTarget }), leaseOwner: owner, }; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index bb5860e..0544dde 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -258,7 +258,7 @@ export type ChangeConsumerInitialMode = "current" | "future" | "history"; export interface ChangeConsumerConfig { /** Exact configured collection NSIDs. Short aliases are deliberately rejected. */ collections: string[]; - /** Projection phases to observe. Defaults to both historical and live. */ + /** Projection phases to observe. Defaults to both; `initial: "current"` requires both. */ phases?: ProjectionPhase[]; /** How the consumer establishes its first durable position. */ initial: ChangeConsumerInitialMode; @@ -686,6 +686,11 @@ const MAX_CHANGE_CONSUMER_COLLECTIONS = 64; const MAX_CHANGE_COVERAGE_PAIRS = 256; const MAX_CHANGE_DEFINITIONS_BYTES = 64 * 1_024; +/** Locale-independent order for durable definitions compared across runtimes. */ +function compareCanonicalText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + /** Whether this configuration requires the optional transactional change log. */ export function changesEnabled(config: ContrailConfig): boolean { return Object.keys(config.changes?.consumers ?? {}).length > 0; @@ -712,8 +717,8 @@ export function changeLogCoverage( } return [...pairs.values()].sort( (left, right) => - left.collection.localeCompare(right.collection) || - left.phase.localeCompare(right.phase), + compareCanonicalText(left.collection, right.collection) || + compareCanonicalText(left.phase, right.phase), ); } @@ -721,7 +726,7 @@ export function changeLogCoverage( export function canonicalChangeDefinitions(config: ContrailConfig): string { return JSON.stringify( Object.entries(config.changes?.consumers ?? {}) - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareCanonicalText(left, right)) .map(([id, consumer]) => ({ id, collections: [...consumer.collections].sort(), @@ -831,6 +836,14 @@ export function validateConfig(config: ContrailConfig): void { `Change consumer "${id}" requires unique historical/live phases`, ); } + if ( + consumer.initial === "current" && + !(phases.includes("historical") && phases.includes("live")) + ) { + throw new Error( + `Current-state change consumer "${id}" must observe both historical and live phases`, + ); + } if (!(["current", "future", "history"] as string[]).includes(consumer.initial)) { throw new Error(`Change consumer "${id}" has an invalid initial mode`); } diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index 435ede2..8a23e1b 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -206,6 +206,9 @@ describe("current-state change consumer bootstrap", () => { await apply(db, withSearch, event({ rkey: "d", time: 6 })); const ordinary = await claimChanges(db, "search", { now: 400 }); expect(ordinary).toMatchObject({ from: "2", through: "3" }); + expect(ordinary).not.toHaveProperty("bootstrapToken"); + const ordinaryDelivery = await hydrateChanges(db, withSearch, ordinary!); + expect(ordinaryDelivery).not.toHaveProperty("destinationToken"); }); it("persists snapshot failure backoff and resumes the same page", async () => { @@ -243,7 +246,7 @@ describe("current-state change consumer bootstrap", () => { await initSchema(db, eventOnly); const expanded = config({ keeper: { collections: [EVENT], phases: ["live"], initial: "history" }, - notes: { collections: [NOTE], phases: ["live"], initial: "current" }, + notes: { collections: [NOTE], initial: "current" }, }); await expect(initSchema(db, expanded)).rejects.toThrow( "expands collection/phase coverage", diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index bb5a3c3..17f3303 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -142,6 +142,21 @@ describe("transactional projection change log", () => { }, }), ).toThrow("unique historical/live phases"); + expect( + () => + new Contrail({ + ...base, + changes: { + consumers: { + search: { + collections: [EVENT], + phases: ["live"], + initial: "current", + }, + }, + }, + }), + ).toThrow("must observe both historical and live phases"); }); it("has no change-log schema or writes when disabled", async () => { @@ -412,6 +427,25 @@ describe("transactional projection change log", () => { ); }); + it("matches durable consumer IDs independently of locale sort order", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + changes: { + consumers: { + a_: { collections: [EVENT], initial: "future" }, + "a-": { collections: [EVENT], initial: "future" }, + }, + }, + }); + + await initSchema(db, resolved); + await expect(initSchema(db, resolved)).resolves.toBeUndefined(); + const rows = await db + .prepare("SELECT consumer_id FROM change_consumers ORDER BY consumer_id") + .all<{ consumer_id: string }>(); + expect(rows.results.map((row) => row.consumer_id)).toEqual(["a-", "a_"]); + }); + it("fails closed for unsafe enable, disable, and definition changes", async () => { const populated = createSqliteDatabase(":memory:"); const disabled = config(); -- 2.51.2 From 95edda0ebc4c1087c0c744d69f480742cedf2f51 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:09:32 +0200 Subject: [PATCH 08/10] Scope test build dependency to Contrail --- packages/contrail/turbo.json | 9 +++++++++ turbo.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 packages/contrail/turbo.json diff --git a/packages/contrail/turbo.json b/packages/contrail/turbo.json new file mode 100644 index 0000000..8649e73 --- /dev/null +++ b/packages/contrail/turbo.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://turbo.build/schema.json", + "extends": ["//"], + "tasks": { + "test": { + "dependsOn": ["^build", "build"] + } + } +} diff --git a/turbo.json b/turbo.json index 2279a64..94b9292 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,7 @@ "dependsOn": ["^build"] }, "test": { - "dependsOn": ["^build", "build"], + "dependsOn": ["^build"], "outputs": [] }, "dev": { -- 2.51.2 From b707deff64d79ddd66bd85b1196450b70fb44b10 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:19:01 +0200 Subject: [PATCH 09/10] integrate feedback --- .../contrail/src/core/change-bootstrap.ts | 6 +- packages/contrail/src/core/changes.ts | 6 +- packages/contrail/src/core/delivery.ts | 2 +- .../contrail/tests/change-bootstrap.test.ts | 5 +- .../contrail/tests/change-consumers.test.ts | 10 ++- packages/contrail/tests/delivery.test.ts | 81 +++++++++++++++++++ 6 files changed, 100 insertions(+), 10 deletions(-) diff --git a/packages/contrail/src/core/change-bootstrap.ts b/packages/contrail/src/core/change-bootstrap.ts index 0cc6a97..2115bfc 100644 --- a/packages/contrail/src/core/change-bootstrap.ts +++ b/packages/contrail/src/core/change-bootstrap.ts @@ -443,6 +443,8 @@ async function failBootstrapLease( ) { throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); } + // Expiry permits reclamation; owner equality remains the CAS guard. Preserve + // failure/backoff when slow work expires but no replacement has claimed it. const failed = await db .prepare( `UPDATE change_consumers @@ -451,7 +453,6 @@ async function failBootstrapLease( last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_token = ? AND lease_owner = ? - AND lease_expires_at > ? RETURNING consumer_id`, ) .bind( @@ -463,12 +464,11 @@ async function failBootstrapLease( claim.generation, claim.bootstrapToken, claim.leaseOwner, - timestamp, ) .first<{ consumer_id: string }>(); if (!failed) { throw new ChangeLeaseLostError( - `Current bootstrap claim for ${claim.consumerId} is stale or expired`, + `Current bootstrap claim for ${claim.consumerId} is stale or no longer owned`, ); } } diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 3627147..0008639 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -854,6 +854,8 @@ export async function failChanges( ) { throw new TypeError("nextAttemptAt must be null or a future safe timestamp"); } + // Expiry permits reclamation; owner equality remains the CAS guard. Preserve + // failure/backoff when slow work expires but no replacement has claimed it. const failed = await db .prepare( `UPDATE change_consumers @@ -862,7 +864,6 @@ export async function failChanges( last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? - AND lease_expires_at > ? RETURNING attempts, next_attempt_at`, ) .bind( @@ -874,12 +875,11 @@ export async function failChanges( claim.generation, claim.from, claim.leaseOwner, - now, ) .first<{ attempts: number | string; next_attempt_at: number | string | null }>(); if (!failed) { throw new ChangeLeaseLostError( - `Change claim for ${claim.consumerId} is stale or expired`, + `Change claim for ${claim.consumerId} is stale or no longer owned`, ); } return { diff --git a/packages/contrail/src/core/delivery.ts b/packages/contrail/src/core/delivery.ts index 65a2283..4a0ba26 100644 --- a/packages/contrail/src/core/delivery.ts +++ b/packages/contrail/src/core/delivery.ts @@ -314,7 +314,7 @@ async function runCurrent( if (before.state === "activating") { const claim = await state.changes.claimActivation( consumerId, - { now: state.clock() }, + { leaseMs: state.claim.leaseMs, now: state.clock() }, state.db, ); if (!claim) return "empty"; diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index 8a23e1b..c4733aa 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -211,7 +211,7 @@ describe("current-state change consumer bootstrap", () => { expect(ordinaryDelivery).not.toHaveProperty("destinationToken"); }); - it("persists snapshot failure backoff and resumes the same page", async () => { + it("persists expired snapshot failure backoff and resumes the same page", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config({ keeper, @@ -220,13 +220,14 @@ describe("current-state change consumer bootstrap", () => { await initSchema(db, resolved); await apply(db, resolved, event({ rkey: "a", time: 1 })); const page = await claimCurrentSnapshotPage(db, resolved, "search", { + leaseMs: 10, now: 100, }); await failCurrentSnapshotPage( db, page!, { code: "destination_unavailable", nextAttemptAt: 200 }, - { now: 101 }, + { now: 111 }, ); expect( await claimCurrentSnapshotPage(db, resolved, "search", { now: 150 }), diff --git a/packages/contrail/tests/change-consumers.test.ts b/packages/contrail/tests/change-consumers.test.ts index 71b6184..f1d1528 100644 --- a/packages/contrail/tests/change-consumers.test.ts +++ b/packages/contrail/tests/change-consumers.test.ts @@ -246,7 +246,7 @@ describe("durable change consumers", () => { }); }); - it("allows only one concurrent lease and rejects an expired owner", async () => { + it("allows only one concurrent lease and rejects a replaced owner", async () => { const db = createSqliteDatabase(":memory:"); const config = readyEventConsumer(); await initSchema(db, config); @@ -269,6 +269,14 @@ describe("durable change consumers", () => { await expect( acknowledgeChanges(db, stale, { now: 152 }), ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await expect( + failChanges( + db, + stale, + { code: "stale_failure", nextAttemptAt: 300 }, + { now: 152 }, + ), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); await acknowledgeChanges(db, replacement!, { now: 152 }); }); diff --git a/packages/contrail/tests/delivery.test.ts b/packages/contrail/tests/delivery.test.ts index 2835fe5..df983e3 100644 --- a/packages/contrail/tests/delivery.test.ts +++ b/packages/contrail/tests/delivery.test.ts @@ -192,6 +192,46 @@ describe("change delivery runtime", () => { }); }); + it("persists handler failure after the claim lease expires", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + webhook: { collections: [EVENT], initial: "history" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + + let current = 100; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { + webhook: async () => { + current = 151; + throw new Error("slow destination failure"); + }, + }, + runtime: { + maxRounds: 1, + claim: { leaseMs: 50 }, + baseRetryMs: 100, + maxRetryMs: 100, + jitter: 0, + clock: () => current, + }, + }); + + expect(result).toMatchObject({ delivered: 0, failures: 1 }); + expect((await getChangesStatus(db)).consumers[0]).toMatchObject({ + position: "0", + attempts: 1, + nextAttemptAt: 251, + lastErrorCode: "handler_error", + }); + }); + it("drives current snapshot, catch-up, and idempotent activation", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config({ @@ -261,6 +301,47 @@ describe("change delivery runtime", () => { }); }); + it("uses the configured lease for a slow activation", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + search: { collections: [EVENT], initial: "current" }, + }); + const contrail = new Contrail({ ...resolved, db }); + await contrail.init(); + await append(db, resolved, "one", 1); + + let current = 100; + const result = await runChangeDeliverySlice({ + changes: contrail.changes, + config: resolved, + db, + env: {}, + deliveries: { search: async () => {} }, + bootstraps: { + search: { + snapshot: async () => {}, + activate: async (activation) => { + expect(activation.leaseExpiresAt).toBe(40_100); + current = 31_100; + }, + }, + }, + runtime: { + maxRounds: 6, + maxDurationMs: 60_000, + claim: { leaseMs: 40_000 }, + jitter: 0, + clock: () => current, + }, + }); + + expect(result).toMatchObject({ activations: 1, failures: 0 }); + expect(await contrail.changes.bootstrapStatus("search")).toMatchObject({ + state: "ready", + position: "1", + }); + }); + it("aborts destination work at the runtime deadline", async () => { const db = createSqliteDatabase(":memory:"); const resolved = config({ -- 2.51.2 From 746367bc587d0b1c2752c2a7fd7cda2e0c9235fb Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:58:43 +0200 Subject: [PATCH 10/10] small fixes, update docs --- README.md | 17 +- apps/atmo-rsvp/README.md | 2 - apps/cloudflare-workers/README.md | 2 +- docs/00-getting-started.md | 32 ++ docs/01-client.md | 34 ++ docs/01-indexing.md | 184 ----------- docs/02-configure-and-query.md | 83 +++++ docs/02-querying.md | 195 ----------- docs/03-deploy-cloudflare.md | 101 ++++++ docs/04-feeds.md | 122 ------- docs/09-labels.md | 142 -------- docs/advanced/README.md | 9 + docs/advanced/feeds.md | 41 +++ docs/advanced/labels.md | 37 +++ docs/advanced/outbox.md | 94 ++++++ docs/frameworks/sveltekit-cloudflare.md | 213 ------------ docs/public-services/api-atmo-rsvp.md | 246 -------------- docs/public-services/creating.md | 311 ------------------ docs/public-services/using.md | 164 --------- packages/contrail/README.md | 1 - .../contrail/src/core/change-bootstrap.ts | 22 +- packages/contrail/src/core/change-log.ts | 1 + packages/contrail/src/core/changes.ts | 109 +++++- packages/contrail/src/core/db/schema.ts | 6 + .../contrail/tests/change-bootstrap.test.ts | 40 +++ .../contrail/tests/change-consumers.test.ts | 28 +- packages/contrail/tests/change-log.test.ts | 14 +- 27 files changed, 629 insertions(+), 1621 deletions(-) create mode 100644 docs/00-getting-started.md create mode 100644 docs/01-client.md delete mode 100644 docs/01-indexing.md create mode 100644 docs/02-configure-and-query.md delete mode 100644 docs/02-querying.md create mode 100644 docs/03-deploy-cloudflare.md delete mode 100644 docs/04-feeds.md delete mode 100644 docs/09-labels.md create mode 100644 docs/advanced/README.md create mode 100644 docs/advanced/feeds.md create mode 100644 docs/advanced/labels.md create mode 100644 docs/advanced/outbox.md delete mode 100644 docs/frameworks/sveltekit-cloudflare.md delete mode 100644 docs/public-services/api-atmo-rsvp.md delete mode 100644 docs/public-services/creating.md delete mode 100644 docs/public-services/using.md diff --git a/README.md b/README.md index 7c6c4f0..deaa28d 100644 --- a/README.md +++ b/README.md @@ -133,19 +133,16 @@ import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; ``` -The runnable [`apps/sqlite`](apps/sqlite) example wires the standard backfill CLI to a local SQLite file, including the optional Alluvium base/archive path. See [Indexing](docs/01-indexing.md) for adapter setup and [Querying](docs/02-querying.md) for the query and hydration model. +The runnable [`apps/sqlite`](apps/sqlite) example wires the standard backfill CLI to a local SQLite file, including the optional Alluvium base/archive path. ## Documentation -- [Indexing](docs/01-indexing.md) -- [Querying](docs/02-querying.md) -- [Feeds](docs/04-feeds.md) -- [Labels](docs/09-labels.md) -- Public Contrail services: - - [Creating a service](docs/public-services/creating.md) - - [Using a service](docs/public-services/using.md) - - [Example: api.atmo.rsvp](docs/public-services/api-atmo-rsvp.md) -- [SvelteKit + Cloudflare](docs/frameworks/sveltekit-cloudflare.md) +1. [Get started locally](docs/00-getting-started.md) +2. [Add the typed client](docs/01-client.md) +3. [Configure and query](docs/02-configure-and-query.md) +4. [Deploy to Cloudflare Workers](docs/03-deploy-cloudflare.md) + +See [advanced topics](docs/advanced/README.md) for other runtimes and features. ## Repository layout diff --git a/apps/atmo-rsvp/README.md b/apps/atmo-rsvp/README.md index bdb8cf5..83a75ae 100644 --- a/apps/atmo-rsvp/README.md +++ b/apps/atmo-rsvp/README.md @@ -92,5 +92,3 @@ pnpx @atmo-dev/contrail connect https://api.atmo.rsvp ``` That verifies the anonymous and service-auth contracts, verifies the canonical contract and Lexicon digests, writes a provider lock, installs provider-owned Lexicons, and runs Atcute TypeScript generation. Reconnecting an existing project requires `--update`. - -See [Example: api.atmo.rsvp](../../docs/public-services/api-atmo-rsvp.md) for the complete method, authentication, acquisition, and deployment walkthrough. diff --git a/apps/cloudflare-workers/README.md b/apps/cloudflare-workers/README.md index 114d5cf..7db07bc 100644 --- a/apps/cloudflare-workers/README.md +++ b/apps/cloudflare-workers/README.md @@ -71,4 +71,4 @@ This reports the durable state without exposing account DIDs or raw upstream err - **add a collection:** append to `collections` in `src/contrail.config.ts`; redeploy; `pnpm contrail backfill --remote` to backfill the new one. - **add full-text search:** `searchable: ["field1", "field2"]`, redeploy, no backfill needed (fts indexes repopulate on ingest). -- **add relations / references:** see [indexing docs](../../docs/01-indexing.md). +- **add relations / references:** see [configuration guide](../../docs/02-configure-and-query.md). diff --git a/docs/00-getting-started.md b/docs/00-getting-started.md new file mode 100644 index 0000000..2e5cad0 --- /dev/null +++ b/docs/00-getting-started.md @@ -0,0 +1,32 @@ +# Get started locally + +Run a local AppView for a public AT Protocol collection with Node.js 22.13 or newer. + +Create an empty directory with one file: + +```ts +// contrail.config.ts +export default { + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { startsAt: { type: "range" } }, + }, + }, +}; +``` + +Then run: + +```bash +pnpx @atmo-dev/contrail dev +``` + +Contrail resolves the Lexicons, backfills existing records, follows new records, and serves a resumable SQLite AppView at `http://127.0.0.1:8787`. + +```bash +curl 'http://127.0.0.1:8787/xrpc/com.example.event.listRecords?limit=10' +``` + +Replace the collection and fields with your own. Next: [add the typed client to your app](./01-client.md). diff --git a/docs/01-client.md b/docs/01-client.md new file mode 100644 index 0000000..e498f93 --- /dev/null +++ b/docs/01-client.md @@ -0,0 +1,34 @@ +# Add the typed client + +With your [local AppView](./00-getting-started.md) set up, add Contrail and Atcute to your application: + +```bash +pnpm add @atmo-dev/contrail @atcute/client @atcute/lexicons +pnpx @atmo-dev/contrail connect ../my-appview +``` + +Point `connect` at the directory containing `contrail.config.ts`. It resolves the source and query Lexicons, then uses Atcute to generate a typed client in `src/contrail/`. + +Use it from your app: + +```ts +import { createLocalContrailClient } from "./contrail/index.js"; + +const contrail = createLocalContrailClient(); +const response = await contrail.get("com.example.event.listRecords", { + params: { + startsAtMin: new Date().toISOString(), + limit: 20, + }, +}); + +if (!response.ok) throw new Error(`Contrail returned ${response.status}`); + +for (const event of response.data.records) { + console.log(event.value.name, event.value.startsAt); +} +``` + +The method name, parameters, and response are all typed from the Lexicons. Re-run `connect` when the AppView config changes. + +Next: [configure filters, sorting, and hydration](./02-configure-and-query.md). To use an existing deployed AppView, pass its HTTPS URL to `contrail connect` instead of a config path. diff --git a/docs/01-indexing.md b/docs/01-indexing.md deleted file mode 100644 index f4117ba..0000000 --- a/docs/01-indexing.md +++ /dev/null @@ -1,184 +0,0 @@ -# Indexing - -Contrail's core job: mirror atproto records into your DB and expose them via XRPC. You describe what to index with a config object; everything else is automatic. - -## Collection shape - -A realistic two-collection example: events and RSVPs. RSVPs point at events via `subject.uri`; events expose per-status RSVP counts. - -```ts -collections: { - event: { - collection: "community.lexicon.calendar.event", // full NSID - queryable: { - mode: {}, // ?mode=online - startsAt: { type: "range" }, // ?startsAtMin=...&startsAtMax=... - }, - searchable: ["name", "description"], // FTS5 / tsvector - relations: { - rsvps: { - collection: "rsvp", // short name of the child collection - groupBy: "status", // field on the child record - groups: { - going: "community.lexicon.calendar.rsvp#going", - interested: "community.lexicon.calendar.rsvp#interested", - }, - }, - }, - }, - rsvp: { - collection: "community.lexicon.calendar.rsvp", - queryable: { status: {} }, - references: { - event: { collection: "event", field: "subject.uri" }, // RSVP's field → event's URI - }, - }, -} -``` - -- **queryable** — string equality or range, exposed as query params. -- **searchable** — FTS5 on D1/Postgres. Not available on `node:sqlite`. -- **relations** — many-to-one with materialized counts. The `event` collection gains `rsvpsCount`, `rsvpsGoingCount`, `rsvpsInterestedCount` columns — filter (`?rsvpsGoingCountMin=10`) and sort (`?sort=rsvpsGoingCount`) on them. Hydrate inline with `?hydrateRsvps=5`. -- **references** — forward lookups from child → parent. `?hydrateEvent=true` on an RSVP query embeds the referenced event record. - -## Backfill (historical data) - -Run once at setup to pull every record that exists today. - -```ts -await contrail.backfillAll({ concurrency: 100 }); // discover + backfill, logs progress -``` - -Under the hood this is two steps you can call separately if you want finer control: - -```ts -await contrail.discover(); // walk relays, register DIDs -await contrail.backfill({ - concurrency: 100, // identity resolution - pdsConcurrency: 20, // active PDS hosts - didsPerPds: 3, // accounts per active PDS -}); -``` - -`backfill()` picks up each account/collection at its saved PDS cursor. A row is marked complete only after the PDS listing reaches its end. Timeouts, failed identity resolution, `429`, and `5xx` responses leave the row pending with its last error. - -Each initial invocation attempts a failed account once by default, then gets out of the way. Failed rows retain their cursors and receive an exponential `next_retry_at`. Scheduled retries start at 15 minutes, double to a maximum of 48 hours, and stop after ten failed scheduled attempts. Cloudflare's scheduled Worker retries a small due slice after each live-ingest cycle; an explicit later invocation resets exhausted rows and forces another pass. `backfillAll()` returns a durable `status` summary alongside the number of discovered accounts and accepted records. - -Historical loading writes canonical records first, then rebuilds FTS and materialized relation counts with set-based SQL. A durable dirty marker keeps status `incomplete` if the process stops between those phases; the next manual or scheduled backfill repairs the projections before reporting readiness. Live ingestion and scheduled account retries continue maintaining both projections incrementally. - -### Workers CLI - -For Cloudflare Workers deploys, `@atmo-dev/contrail` ships a `contrail` bin that handles the `wrangler.getPlatformProxy` dance — no script file, no package.json alias needed: - -```bash -pnpm contrail backfill # local D1 (wrangler dev's bindings) -pnpm contrail backfill --remote # production D1 -``` - -Auto-detects configs at `contrail.config.ts`, `src/contrail.config.ts`, `src/lib/contrail.config.ts`, or `app/contrail.config.ts` (first match wins). Override with `--config `. Other flags include `--binding ` (default `DB`), `--concurrency ` for identity resolution (default 100), `--pds-concurrency ` (default 20), `--dids-per-pds ` (default 3), and `--max-attempts ` (default 1). Once every known account has either completed or received a deferred failure, the initial pass is complete and scheduled retries continue in the background. Interrupted or undiscovered work still reports the pass as incomplete. - -If you'd rather embed backfill inside your own script, `@atmo-dev/contrail/workers` exports the same logic as a function: - -```ts -import { backfillAll } from "@atmo-dev/contrail/workers"; -import { config } from "../src/contrail.config"; - -await backfillAll({ config, remote: process.argv.includes("--remote") }); -``` - -For node/postgres deploys, skip both — you already have a `db` in hand; just `await contrail.backfillAll({}, db)` directly. - -## Ingestion (ongoing new records) - -After the initial `backfillAll()`, keep the index fresh with new records as they're published. Pick the mode that matches your runtime. - -### Cron-driven (cloudflare workers) - -Workers can't hold long-lived connections, so run one catch-up cycle per cron fire: - -```ts -// wrangler.jsonc: "triggers": { "crons": ["*/1 * * * *"] } -async scheduled(_ev, env, ctx) { - ctx.waitUntil((async () => { - await contrail.ingest({}, env.DB); - await contrail.retryBackfill({}, env.DB); // small due slice - })()); -} -``` - -`ingest()` connects to Jetstream, streams events since the saved cursor, stops when caught up. Running every minute is fine — the next fire resumes where this one left off. Each cycle is bounded, so it can't blow past the Worker time limit. - -**Local dev:** wrangler's cron scheduler only runs in deployed production. For local dev use `pnpm contrail dev` — it runs `wrangler dev --test-scheduled`, fires `/__scheduled` on your configured cron interval, and offers to start or resume backfill whenever known work remains. - -### Persistent (node / any long-lived server) - -If your runtime can keep a socket open, skip the cron entirely: - -```ts -const ac = new AbortController(); -await contrail.runPersistent({ - batchSize: 50, // flush every N events (default: 50) - flushIntervalMs: 5000, // or every N ms, whichever first - signal: ac.signal, -}); -// ac.abort() flushes the current batch and saves the cursor before returning -``` - -One process, one socket, auto-reconnect on drops. Lower latency than cron mode (events land within seconds instead of up-to-a-minute), but needs a runtime that can run indefinitely. - -### Immediate (`notify()`) - -Use this when your own app writes to a user's PDS and needs the change indexed *now* — waiting for the next cron / Jetstream flush is too slow: - -```ts -await contrail.notify(uri); // one record -await contrail.notify([u1, u2, u3]); // batch, up to 25 -``` - -Fetches directly from the user's PDS and indexes synchronously. When Jetstream later delivers the same event, the duplicate is detected by CID and skipped. - -### Which one do I use? - -| | backfillAll | ingest | runPersistent | notify | -|---|---|---|---|---| -| when | once, at setup | every cron fire | start once, runs forever | per-write, on demand | -| runtime | local script | cloudflare workers | node / long-lived server | anywhere | -| scope | all historical records | events since last cursor | events since last cursor, live | specific URIs | -| latency | — | ~minute | ~seconds | immediate | - -Typical combos: -- **workers app:** `backfillAll()` once + `ingest()` on cron + optional `notify()` for self-writes -- **node server:** `backfillAll()` once + `runPersistent()` forever + optional `notify()` for self-writes - -## Recovery after an outage - -Normal ingestion resumes from its saved Jetstream cursor, so a short outage needs no special command: restart `ingest()` or `runPersistent()` and let it catch up. - -Contrail does not perform a full PDS sweep as a repair mechanism. Such a sweep is expensive, cannot discover repositories it never knew about, and cannot safely infer remote deletions after partial failures. If the saved cursor is older than the source's retained history, rebuild into a fresh database with `backfillAll()` rather than trusting a partial reconciliation. A replay-capable source and first-class projection rebuild command are planned follow-up work. - -Reading the indexed data — filters, sorts, hydration, search, pagination — has its own doc: [Querying](./02-querying.md). - -## Adapters - -| Adapter | Use when | FTS | -|---|---|---| -| Cloudflare D1 | Workers | ✅ | -| `@atmo-dev/contrail/sqlite` | Node 22+ local dev | ❌ | -| `@atmo-dev/contrail/postgres` | Node server | ✅ | - -```ts -import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; -const db = createPostgresDatabase(pool); -``` - -## Top-level config - -| Key | Default | | -|---|---|---| -| `namespace` | — | Reverse-domain for XRPC paths | -| `profiles` | `["app.bsky.actor.profile"]` | Profile NSIDs, auto-hydrated via `?profiles=true` | -| `jetstreams` | Bluesky | Jetstream URLs | -| `relays` | Bluesky | Relay URLs for discovery | -| `notify` | off | Prefer an in-process call or a secret string requiring `Bearer`; open `true` mode is not recommended | -| `feeds` | — | See [Feeds](./04-feeds.md) | -| `labels` | — | See [Labels](./09-labels.md) | diff --git a/docs/02-configure-and-query.md b/docs/02-configure-and-query.md new file mode 100644 index 0000000..2b5e27f --- /dev/null +++ b/docs/02-configure-and-query.md @@ -0,0 +1,83 @@ +# Configure and query + +Each entry in `collections` becomes typed `getRecord` and `listRecords` methods. Declare only the fields and relationships your app needs: + +```ts +// contrail.config.ts +export default { + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { + mode: {}, + startsAt: { type: "range" }, + }, + searchable: ["name", "description"], + relations: { + rsvps: { + collection: "rsvp", + groupBy: "status", + groups: { + going: "community.lexicon.calendar.rsvp#going", + }, + }, + }, + }, + rsvp: { + collection: "community.lexicon.calendar.rsvp", + queryable: { + status: {}, + "subject.uri": {}, + }, + references: { + event: { collection: "event", field: "subject.uri" }, + }, + }, + }, +}; +``` + +This produces the following client parameters: + +| Config | Query parameter | +|---|---| +| `mode: {}` | `mode` | +| `startsAt: { type: "range" }` | `startsAtMin`, `startsAtMax` | +| `searchable` | `search` | +| `relations.rsvps` | `rsvpsCountMin`, `hydrateRsvps` | +| `groups.going` | `rsvpsGoingCountMin` | +| `references.event` | `hydrateEvent` | + +Dotted fields become camel case: `subject.uri` becomes `subjectUri`. Every list method also supports `actor` (a DID or handle), `sort`, `order`, `limit`, `cursor`, and `profiles`. + +## Query from the client + +After changing the config, re-run `contrail connect` in your app. The generated Atcute client now knows the new parameters and response types: + +```ts +const response = await contrail.get("com.example.event.listRecords", { + params: { + mode: "in-person", + startsAtMin: new Date().toISOString(), + rsvpsGoingCountMin: 5, + sort: "startsAt", + order: "asc", + hydrateRsvps: 3, + profiles: true, + limit: 20, + }, +}); + +if (!response.ok) throw new Error(`Contrail returned ${response.status}`); + +const { records, profiles, cursor } = response.data; +``` + +Every record has `uri`, `cid`, and its original record body in `value`. Hydrated relations and references are added to that record; requested profiles are returned once in the top-level `profiles` array. + +Pass the returned opaque `cursor` into the same query to get the next page. `limit` defaults to 50 and may be 1–200. + +Full-text `search` works with D1 and PostgreSQL. The zero-config local SQLite AppView does not provide full-text search. + +Next: [deploy to Cloudflare Workers](./03-deploy-cloudflare.md). diff --git a/docs/02-querying.md b/docs/02-querying.md deleted file mode 100644 index 46dac29..0000000 --- a/docs/02-querying.md +++ /dev/null @@ -1,195 +0,0 @@ -# Querying - -Once [indexing](./01-indexing.md) is set up, every collection you declared gets a pair of XRPC endpoints under `/xrpc/{namespace}.{short}.*`: - -| Endpoint | Returns | -|---|---| -| `{namespace}.{short}.listRecords` | Paginated list with filters, sorts, hydration | -| `{namespace}.{short}.getRecord?uri=…` | Single record by AT-URI | - -Top-level methods include `{namespace}.getProfile`, `{namespace}.getCursor`, `{namespace}.notifyOfUpdate`, and optionally `{namespace}.getFeed` and `{namespace}.lexicons`. - -## HTTP (what most callers use) - -Every config field becomes a predictable URL param: - -``` -/xrpc/com.example.event.listRecords?mode=online&startsAtMin=2026-01-01&rsvpsGoingCountMin=10&sort=startsAt&order=asc&hydrateRsvps=5 -/xrpc/com.example.event.getRecord?uri=at://did:plc:.../...&hydrateRsvps=5 -``` - -| Config produces | URL param | -|---|---| -| `queryable: { field: {} }` | `?field=value` (equality) | -| `queryable: { field: { type: "range" } }` | `?fieldMin=…`, `?fieldMax=…` | -| `relations: { rel: {...} }` | `?relCountMin=N`, `?sort=relCount`, `?hydrateRel=N` | -| `relations: { rel: { groups: { going } } }` | `?relGoingCountMin=N`, `?sort=relGoingCount` | -| `references: { ref: {...} }` | `?hydrateRef=true` | - -Dotted field names become camelCase params — `queryable: { "subject.uri": {} }` → `?subjectUri=…`. - -## Operational status - -`GET /status` returns the current JSON overview. It includes indexed record totals, live-ingest freshness, and durable backfill state: - -- discovery source progress; -- mutually exclusive account totals for `complete`, `pending`, `retrying`, and `failed`; -- known-account completion percentage; -- the same mutually exclusive totals per collection; and -- scheduled/due account retries plus the next retry time. - -`state` is `running` while a manual or scheduled slice holds the backfill lease. It becomes `complete` after discovery and the initial pass finish, even when account-level `retrying` or `failed` counts are non-zero. `pending` means no failure has occurred yet, `retrying` means at least one attempt failed but automatic attempts remain, and `failed` means the ten-attempt automatic budget is exhausted. An explicit backfill resets that budget. `incomplete` means discovery or an initial account attempt has not finished. - -"Known" is deliberate: while relay discovery is incomplete, Contrail cannot honestly claim how many accounts remain undiscovered. `/health` remains a lightweight liveness response and does not claim that historical backfill is complete. - -`GET /xrpc/{namespace}.getCursor` returns the committed primary ordered-source position when `orderedSource` is configured. The `{ source, epoch, cursor }` tuple is opaque: compare complete tuples for equality only, and treat a source or epoch change as a full reset. A consumer that needs a stable query snapshot can read the position before and after its query and retry when the two positions differ. - -## Programmatic - -```ts -const { records, cursor } = await contrail.query("event", { - filters: { mode: "online" }, - rangeFilters: { startsAt: { min: "2026-01-01" } }, - countFilters: { rsvp: 10 }, // keyed by child collection short name - sort: { recordField: "startsAt", direction: "asc" }, - limit: 20, -}); -``` - -The programmatic shape doesn't use the URL param names — keys are the underlying field/collection identifiers: - -- `filters` / `rangeFilters` — keyed by the field name from your config (`startsAt`, `subject.uri`), not the camelCased URL param. -- `countFilters` — keyed by the target collection's short name for totals, or by the full `nsid#group` token for group counts. E.g., `{ rsvp: 10 }` for "at least 10 RSVPs total," or `{ "community.lexicon.calendar.rsvp#going": 10 }` for "at least 10 going." -- `sort` — `{ recordField, direction }` for field sorts, `{ countType, direction }` for count sorts (where `countType` is the same collection-short-name or `nsid#group` as above). Field sorts preserve the SQL value type, and records whose field is missing or `null` sort last in either direction. - -For count filters / sorts, the HTTP side is nicer than the programmatic side — consider going through `createHandler` + `fetch` even for in-process calls if you want the friendly names. Or use `createServerClient` from `@atmo-dev/contrail/server` for a typed XRPC client that runs in-process (no fetch roundtrip). - -## Pagination - -``` -?limit=25&cursor= -``` - -`cursor` is opaque — pass back whatever `listRecords` returned in its `cursor` field. `limit` is 1–200 (default 50). Cursors embed the complete ordering, including relevance rank for search and URI as the final unique tiebreaker. They also embed the sort kind, so a cursor from a `sort=startsAt` query is ignored by a `sort=rsvpsCount` query instead of silently returning wrong results. - -```ts -let cursor: string | undefined; -do { - const page = await contrail.query("event", { limit: 100, cursor }); - // process page.records - cursor = page.cursor; -} while (cursor); -``` - -## Hydration - -Each record response is a flat shape: - -```jsonc -{ - "uri": "at://did:plc:.../community.lexicon.calendar.event/...", - "cid": "...", - "value": { "name": "Rust meetup", "startsAt": "2026-03-16T...", ... }, - "rsvpsCount": 42, // from relations - "rsvpsGoingCount": 30, - // relations + references appear here only when hydrated -} -``` - -The `value` field carries the record body — same shape as atproto's `com.atproto.repo.listRecords#record`. `did`, `collection`, `rkey`, and `time_us` are also returned alongside as optional extras. - -### `?hydrateRel=N` (relations) - -Embeds the latest N child records per group, inline under the parent: - -``` -/xrpc/com.example.event.listRecords?hydrateRsvps=5 -``` - -Returns: - -```jsonc -{ - "records": [{ - "uri": "at://.../event/...", - "value": { "name": "..." }, - "rsvpsCount": 42, - "rsvps": { - "going": [ {uri, cid, value}, ... 5 items ], - "interested":[ {uri, cid, value}, ... 5 items ] - } - }] -} -``` - -Max 50 per group. For grouped relations you get one array per group value; for ungrouped relations just a flat array. - -### `?hydrateRef=true` (references) - -Embeds the single referenced parent record — useful for RSVP lists that need to show event details: - -``` -/xrpc/com.example.rsvp.listRecords?subjectUri=at://.../event/...&hydrateEvent=true -``` - -Each RSVP record in the response gains an `event: {uri, cid, value}` field. - -### `?profiles=true` - -Opt in to profile + handle hydration for every DID referenced in the result: - -``` -/xrpc/com.example.event.listRecords?profiles=true -``` - -Response grows a top-level `profiles` array, one entry per (DID, configured profile NSID): - -```jsonc -{ - "records": [...], - "profiles": [ - { - "did": "did:plc:alice...", - "handle": "alice.bsky.social", - "uri": "at://did:plc:alice.../app.bsky.actor.profile/self", - "cid": "...", - "collection": "app.bsky.actor.profile", - "rkey": "self", - "value": { /* profile record body */ } - } - ] -} -``` - -A DID with no profile record (or whose handle resolved but profile didn't) shows up as a bare `{ did, handle }` entry — `uri`/`cid`/`value` are omitted. With multiple profile NSIDs configured, you'll see one entry per (DID × NSID) that resolved. - -Which profile NSID(s) to hydrate from is configured at the top level of Contrail's config (`profiles`, defaults to `["app.bsky.actor.profile"]`). - -## Full-text search - -``` -?search=meetup -?search=meetup* -?search="rust meetup" -?search=rust OR typescript -``` - -Combinable with every other filter and sort. Backed by SQLite FTS5 (D1) or Postgres tsvector (Postgres adapter). Not available on `node:sqlite` — that adapter doesn't ship FTS5. - -When searching, results are ranked by relevance by default. Override with an explicit `sort` param. - -## Examples - -``` -# Upcoming events with 10+ going RSVPs, with RSVP records + profiles -?startsAtMin=2026-03-16&rsvpsGoingCountMin=10&hydrateRsvps=5&profiles=true - -# Events for a specific user (by handle — triggers on-demand backfill) -?actor=alice.bsky.social&profiles=true - -# RSVPs for one event, with the event record embedded -?subjectUri=at://did:plc:.../event/...&hydrateEvent=true&profiles=true - -# Search + filter + sort -?search=meetup&mode=online&sort=startsAt&order=asc -``` diff --git a/docs/03-deploy-cloudflare.md b/docs/03-deploy-cloudflare.md new file mode 100644 index 0000000..5252755 --- /dev/null +++ b/docs/03-deploy-cloudflare.md @@ -0,0 +1,101 @@ +# Deploy to Cloudflare Workers + +Turn the same local AppView into a public Worker backed by D1. + +## Install + +From the directory containing `contrail.config.ts`: + +```bash +pnpm init +pnpm add @atmo-dev/contrail +pnpm add -D @atcute/lex-cli wrangler typescript +``` + +Add an ordered source to the config. Keep its epoch stable for the lifetime of this database: + +```ts +// contrail.config.ts +import type { ContrailConfig } from "@atmo-dev/contrail"; + +const config: ContrailConfig = { + namespace: "com.example", + orderedSource: { + source: "jetstream", + epoch: "my-appview-v1", + }, + collections: { + // ...your existing collections + }, +}; + +export default config; +``` + +Generate the public query Lexicons and their referenced record Lexicons: + +```bash +pnpm contrail lexicons all --public +``` + +## Create the Worker + +```ts +// worker.ts +import { createWorker } from "@atmo-dev/contrail/worker"; +import config from "./contrail.config"; +import { lexicons } from "./lexicons/generated"; + +export default createWorker(config, { + lexicons, + publicService: { + endpoint: "https://my-appview.example.com", + }, +}); +``` + +Use the Worker's actual `workers.dev` or custom-domain URL as `endpoint`. + +Create `wrangler.jsonc`: + +```jsonc +{ + "name": "my-appview", + "main": "worker.ts", + "compatibility_date": "2025-12-25", + "observability": { "enabled": true }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "my-appview", + "database_id": "PASTE_DATABASE_ID_HERE" + } + ], + "triggers": { "crons": ["*/1 * * * *"] } +} +``` + +## Deploy and backfill + +```bash +pnpm wrangler d1 create my-appview # copy its ID into wrangler.jsonc +pnpm wrangler deploy +pnpm contrail backfill --remote +``` + +The one-minute cron keeps the AppView current. Check it with: + +```bash +curl https://my-appview.example.com/status +curl 'https://my-appview.example.com/xrpc/com.example.event.listRecords?limit=10' +``` + +Finally, point the application from the previous guide at the deployment: + +```bash +pnpx @atmo-dev/contrail connect https://my-appview.example.com +``` + +When the config changes, regenerate the Lexicons, deploy, backfill any new collections, and reconnect the client with `--update`. + +For optional outbox deliveries, feeds, and labels, see [advanced topics](./advanced/README.md). diff --git a/docs/04-feeds.md b/docs/04-feeds.md deleted file mode 100644 index 0f5c5e1..0000000 --- a/docs/04-feeds.md +++ /dev/null @@ -1,122 +0,0 @@ -# Feeds - -Personalized "what the people I follow are doing" timelines, fanned out at write time. Opt-in; no cost if you don't enable it. - -## Mental model - -> A feed is a (follow-collection, [target-collections]) pair, named by you. Every time someone an *actor* follows posts to a target collection, contrail inserts one row into `feed_items` for that actor. - -Reading a feed is a join through `feed_items` plus the standard pipeline (filters, sorts, hydration, references). The actor parameter on a read is *whose feed* you want — there is no anonymous feed read. - -## Enable - -```ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -const config: ContrailConfig = { - namespace: "com.example", - collections: { - follow: { collection: "app.bsky.graph.follow" }, - post: { collection: "app.bsky.feed.post", queryable: { /* ... */ } }, - }, - feeds: { - timeline: { - follow: "follow", // short name (key in `collections`), NOT the NSID - targets: ["post"], - maxItems: 500, // optional, default 200 - }, - }, -}; -``` - -Both the follow collection and every target collection must be declared in `collections`. Names in `feeds` are the **short names** (the keys of `collections`), not NSIDs. Config validation throws if you reference an unknown short name. - -## Follow-record shape - -The follow collection's record must have a `subject` field at the top level whose value is the followed DID. `app.bsky.graph.follow` matches this naturally: - -```json -{ "subject": "did:plc:abc...", "createdAt": "2026-01-01T00:00:00Z" } -``` - -Custom follow lexicons work as long as `subject` is the followed DID at JSON path `$.subject`. Contrail extracts via that path during ingest fan-out and during follow-event backfill. - -## Schema - -Two tables, one shared across all feeds: - -| Table | Purpose | -|---|---| -| `feed_items (actor, uri, collection, time_us)` | One row per (viewer, target record). Primary key `(actor, uri)` so a single target record can appear in many feeds. | -| `feed_backfills (actor, feed, completed)` | Marker so first-read backfill only runs once per (actor, feed). | - -Indexes: `(actor, collection, time_us DESC)` and `(actor, time_us DESC)` on `feed_items`, plus a JSON `subject` index on each follow collection's records table for the fan-out join. - -## Read - -``` -GET /xrpc/{namespace}.getFeed?feed=timeline&actor=&limit=50 -``` - -| Param | Meaning | -|---|---| -| `feed` | Feed name from `config.feeds` (required) | -| `actor` | Whose feed — DID or handle (required) | -| `collection` | Restrict to one target collection's short name (default: first in `targets`) | -| `limit`, `cursor`, filters from the target's `queryable`, hydration flags, sort/order | Same as `listRecords` on the target collection | - -The `actor` parameter is **whose feed** you're reading, not a filter on record creator. Feeds are always per-user. - -```ts -const feed = await fetch( - `/xrpc/com.example.getFeed?feed=timeline&actor=${did}&limit=50&profiles=true` -).then((r) => r.json()); -// feed.records — target records by users `actor` follows, newest first -// feed.profiles — hydrated profile records for record authors -``` - -## How fan-out works - -Three moments: - -1. **A target write.** Someone followed by N actors posts to a target collection. Contrail inserts N `feed_items` rows in one statement (`INSERT … SELECT … FROM WHERE subject = ?`). Cost is linear in N — there is no max-followers cap; a viral author with 1M followers is 1M inserts. - -2. **A follow write.** An actor follows a new user. Contrail backfills the most recent **100** target records from that user into the new follower's feed. The 100 is hardcoded in `core/router/feed.ts` — separate from the per-feed `maxItems` cap, and not tunable per feed today. - -3. **First read for an (actor, feed) pair.** Contrail backfills the actor's follow records from their PDS (so their `feed_items` rows can be computed), then populates `feed_items` from existing target records by users they already follow. Marked complete in `feed_backfills` so it runs once per pair. - -## Pruning - -Feeds are capped: each actor keeps at most `maxItems` rows per target collection (default 200, newest first). Older rows past the cap are deleted by a background cleanup that piggybacks on ingestion — there is no separate prune job. - -A few terms used below: - -- **Tick** — one cycle of the ingest loop. In cron mode the worker wakes on a schedule (e.g. once a minute) and each wake-up is a tick; in the persistent loop it's each batch flush. -- **Sweep** — the cleanup that walks `feed_items` actor by actor and deletes whatever is over an actor's cap. -- **Slice** — a sweep doesn't scan the whole table at once. Each tick it handles a chunk of up to `FEED_PRUNE_SWEEP_ACTORS` actors (default 500). That chunk is one slice. -- **Cursor / full pass** — a bookmark for the last actor a slice stopped on, so the next slice resumes after it instead of restarting. When the cursor reaches the last actor it *wraps* back to the start; one start-to-end trip is a *full pass*. - -**When the sweep runs.** A feed can only go over its cap right after a feed-mutating record (a target fan-out or a follow backfill) is applied, so the sweep is skipped entirely on ticks that ingested nothing feed-relevant. It runs when the current tick — a cron run, a persistent-loop flush, or a `notifyOfUpdate` call — applied a feed-mutating record. As a safety net it also runs on a recovery interval (`FEED_PRUNE_RECOVERY_INTERVAL_MS`, 6h), so rows that went over cap without a fresh ingest (a lowered cap, a bulk import) still get cleaned up — including on a stream that is otherwise idle. - -Doing one slice per tick keeps each tick's cost flat no matter how big the table grows. The recovery timer measures from the last *completed full pass* (not the last slice): a fresh pass becomes due one recovery interval after the previous one finished, then advances a slice per tick until the cursor wraps. So a full pass *completes* roughly every `recovery interval + lap time`, where lap time is `ceil(actors / FEED_PRUNE_SWEEP_ACTORS)` ticks — e.g. with 100k actors and one-minute cron ticks, ~6h + ~3h20m. That keeps the whole table draining on a bounded cadence; it is not a hard "fully clean every 6h" guarantee. Raise `FEED_PRUNE_SWEEP_ACTORS` if you need the lap time shorter at large actor counts. - -**Fan-out isn't cleaned up instantly.** A slice cleans up whatever actors the cursor lands on next — not specifically the actors whose feeds just changed. So when a popular author posts and fans out to many followers: a follower the cursor *hasn't reached yet* this pass is trimmed later in the same pass (soon), but a follower the cursor has *already passed* waits for the next pass — and on a quiet stream the next pass only starts on the recovery interval. So the worst case for an over-cap follower is roughly one recovery interval (`FEED_PRUNE_RECOVERY_INTERVAL_MS`, 6h), not the next tick. - -This is on purpose: an author can have unboundedly many followers, and trimming every one on the spot would either overrun the per-tick request budget (one delete per follower) or overrun D1's per-query CPU limit (one big delete over all of them, which can reset the shared Durable Object). `feed_items` is just a cache, so a follower sitting a little over cap for up to an interval does no harm. Deployments with fewer than `FEED_PRUNE_SWEEP_ACTORS` (500) distinct feed actors clean the whole table on every triggered tick, so they never see this lag at all. Pruning the touched actors directly (instead of the rolling cursor) would remove the lag but trade the bounded per-tick cost for cost proportional to fan-out size; see the issue tracker for that trade-off. - -## Deletes - -Deleting a target record removes its `feed_items` rows across all actors. Deleting a follow record currently does not retroactively prune the feed_items inserted during the original follow backfill — they age out via the global pruner instead. - -## XRPCs - -- `{namespace}.getFeed` — read - -That's it. Feeds are read-only over XRPC; writes to follow / target collections happen through `com.atproto.repo.putRecord` on the user's PDS as normal, and Jetstream ingestion drives the fan-out. - -## What's not here - -- No per-feed prune cap; the global pruner uses the largest `maxItems` across all feeds. -- The 100-record backfill on a new follow is hardcoded — not tunable per feed. -- No max-followers cap on target writes — a target record by a user with 1M followers means 1M `feed_items` inserts. For apps expecting that scale, partition feeds or rate-limit upstream. -- Feeds contain only public target records. diff --git a/docs/09-labels.md b/docs/09-labels.md deleted file mode 100644 index d34b702..0000000 --- a/docs/09-labels.md +++ /dev/null @@ -1,142 +0,0 @@ -# Labels - -Atproto-native moderation hydration. Subscribe to one or more labelers, index their labels, and attach them to records and profiles in your XRPC responses. Opt-in; zero cost if you don't enable it. - -## Mental model - -> A **label** is a `(src, uri, val)` triple authored by a labeler DID. A **labeler** is a regular atproto account that publishes signed annotations about other accounts and records via `com.atproto.label.subscribeLabels`. - -- One contrail deployment can subscribe to many labelers. -- The caller of your XRPC picks which subset to honor per request via the `atproto-accept-labelers` header (or `?labelers=` query param when headers are awkward — SSE/WS). -- Labels hydrate onto every `listRecords`, `getRecord`, `getProfile`, and `?profiles=true` response without changing your collection config. -- This module only consumes labels. Producing them — your appview emitting its own labels — is a separate question. See *Future work* below. - -## Enable - -```ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -const config: ContrailConfig = { - namespace: "com.example", - collections: { /* ... */ }, - labels: { - sources: [ - { did: "did:plc:ar7c4by46qjdydhdevvrndac" }, // bsky moderation - { did: "did:plc:newsmast" }, - ], - }, -}; -``` - -`initSchema` creates a `labels` table and a `labeler_cursors` table. Both live on the main DB; nothing per-collection. - -## Caller selection - -Per request, contrail picks accepted labelers in this order: - -1. `atproto-accept-labelers: did:plc:a, did:plc:b` — the spec's HTTP header. -2. `?labelers=did:plc:a,did:plc:b` — fallback for transports that can't set headers easily. -3. `config.labels.defaults` — operator policy. -4. Every entry in `config.labels.sources`. - -The list is intersected with what's actually configured (unknowns dropped — only labelers we've subscribed to have rows to hydrate from) and capped at `maxPerRequest` (default 20). Contrail echoes the applied set back via `atproto-content-labelers`. - -``` -GET /xrpc/com.example.event.listRecords - atproto-accept-labelers: did:plc:ar7c4by46qjdydhdevvrndac -``` - -→ - -```jsonc -// Response: atproto-content-labelers: did:plc:ar7c4by46qjdydhdevvrndac -{ - "records": [ - { - "uri": "at://did:plc:.../com.example.event/...", - "value": { /* ... */ }, - "labels": [ - { - "src": "did:plc:ar7c4by46qjdydhdevvrndac", - "uri": "at://did:plc:.../com.example.event/...", - "val": "spam", - "cts": "2026-04-25T00:00:00.000Z" - } - ] - } - ] -} -``` - -`labels` matches `com.atproto.label.defs#label` field-for-field — pass it straight to atproto SDK moderation helpers. - -### `defaults: []` - -Set defaults to an empty array if you want strict opt-in: callers that send no header / param see no labels at all. - -## Hydration semantics - -For each `(src, uri, val)` tuple visible to the caller, hydration picks the row with the highest `cts`. If that row has `neg=true`, the label is treated as retracted and dropped. Expired rows (`exp` past `now`) are filtered at the SQL level. CID-pinned labels apply only when the indexed record's CID matches. - -Account-level labels (subject = bare DID) hydrate onto profiles. They appear inside each `ProfileEntry.labels` of the `profiles` array on `?profiles=true` responses, and on `getProfile`. - -## Ingestion - -`com.atproto.label.subscribeLabels` is a per-labeler WebSocket firehose with a CBOR frame envelope. Contrail mirrors its existing Jetstream pipeline: - -| Mode | Function | When | -|---|---|---| -| Cron-driven | `contrail.ingestLabels()` | Cloudflare Workers — one drain per cron tick | -| Persistent | `contrail.runPersistentLabels()` | Node / long-lived servers — one socket per labeler, auto-reconnect | -| One-shot backfill | `pnpm contrail backfill --only labels [--remote]` | Local script, drains until each labeler reports caught up. (`pnpm contrail backfill` runs both records and labels.) | - -When `config.labels` is set, the bundled `createWorker` already calls `ingestLabels()` from `scheduled()` alongside `ingest()` — no boilerplate. - -```ts -// node / long-lived -const ac = new AbortController(); -await Promise.all([ - contrail.runPersistent({ signal: ac.signal }), - contrail.runPersistentLabels({ signal: ac.signal }), -]); -``` - -Per-labeler cursors live in `labeler_cursors` (`{did, cursor, endpoint, resolved_at}`). Endpoints are resolved from the DID doc's `service[id="#atproto_labeler"]` and cached for 6h. On `#info { name: "OutdatedCursor" }` frames, contrail resets the cursor to `0` so the next cycle re-backfills. - -### `backfill: false` - -Per source. Default: backfill from `cursor=0` on first sight. Set `false` to start at "now" — useful for very chatty labelers where you don't need history. - -```ts -labels: { - sources: [{ did: "did:plc:somenoisylabeler", backfill: false }], -} -``` - -## Storage - -```sql -CREATE TABLE labels ( - src TEXT NOT NULL, -- labeler DID - uri TEXT NOT NULL, -- subject: at://... or did:... - val TEXT NOT NULL, -- label value - cid TEXT, -- optional record-version pin - neg INTEGER NOT NULL DEFAULT 0, - exp INTEGER, -- expiry, unix sec - cts INTEGER NOT NULL, -- creation time, unix sec - sig BLOB, -- signature bytes (stored, not verified in v1) - PRIMARY KEY (src, uri, val, cts) -); -``` - -The PK includes `cts`, so a `neg=true` retraction is a *new row* that replaces the previous decision via the read-time collapse rule above — never an in-place mutation. This matches the spec, tolerates out-of-order delivery, and survives a labeler that flip-flops. - -## What's not here - -- **Signature verification.** `sig` is stored if the labeler supplies it, but contrail does not verify it in v1. Document as TODO; most appviews skip it. -- **Outbound `subscribeLabels`.** Contrail consumes labels but does not act as a labeler or republish them. -- **Label definitions / preferences UX.** Custom label names, blur behaviors, severity, and per-user preference state belong on the *client*, fetched directly from each labeler. Contrail intentionally stays out of this. - -## Design - -Follows the [atproto label spec](https://atproto.com/specs/label) literally. The wire format on responses matches `com.atproto.label.defs#label` so existing atproto SDKs can consume it directly. Storage is the data model normalized into rows; ingestion mirrors Jetstream both in code shape and in operator UX. diff --git a/docs/advanced/README.md b/docs/advanced/README.md new file mode 100644 index 0000000..49e9282 --- /dev/null +++ b/docs/advanced/README.md @@ -0,0 +1,9 @@ +# Advanced topics + +Start with the four-page happy path: [local AppView](../00-getting-started.md), [typed client](../01-client.md), [configuration and queries](../02-configure-and-query.md), and [Cloudflare deployment](../03-deploy-cloudflare.md). + +Optional features: + +- [Outbox](./outbox.md) +- [Feeds](./feeds.md) +- [Labels](./labels.md) diff --git a/docs/advanced/feeds.md b/docs/advanced/feeds.md new file mode 100644 index 0000000..6e57cf6 --- /dev/null +++ b/docs/advanced/feeds.md @@ -0,0 +1,41 @@ +# Feeds + +Feeds provide a per-user timeline of records authored by people that user follows. + +## Configure + +```ts +export default { + namespace: "com.example", + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + feeds: { + network: { + targets: [{ collection: "event", maxItems: 200 }], + }, + }, +}; +``` + +The default follow collection is `app.bsky.graph.follow`. Contrail adds it internally with `discover: false`; declare a different collection and set `follow` only when your app uses another follow record type. + +## Query + +```ts +const response = await contrail.get("com.example.getFeed", { + params: { + feed: "network", + actor: signedInDid, + collection: "community.lexicon.calendar.event", + profiles: true, + limit: 20, + }, +}); +``` + +`actor` means “whose feed,” not “record author.” It accepts a DID or handle. Filters, sorting, pagination, and hydration work like `listRecords` for the selected target collection. + +The first read starts a bounded backfill of that actor's follows, so its initial result may be partial. New target records are then fanned out during normal ingestion. Old items are pruned to each target's `maxItems` cap. + +Fan-out cost grows with an author's number of indexed followers. Feeds are therefore a projection for bounded application communities, not a replacement for a network-wide timeline service. diff --git a/docs/advanced/labels.md b/docs/advanced/labels.md new file mode 100644 index 0000000..a636329 --- /dev/null +++ b/docs/advanced/labels.md @@ -0,0 +1,37 @@ +# Labels + +Contrail can subscribe to AT Protocol labelers and attach their labels to records and profiles. + +## Configure + +```ts +export default { + namespace: "com.example", + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + labels: { + sources: [ + { did: "did:plc:ar7c4by46qjdydhdevvrndac" }, + ], + }, +}; +``` + +Normal `contrail backfill`, Worker cron ingestion, and `runPersistent()` include configured labelers automatically. + +## Select labelers + +A request chooses labelers with the standard header: + +```text +Atproto-Accept-Labelers: did:plc:ar7c4by46qjdydhdevvrndac +``` + +Use `?labelers=did:plc:...` when setting a header is inconvenient. Without either, Contrail uses `labels.defaults`, or all configured sources when defaults are omitted. Set `defaults: []` to require callers to opt in. + +Selected labels appear as `record.labels`. Account labels appear on hydrated profile entries. The response's `Atproto-Content-Labelers` header reports which configured labelers were applied. + +Contrail drops expired labels, applies CID-pinned labels only to the matching record version, and treats newer `neg: true` labels as retractions. + +Label signatures are stored but are not currently verified. Contrail consumes labels; it does not publish a label stream or provide moderation-preference UI. diff --git a/docs/advanced/outbox.md b/docs/advanced/outbox.md new file mode 100644 index 0000000..db069e1 --- /dev/null +++ b/docs/advanced/outbox.md @@ -0,0 +1,94 @@ +# Outbox + +> Experimental. Enable the outbox only on a fresh, empty Contrail database. + +The outbox delivers indexed record changes to external projections such as search indexes, webhooks, or caches. Contrail appends each change in the same database transaction as the canonical record and source cursor; destination failures never roll back ingestion. + +## Configure a consumer + +```ts +const config: ContrailConfig = { + namespace: "com.example", + collections: { + event: { collection: "community.lexicon.calendar.event" }, + }, + changes: { + consumers: { + search: { + collections: ["community.lexicon.calendar.event"], + phases: ["historical", "live"], + initial: "history", + }, + }, + }, +}; +``` + +Collections are full NSIDs, not config short names. Omitting `phases` includes both historical backfill and live ingestion. + +## Deliver from a Worker + +Add one handler for every configured consumer: + +```ts +type Env = { SEARCH_ENDPOINT: string }; + +export default createWorker(config, { + deliveries: { + search: async (batch, { env, signal }) => { + const response = await fetch(env.SEARCH_ENDPOINT, { + method: "POST", + signal, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + cursor: batch.cursor, + upserts: batch.currentRecords, + deletes: batch.absentUris, + }), + }); + if (!response.ok) throw new Error(`Search returned ${response.status}`); + }, + }, +}); +``` + +`createWorker` runs bounded delivery rounds after scheduled ingestion. Throwing retries the batch with backoff; returning successfully acknowledges it. + +Delivery is **at least once**. A destination may apply a batch before the acknowledgement fails, so handlers must be idempotent. Upsert and delete by record URI rather than incrementing counters. + +Claims coalesce repeated changes to the same URI. `currentRecords` contains the latest indexed values when the batch is delivered; `absentUris` contains records that are currently deleted. + +## Initial state + +| `initial` | Starts with | +|---|---| +| `history` | All retained historical and live changes | +| `future` | Changes written after the consumer is registered | +| `current` | A current-state snapshot, a fixed catch-up tail, then atomic destination activation | + +`current` is intended for building a candidate index without a read gap. It additionally requires matching `changeBootstraps` snapshot and activation handlers. + +For a long-lived Node process, run delivery beside ingestion: + +```ts +await Promise.all([ + contrail.runPersistent({ signal }), + contrail.runPersistentDeliveries({ + env, + deliveries: { search: deliverSearch }, + runtime: { signal }, + }), +]); +``` + +## Operate + +```bash +pnpm contrail changes status --remote +pnpm contrail changes retry search --remote +pnpm contrail changes prune --remote +``` + +Status reports each consumer's position, backlog, lease, and retry state. Pruning never passes the slowest durable consumer. `changes skip` is an explicit audited data-loss operation and should be reserved for recovery. + +Once enabled, ordinary startup fails closed if a consumer is removed or changed incompatibly. Adding a consumer is safe only when its collection/phase coverage was already retained; otherwise build a fresh database generation. diff --git a/docs/frameworks/sveltekit-cloudflare.md b/docs/frameworks/sveltekit-cloudflare.md deleted file mode 100644 index dd384dd..0000000 --- a/docs/frameworks/sveltekit-cloudflare.md +++ /dev/null @@ -1,213 +0,0 @@ -# SvelteKit + Cloudflare Workers - -How to add contrail to an existing SvelteKit project deployed on Cloudflare Workers (via `@sveltejs/adapter-cloudflare`). Gives you XRPC endpoints alongside your pages, Jetstream ingestion on cron, and a typed in-process client for server loaders. - -Assumes you already have a SvelteKit app with `@sveltejs/adapter-cloudflare` and a D1 binding. If you don't, [`apps/sveltekit-cloudflare-workers`](https://github.com/flo-bit/contrail/tree/main/apps/sveltekit-cloudflare-workers) is a complete starting point. - -## Install - -```bash -pnpm add @atmo-dev/contrail @atcute/client -``` - -## Project layout - -``` -src/ - lib/ - contrail.config.ts # your config — auto-detected by the CLI - contrail/ - index.ts # Contrail instance + ensureInit + server client - routes/ - xrpc/[...path]/+server.ts # mounts all contrail XRPC endpoints - api/cron/+server.ts # hit by the cron trigger (see below) -wrangler.jsonc -``` - -## 1. Declare the config - -```ts -// src/lib/contrail.config.ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -export const config: ContrailConfig = { - namespace: "com.example", - collections: { - event: { - collection: "community.lexicon.calendar.event", - queryable: { startsAt: { type: "range" } }, - searchable: ["name", "description"], - }, - }, -}; -``` - -## 2. The Contrail instance - -```ts -// src/lib/contrail/index.ts -import { Contrail } from "@atmo-dev/contrail"; -import { createHandler, createServerClient } from "@atmo-dev/contrail/server"; -import type { Client } from "@atcute/client"; -import { config } from "../contrail.config"; - -export const contrail = new Contrail(config); - -let initialized = false; -export async function ensureInit(db: D1Database) { - if (!initialized) { await contrail.init(db); initialized = true; } -} - -const handle = createHandler(contrail); - -/** In-process XRPC client for loaders and actions. */ -export function getServerClient(db: D1Database): Client { - return createServerClient(async (req) => { - await ensureInit(db); - return handle(req, db) as Promise; - }); -} -``` - -Why the lazy `ensureInit`: Workers cold-start many times; doing schema init on the first request keeps the boot path fast and means `contrail.init()` doesn't need top-level `await` (which the adapter doesn't love). - -## 3. Mount the XRPC routes - -One catch-all that forwards to contrail's handler: - -```ts -// src/routes/xrpc/[...path]/+server.ts -import type { RequestHandler } from "./$types"; -import { createHandler } from "@atmo-dev/contrail/server"; -import { contrail, ensureInit } from "$lib/contrail"; - -const handle = createHandler(contrail); - -async function h(req: Request, platform: App.Platform | undefined) { - const db = platform!.env.DB; - await ensureInit(db); - return handle(req, db) as Promise; -} - -export const GET: RequestHandler = ({ request, platform }) => h(request, platform); -export const POST: RequestHandler = ({ request, platform }) => h(request, platform); -``` - -Now every `com.example.*.listRecords` / `com.example.*.getRecord` / `com.example.notifyOfUpdate` / etc. is served under `/xrpc/...`. - -## 4. Using the typed client in loaders - -```ts -// src/routes/+page.server.ts -import { getServerClient } from "$lib/contrail"; -import type { PageServerLoad } from "./$types"; - -export const load: PageServerLoad = async ({ platform }) => { - const rpc = getServerClient(platform!.env.DB); - const res = await rpc.get("com.example.event.listRecords", { - params: { startsAtMin: "2026-01-01", limit: 20 }, - }); - return { events: res.ok ? res.data.records : [] }; -}; -``` - -`createServerClient` bypasses the network — the loader runs Contrail's public XRPC handler in-process. - -## 5. Cron ingest — the workaround - -SvelteKit's `@sveltejs/adapter-cloudflare` doesn't expose a `scheduled()` export on the generated worker ([issue #4841](https://github.com/sveltejs/kit/issues/4841)). The fix is an HTTP endpoint that does the ingest, plus a post-build patch on `_worker.js` that appends a `scheduled` handler calling it. The patch is what `contrail append-scheduled` does. - -**Endpoint:** - -```ts -// src/routes/api/cron/+server.ts -import type { RequestHandler } from "./$types"; -import { contrail, ensureInit } from "$lib/contrail"; - -export const POST: RequestHandler = async ({ request, platform }) => { - if (request.headers.get("X-Cron-Secret") !== platform!.env.CRON_SECRET) { - return new Response("Unauthorized", { status: 401 }); - } - const db = platform!.env.DB; - await ensureInit(db); - await contrail.ingest({}, db); - return new Response("OK"); -}; -``` - -**Wire `contrail append-scheduled` into your `build` script:** - -```jsonc -// package.json -"scripts": { - "build": "vite build && contrail append-scheduled" -} -``` - -`contrail append-scheduled` patches `.svelte-kit/cloudflare/_worker.js` to append a `scheduled()` export that POSTs to `/api/cron` with `env.CRON_SECRET`. Override with `--worker `, `--cron-path `, or `--secret-env ` if your project diverges. - -`CRON_SECRET` is any random string — generate one, set it as a secret with `wrangler secret put CRON_SECRET`. The cron handler self-auths with it so nobody external can trigger your ingest. - -## 6. Wrangler config - -```jsonc -// wrangler.jsonc -{ - "main": ".svelte-kit/cloudflare/_worker.js", - "compatibility_date": "2025-12-25", - "compatibility_flags": ["nodejs_compat_v2"], - "assets": { "binding": "ASSETS", "directory": ".svelte-kit/cloudflare" }, - "d1_databases": [ - { "binding": "DB", "database_name": "yourapp", "database_id": "..." } - ], - "triggers": { "crons": ["*/1 * * * *"] } -} -``` - -Type the D1 binding in `src/app.d.ts`: - -```ts -declare global { - namespace App { - interface Platform { - env: { - DB: D1Database; - CRON_SECRET: string; - // ...other bindings - }; - } - } -} -``` - -## 7. Deploy + backfill - -```bash -pnpm wrangler d1 create yourapp # copy the id into wrangler.jsonc -pnpm build && pnpm wrangler deploy -pnpm wrangler secret put CRON_SECRET # paste any random string -pnpm contrail backfill --remote # one-time historical backfill -``` - -From now on: - -- Pages and XRPC endpoints are served under your domain. -- The cron fires every minute, hitting `/api/cron`, which runs `contrail.ingest()`. -- Loaders that need live data use `getServerClient()` for zero-overhead typed calls. -- After a short outage, ingestion resumes from its saved cursor. If source history has expired, rebuild into a fresh database with `pnpm backfill:remote`. - -## Where to go next - -- [Indexing](../01-indexing.md) — config options, adapter choices -- [Querying](../02-querying.md) — filters, sorts, hydration, search -- [Feeds](../04-feeds.md) — personalized timelines via follow + target collections -- [Labels](../09-labels.md) — moderation label hydration - -Use Atcute directly for Lexicon pulling, validation, and TypeScript generation. - -## Common gotchas - -- **Top-level await in `$lib/contrail/index.ts`** will fail to bundle — use the lazy `ensureInit` pattern above. -- **`ensureInit` is per-isolate, not global.** Cloudflare cold-starts spin new isolates; each one pays one init call on its first request. `contrail.init()` is idempotent so this is safe, just not instant. -- **SvelteKit's `adapter-cloudflare` regenerates `_worker.js` on every build**, so `contrail append-scheduled` has to run *after* `vite build`. Don't try to put it in `prebuild`. -- **`D1Database` type in platform env** needs `@cloudflare/workers-types` in `devDependencies` and `types` in your tsconfig. diff --git a/docs/public-services/api-atmo-rsvp.md b/docs/public-services/api-atmo-rsvp.md deleted file mode 100644 index cfb2345..0000000 --- a/docs/public-services/api-atmo-rsvp.md +++ /dev/null @@ -1,246 +0,0 @@ -# Example: api.atmo.rsvp - -[`https://api.atmo.rsvp`](https://api.atmo.rsvp) is a public Contrail read-through service for AT Protocol calendar events and RSVPs. It demonstrates anonymous collection queries, profile hydration, a personalized network feed, authenticated update notifications, verified remote discovery, and an immutable D1 deployment generation. - -## Discovery - -```text -https://api.atmo.rsvp/.well-known/contrail -https://api.atmo.rsvp/.well-known/did.json -https://api.atmo.rsvp/lexicons -https://api.atmo.rsvp/status -``` - -The XRPC namespace is DNS-authoritative: - -```text -rsvp.atmo.* -``` - -The base service DID and exact service-auth audience are: - -```text -service DID: did:web:api.atmo.rsvp -audience: did:web:api.atmo.rsvp#contrail -``` - -The fragmented service reference is the OAuth and JWT audience. The base DID identifies the DID document published by the API. The API does not use either value to sign user records and does not act as a PDS. - -## Anonymous methods - -```text -rsvp.atmo.getCursor -rsvp.atmo.getProfile -rsvp.atmo.event.getRecord -rsvp.atmo.event.listRecords -rsvp.atmo.rsvp.getRecord -rsvp.atmo.rsvp.listRecords -``` - -Event queries support equality and date-range filtering, full-text search over names and descriptions, stable keyset pagination, RSVP relation counts, RSVP hydration, and actor profile hydration. - -RSVP queries support status, subject URI, and creation-time filtering. They can hydrate the referenced event and actor profiles. - -`getProfile` resolves an actor and reads their indexed `app.bsky.actor.profile` record. Missing public profile data may be fetched from that actor's PDS as part of the read-through request. - -## Protected methods - -Discovery lists these separately under AT Protocol service auth: - -```text -rsvp.atmo.getFeed -rsvp.atmo.notifyOfUpdate -``` - -The module generated by `contrail connect` exposes the required OAuth permission: - -```ts -import { contrail } from "./contrail/index.js"; - -export const scopes = ["atproto", contrail.scope]; -// contrail.scope is: -// rpc?aud=did:web:api.atmo.rsvp%23contrail&lxm=rsvp.atmo.getFeed&lxm=rsvp.atmo.notifyOfUpdate -``` - -After login, derive one client from the user's existing authenticated AT Protocol client: - -```ts -const client = contrail.authenticated(authenticatedClient, { - onNotificationError(error, { uris }) { - console.warn("Contrail notification failed", uris, error); - }, -}); -``` - -Advertised service methods route to Contrail; other methods route to the PDS. Protected methods automatically obtain and cache exact method-bound tokens. - -Missing, expired, wrong-audience, wrong-method, and invalid-signature tokens receive `401` with a `WWW-Authenticate` challenge. - -## Personalized network feed - -The configured feed is: - -```text -feed=network -``` - -It contains recent events and RSVPs authored by actors followed by the signed-in user. Per-actor projection caps are: - -| Collection | Maximum retained items | -|---|---:| -| `community.lexicon.calendar.event` | 100 | -| `community.lexicon.calendar.rsvp` | 250 | - -A typed query looks like: - -```ts -const response = await client.get( - "rsvp.atmo.getFeed", - { - params: { - feed: "network", - actor: signedInDid, - collection: "community.lexicon.calendar.event", - profiles: true, - limit: 20, - }, - }, -); -``` - -The requested actor must resolve to the token issuer. Service auth therefore prevents one authenticated account from creating or refreshing arbitrary personalized feed projections for other actors. - -The underlying `app.bsky.graph.follow` records are internal. The service does not advertise raw follow collection methods. It indexes follows authored by known actors only when the follow subject is already in the service's acquisition scope. Constellation enrichment helps connect newly observed calendar authors to existing in-scope followers. - -The feed endpoint may start a bounded background follow backfill the first time an actor requests their feed. Until that finishes, an initial response may be empty or partial. The operation remains a read-through cache fill rather than a network-wide social graph crawl. - -## Immediate update notification - -Successful event and RSVP writes through the combined client automatically notify Contrail: - -```ts -const response = await client.post("com.atproto.repo.createRecord", { - input: { - repo: signedInDid, - collection: "community.lexicon.calendar.event", - record: event, - }, -}); -``` - -The original PDS response is returned unchanged. Notification failures are nonfatal and reported through `onNotificationError`. The protected `rsvp.atmo.notifyOfUpdate` procedure remains available for explicit batches or records written elsewhere. - -The endpoint enforces all of the following: - -- at most 25 URIs per request; -- every URI is a canonical record AT URI; -- every URI belongs to the service-token issuer; -- the collection is tracked by this deployment; -- the record body and CID are fetched from the issuer's current PDS; and -- only an explicit XRPC `RecordNotFound` response is treated as deletion. - -Transient DNS, identity, network, timeout, PDS, or malformed-response failures preserve existing indexed state and are returned as bounded per-record errors to the authenticated caller. - -## Profiles - -The deployment indexes: - -```text -app.bsky.actor.profile -``` - -but keeps the underlying profile collection methods internal. Profiles are exposed through: - -- `rsvp.atmo.getProfile`; -- `profiles=true` on event and RSVP reads; and -- `profiles=true` on network feed reads. - -This avoids a redundant raw profile-record API while still providing typed display names, handles, avatars, and profile values alongside calendar data. - -## Acquisition scope - -Relay discovery starts from: - -```text -community.lexicon.calendar.event -community.lexicon.calendar.rsvp -``` - -Profiles and follows are dependent collections. They do not independently discover every Bluesky repository. - -The service is intentionally a shared, possibly incomplete cache. Unavailable identities and PDSes stay visibly pending, retrying, or failed instead of being silently marked complete. Unknown dependent subjects are scope exclusions and do not create tombstones. - -## Runtime validation policy - -This deployment publishes verified API and record Lexicons for discovery and TypeScript generation, but does not enable Contrail's optional runtime record/CID validation during ingestion. - -That distinction is deliberate. A matched benchmark using the current calendar Lexicons rejected 12,291 historical records, reducing indexed events from roughly 14,600 to 4,800 and RSVPs from roughly 6,300 to 3,800. Those records include historical shapes that predate the current published definitions, so enabling latest-schema validation would silently discard most of the useful archive. - -Under the compatibility policy: - -- provider and consumer contracts are still canonical and digest-verified; -- generated Atcute types describe the current expected response values, but are not a runtime guarantee for every historical record; -- startup still rejects an inconsistent advertised API; -- records and CIDs are fetched from authoritative sources rather than accepted from callers; but -- ingestion does not reject records based on runtime Lexicon or canonical-CID checks. - -A future strict deployment needs version-aware historical schemas or an explicitly looser response value, rather than pretending the compatibility loss does not exist. - -## Ordered source position - -The primary ordered source is one pinned Jetstream endpoint with an operator-owned continuity epoch: - -```json -{ - "source": "jetstream", - "epoch": "api-atmo-rsvp-primary-2026-08" -} -``` - -`rsvp.atmo.getCursor` returns the currently committed opaque position. Consumers compare the complete source, epoch, and cursor for equality only. A source or epoch change requires a full refetch. - -Backfills do not send historical notifications. Jetstream projection advances the serving position atomically with accepted live mutations. - -## Production generation - -The current expanded generation was built in native SQLite before activation. Its initial canonical projection contained approximately: - -| Collection | Records | -|---|---:| -| Events | 14,751 | -| RSVPs | 6,299 | -| Profiles | 1,408 | -| Scoped follows | 65,798 | - -The exact live totals change as Jetstream and read-through acquisition continue. - -The provisioning process: - -1. captured a replay boundary before relay discovery; -2. ran the resumable native-SQLite backfill; -3. retained 43 unavailable accounts as explicit scheduled retries; -4. replayed Jetstream to the present; -5. imported canonical tables into a fresh D1 database; -6. rebuilt FTS and materialized RSVP counts; -7. verified all visible rows against durable record versions; -8. exercised discovery, profiles, search, CORS, and protected-route rejection through a candidate Worker; and -9. activated the new Worker and D1 binding together. - -The previous D1 generation remains separate for rollback. No percentage traffic split is used between databases with independent serving positions. - -## Connect a consumer - -```bash -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp -``` - -The provider lock records: - -- the HTTPS endpoint; -- `rsvp.atmo` namespace; -- anonymous methods; -- protected methods and their audience; -- the content-addressed Lexicon digest; and -- the provider-owned Lexicon directory. - -See [Using a public Contrail service](./using.md) for a framework-neutral consumer walkthrough. diff --git a/docs/public-services/creating.md b/docs/public-services/creating.md deleted file mode 100644 index a520a0c..0000000 --- a/docs/public-services/creating.md +++ /dev/null @@ -1,311 +0,0 @@ -# Creating a public Contrail service - -A public Contrail service lets independent applications query one Contrail AppView from a stable HTTPS origin. The provider chooses the indexed collections, projections, query methods, and authentication policy. Consumers discover that API surface, verify its Lexicons, generate local TypeScript types, and make ordinary XRPC requests. - -Public service mode does not turn Contrail into a PDS. Records remain in their authors' repositories, and applications still authenticate users and publish writes through those users' PDSes. - -## Define the index - -Start with a normal Contrail configuration: - -```ts -// src/contrail.config.ts -import type { ContrailConfig } from "@atmo-dev/contrail"; - -export const config: ContrailConfig = { - namespace: "events.example", - orderedSource: { - source: "jetstream", - epoch: "primary-2026-08", - }, - collections: { - event: { - collection: "community.lexicon.calendar.event", - queryable: { - mode: {}, - startsAt: { type: "range" }, - }, - searchable: ["name", "description"], - }, - }, -}; -``` - -The namespace becomes the prefix of generated methods such as: - -```text -events.example.getCursor -events.example.event.getRecord -events.example.event.listRecords -``` - -Use one stable `orderedSource.epoch` for one continuity history. Change the epoch when the Jetstream endpoint set, retention assumptions, or cursor meaning changes. Consumers treat a source or epoch change as a full-refetch boundary. - -## Generate the public Lexicons - -Add Atcute's generator configuration: - -```js -// lex.config.js -import { defineLexiconConfig } from "@atcute/lex-cli"; - -export default defineLexiconConfig({ - generate: { - files: [ - "lexicons/custom/**/*.json", - "lexicons/pulled/**/*.json", - "lexicons/generated/**/*.json", - ], - outdir: "src/lexicon-types/", - }, -}); -``` - -Generate the provider API, pull referenced record Lexicons, and generate TypeScript types: - -```bash -pnpm contrail lexicons all --public -``` - -Check generated drift in CI: - -```bash -pnpm contrail lexicons check --public -``` - -The public surface includes anonymous queries plus explicitly configured service-auth queries and procedures. Private full-surface procedures are not included merely because they exist in application code. - -## Publish discovery from a Worker - -Pass the generated documents and canonical HTTPS origin to `createWorker`: - -```ts -// src/worker.ts -import { createWorker } from "@atmo-dev/contrail/worker"; -import { lexicons } from "../lexicons/generated"; -import { config } from "./contrail.config"; - -export default createWorker(config, { - lexicons, - publicService: { - endpoint: "https://api.example.com", - }, -}); -``` - -This publishes: - -```text -GET /.well-known/contrail -GET /lexicons -GET /lexicons/ -GET /status -``` - -The version-2 discovery manifest contains the endpoint, namespace, methods, collections, service-auth declaration, and a content-addressed Lexicon bundle. It does not hash the complete method set. Startup still fails when advertised methods, capabilities, and bundled Lexicons disagree, and the immutable Lexicon URL retains its digest. - -The public `/status` response contains aggregate readiness and freshness information. It omits DIDs, record bodies, source cursors, raw upstream errors, and other private operational details. - -## Anonymous read-through methods - -Collection queries, `getCursor`, profiles, feeds, and authored custom queries are anonymous unless explicitly protected. An anonymous query may still improve the shared cache by: - -- resolving an actor; -- fetching a missing public record from its PDS; -- populating profile or feed projections; or -- running a trusted custom query handler. - -This is read-through acquisition, not a caller-controlled write API. Only configured collections and trusted provider code can affect the projection. - -## Protecting feeds and notifications with service auth - -AT Protocol service auth lets any suitably authorized AT Protocol client call selected methods without distributing a shared application secret. - -```ts -export const config: ContrailConfig = { - namespace: "events.example", - notify: true, - serviceAuth: { - audience: "did:web:api.example.com#contrail", - methods: ["getFeed", "notifyOfUpdate"], - }, - collections, - feeds: { - network: { - targets: [{ collection: "event", maxItems: 100 }], - }, - }, -}; -``` - -The provider verifies: - -- the JWT signature against the issuer DID's `#atproto` key; -- the exact fragmented service audience; -- the token's exact `lxm` method claim; -- expiration and maximum token age; and -- the route-specific ownership rule. - -For protected feeds, the requested actor must resolve to the token issuer. For `notifyOfUpdate`, every submitted AT URI must belong to the token issuer. Notify still fetches the current authoritative record from that issuer's PDS; callers never submit a record body for Contrail to trust. - -The default PLC/`did:web` resolver keeps a bounded five-minute in-process cache and deduplicates concurrent lookups. Signature failure forces an uncached refresh so key rotation does not remain hidden behind a stale entry. Deployments can still provide their own resolver policy. - -The authenticated methods are listed separately from anonymous methods in discovery. Their query or procedure Lexicons remain in the provider bundle, so consumers still get generated types. - -## Start from owned source - -A consumer developed alongside the provider can generate its initial API surface before any deployment exists: - -```bash -pnpx @atmo-dev/contrail connect ./src/contrail.config.ts -# or discover the standard config beneath another project directory -pnpx @atmo-dev/contrail connect ../api -``` - -This compiles the config directly, writes generated Lexicons and types, and exports local/target client factories. It does not create or modify `contrail.lock.json`. Run the service separately: - -```bash -pnpx @atmo-dev/contrail dev --config ../api/src/contrail.config.ts -``` - -After deploying, `contrail connect https://api.example.com` creates the version-2 provider lock and makes that deployment the generated default target. Future config-source connections can refresh local API types without changing the production lock. - -### OAuth permission versus token binding - -Contrail generates one least-privilege OAuth permission containing the exact fragmented audience and one sorted `lxm` parameter per protected method: - -```text -rpc?aud=did:web:api.example.com%23contrail&lxm=events.example.getFeed&lxm=events.example.notifyOfUpdate -``` - -Current granular OAuth RPC syntax requires the absolute DID service reference; a plain DID may be silently removed from an authorization request. The `%23` is the encoded `#` delimiter. Wildcard `lxm=*` permissions are not generated because they are broader than necessary and can trigger misleading consent descriptions. - -Each call to `com.atproto.server.getServiceAuth` still passes the specific method NSID as `lxm` and the decoded `did:web:api.example.com#contrail` audience, producing a short-lived method-bound token. - -For example, a feed token uses: - -```text -lxm=events.example.getFeed -``` - -and cannot be reused for: - -```text -lxm=events.example.notifyOfUpdate -``` - -### Service DID and audience - -For this configuration, the identities are distinct: - -```text -base service DID: did:web:api.example.com -exact audience: did:web:api.example.com#contrail -public endpoint: https://api.example.com -``` - -Contrail publishes the base DID document at: - -```text -https://api.example.com/.well-known/did.json -``` - -The document `id` is the base DID and its Contrail service entry `id` is the exact fragmented audience. Startup fails if an automatically hosted `did:web` audience resolves to a different DID-document URL. A `did:plc` audience remains externally managed and does not create a local DID route. - -The exact service reference—not the base DID—is the JWT audience passed to `getServiceAuth` and verified by Contrail. The AppView does not need a signing key merely to receive and verify user-issued service tokens. - -## Profiles and internal follow projections - -Profiles can be enabled without exposing raw profile collection methods: - -```ts -profiles: ["app.bsky.actor.profile"], -collections: { - event, - profile: { - collection: "app.bsky.actor.profile", - discover: false, - methods: [], - }, -} -``` - -Likewise, a follow collection can remain an internal feed input: - -```ts -follow: { - collection: "app.bsky.graph.follow", - discover: false, - subjectField: "subject", - methods: [], -} -``` - -`discover: false` prevents network-wide relay discovery for dependent collections. `subjectField: "subject"` excludes follows whose subject is outside the known acquisition scope. Those exclusions do not create tombstones. - -Profiles can then appear through `getProfile` and `profiles=true` hydration, while follows power `getFeed` without creating a public social-graph directory. - -## Runtime validation is explicit per collection - -The deployment already ships one reviewed generated Lexicon bundle. Select which collection policies use it directly: - -```ts -collections: { - event: { - collection: "community.lexicon.calendar.event", - validate: true, - }, - legacy: { - collection: "community.example.legacy", - // Omitted or false means no runtime validation. - }, -}, -validation: { - strict: true, - verifyCid: true, -} -``` - -`createWorker(config, { lexicons })` binds the exact deployment bundle to opted-in collections; no second validation-specific array is needed. Validation applies across every acquisition source and startup fails if an opted-in record schema or transitive reference is absent. Collections without `validate: true` retain compatibility behavior. - -## CORS - -Public services allow browser requests and explicitly permit the `Authorization`, `Content-Type`, and `Atproto-Accept-Labelers` headers. Authentication failures expose `WWW-Authenticate` so browser clients can distinguish missing, expired, wrong-audience, and wrong-method tokens. - -Never place a reusable application secret in browser code. AT Protocol service tokens are short-lived and minted for the authenticated user's DID. - -## Provisioning and activation - -Local development can use the normal local backfill command: - -```bash -pnpm contrail backfill -``` - -For a substantial D1 production deployment, do not run a long bulk load through Wrangler's remote development proxy. Prefer a fresh generation: - -1. capture the ordered-source replay boundary; -2. build canonical state in native SQLite; -3. leave unavailable accounts visibly pending, retrying, or failed; -4. catch up through the ordered source; -5. import canonical tables into a fresh D1 database; -6. rebuild FTS, relation counts, and other derived projections; -7. verify record/version consistency, status, discovery, and representative queries; -8. test the candidate through a non-production Worker; and -9. activate the matching Worker and D1 binding together. - -Keep the previous D1 generation available for rollback. Do not split percentage traffic between independent databases with different serving positions. - -## Provider checklist - -Before announcing an origin: - -- run public Lexicon drift checking and TypeScript typechecking; -- verify the manifest and Lexicon digests differ and both recompute correctly; -- verify every advertised method has a matching query or procedure Lexicon; -- verify protected methods reject missing, wrong-audience, and wrong-`lxm` tokens; -- verify feed actors and notify URIs are bound to the token issuer; -- verify browser CORS preflight with `Authorization`; -- verify `/status` contains no sensitive operational detail; -- verify `getCursor` reports the committed ordered-source position; and -- connect and compile an independent consumer project. diff --git a/docs/public-services/using.md b/docs/public-services/using.md deleted file mode 100644 index 69903ef..0000000 --- a/docs/public-services/using.md +++ /dev/null @@ -1,164 +0,0 @@ -# Using a public Contrail service - -A public Contrail service is a typed, read-through API over public AT Protocol records. This guide uses `https://api.atmo.rsvp`; replace it with the provider you want to use. - -## Install and connect - -```bash -pnpm add @atcute/client @atcute/lexicons @atmo-dev/contrail -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp -``` - -`connect` validates the provider description and content-addressed Lexicon bundle, writes a version-2 `contrail.lock.json`, and generates: - -```text -lex.config.js -src/contrail/ - index.ts - lexicons/ - types/ -``` - -The generated configuration contains everything needed to identify the service and generate its types: - -```js -export default { - contrail: { - endpoint: "https://api.atmo.rsvp", - serviceDid: "did:web:api.atmo.rsvp", - serviceAudience: "did:web:api.atmo.rsvp#contrail", - scope: - "rpc?aud=did:web:api.atmo.rsvp%23contrail&lxm=rsvp.atmo.getFeed&lxm=rsvp.atmo.notifyOfUpdate", - protectedMethods: [ - "rsvp.atmo.getFeed", - "rsvp.atmo.notifyOfUpdate", - ], - collections: [ - "app.bsky.actor.profile", - "app.bsky.graph.follow", - "community.lexicon.calendar.event", - "community.lexicon.calendar.rsvp", - ], - }, - generate: { - files: ["src/contrail/lexicons/**/*.json"], - outdir: "src/contrail/types/", - }, -}; -``` - -For JavaScript output, choose a `.js` client path: - -```bash -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp \ - --client src/contrail/index.js -``` - -Commit `contrail.lock.json` and the generated files. - -When the application owns or can read the provider config, connect directly to that source before a deployment exists: - -```bash -pnpx @atmo-dev/contrail connect ../api/src/contrail.config.ts -pnpx @atmo-dev/contrail dev --config ../api/src/contrail.config.ts -``` - -The config connection compiles Lexicons and types without creating or modifying `contrail.lock.json`. `contrail dev` only runs the loopback service. The generated module exports `createLocalContrailClient()`: - -```ts -import { - contrail as productionContrail, - createLocalContrailClient, -} from "./contrail/index.js"; - -export const contrail = process.env.CONTRAIL_URL - ? createLocalContrailClient(process.env.CONTRAIL_URL) - : productionContrail; -``` - -The helper permits HTTP only for `localhost`, `127.0.0.1`, or `[::1]` and does not inherit a production service-auth audience. Local notification and configured protected methods are loopback-only operations with a null OAuth scope. A deployed service still requires HTTPS and its real service auth. - -## Query anonymous methods - -```ts -import { contrail } from "./contrail/index.js"; - -const response = await contrail.get("rsvp.atmo.event.listRecords", { - params: { - limit: 20, - sort: "startsAt", - order: "asc", - profiles: true, - }, -}); - -if (!response.ok) { - throw new Error(`Contrail query failed: ${response.status}`); -} - -for (const event of response.data.records) { - console.log(event.value.name, event.value.startsAt); -} -``` - -The generated Lexicons provide typed method names, parameters, and responses. - -## Use one authenticated client - -Add the provider's generated least-privilege scope to the application's OAuth scopes. It contains the encoded fragmented audience and only the protected methods advertised by this provider: - -```ts -import { contrail } from "./contrail/index.js"; - -export const scopes = ["atproto", contrail.scope]; -``` - -After login, combine Contrail with the existing authenticated AT Protocol client: - -```ts -const client = contrail.authenticated(authenticatedClient, { - onNotificationError(error, { uris }) { - console.warn("Contrail notification failed", uris, error); - }, -}); - -const response = await client.get("rsvp.atmo.getFeed", { - params: { - feed: "network", - actor: signedInDid, - collection: "community.lexicon.calendar.event", - profiles: true, - limit: 20, - }, -}); -``` - -The same client still handles ordinary PDS calls. Successful writes to connected collections automatically notify Contrail: - -```ts -await client.post("com.atproto.repo.createRecord", { - input: { - repo: signedInDid, - collection: "community.lexicon.calendar.event", - record: event, - }, -}); -``` - -Contrail returns the original PDS response. A notification failure is reported through `onNotificationError` but never turns a committed PDS write into a failed write. Handle-form `deleteRecord` inputs are resolved through the PDS to construct the canonical DID record URI before notification. The login session remains application-owned. - -## Update the connection - -Anonymous generated clients call the endpoint directly without fetching discovery first. Protected calls lazily discover service auth; transient discovery failures remain retryable, while endpoint, base-DID, exact-audience, scope, or protected-method mismatches fail closed. Adding anonymous provider methods does not interrupt methods already known by a generated client. Regenerate when application code wants new API surface or the protected contract changes: - -```bash -pnpx @atmo-dev/contrail connect https://api.atmo.rsvp --update -``` - -Review changes to `contrail.lock.json`, `lex.config.js`, and `src/contrail/`, then run the application's typecheck and tests. A change from the old plain-DID/wildcard permission requires OAuth reauthorization; an existing grant cannot mint tokens for the corrected fragmented audience. `--update` cannot repoint an existing lock or abandon its provider-owned Lexicon root; remove the existing connection deliberately before switching providers or output roots. Version-1 provider locks are intentionally unsupported after the clean manifest-v2 cut and must be removed before reconnecting. A version-2 lock written before exact audiences existed is reported separately and needs the same removal, reconnection, and OAuth reauthorization. - -## Completeness - -A public Contrail service is a shared, possibly incomplete read-through cache. Reads may fetch missing public records or profiles, but an empty result does not prove that no matching record exists on the network. - -Applications still authenticate users and publish records through their PDS. Contrail service auth only authorizes the protected methods advertised by that service. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 947a24d..a4a42f6 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -293,7 +293,6 @@ The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` Connect an independent consumer with `contrail connect `. The version-2 provider lock records the deployment and exact Lexicon bundle, but generated clients do not pin the provider's complete method set at runtime. Existing anonymous methods therefore continue working when a provider adds methods. A repeated connection to the same endpoint and provider-owned output root requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. Version-1 locks must be removed and reconnected. -See [Creating a public service](../../docs/public-services/creating.md), [Using a public service](../../docs/public-services/using.md), and [Example: api.atmo.rsvp](../../docs/public-services/api-atmo-rsvp.md) for complete provider and consumer walkthroughs. ## Runtime record validation diff --git a/packages/contrail/src/core/change-bootstrap.ts b/packages/contrail/src/core/change-bootstrap.ts index 2115bfc..4385bad 100644 --- a/packages/contrail/src/core/change-bootstrap.ts +++ b/packages/contrail/src/core/change-bootstrap.ts @@ -142,7 +142,8 @@ async function release( await db .prepare( `UPDATE change_consumers - SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND lease_owner = ?`, ) .bind(timestamp, consumerId, generation, owner) @@ -224,7 +225,8 @@ export async function claimCurrentSnapshotPage( bootstrap_scan_cursor = CASE WHEN bootstrap_state = 'pending' THEN NULL ELSE bootstrap_scan_cursor END, - lease_owner = ?, lease_expires_at = ?, updated_at = ? + lease_owner = ?, lease_expires_at = ?, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND initial_mode = 'current' AND bootstrap_state IN ('pending', 'scanning') AND generation_id = ( @@ -349,7 +351,8 @@ export async function claimCurrentSnapshotPage( .prepare( `UPDATE change_consumers SET bootstrap_scan_collection = ?, bootstrap_scan_cursor = NULL, - lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_state = 'scanning' AND lease_owner = ?`, ) @@ -372,7 +375,8 @@ export async function claimCurrentSnapshotPage( SELECT head_position FROM change_log_state WHERE id = 1 ), bootstrap_scan_collection = NULL, bootstrap_scan_cursor = NULL, - lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_state = 'scanning' AND lease_owner = ?`, ) @@ -393,7 +397,8 @@ export async function acknowledgeCurrentSnapshotPage( .prepare( `UPDATE change_consumers SET bootstrap_scan_cursor = ?, lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + lease_expires_at = NULL, lease_through_position = NULL, + attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? @@ -449,6 +454,7 @@ async function failBootstrapLease( .prepare( `UPDATE change_consumers SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, attempts = attempts + 1, next_attempt_at = ?, last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? @@ -544,7 +550,8 @@ export async function claimCurrentActivation( const row = await db .prepare( `UPDATE change_consumers - SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + SET lease_owner = ?, lease_expires_at = ?, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND initial_mode = 'current' AND bootstrap_state = 'activating' AND generation_id = ( @@ -596,7 +603,8 @@ export async function completeCurrentActivation( .prepare( `UPDATE change_consumers SET bootstrap_state = 'ready', lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + lease_expires_at = NULL, lease_through_position = NULL, + attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? diff --git a/packages/contrail/src/core/change-log.ts b/packages/contrail/src/core/change-log.ts index 1220a3a..937463e 100644 --- a/packages/contrail/src/core/change-log.ts +++ b/packages/contrail/src/core/change-log.ts @@ -163,6 +163,7 @@ export function buildChangeLogSchema( bootstrap_token TEXT, lease_owner TEXT, lease_expires_at ${bigint}, + lease_through_position ${bigint}, attempts INTEGER NOT NULL DEFAULT 0, next_attempt_at ${bigint}, last_success_at ${bigint}, diff --git a/packages/contrail/src/core/changes.ts b/packages/contrail/src/core/changes.ts index 0008639..a94038d 100644 --- a/packages/contrail/src/core/changes.ts +++ b/packages/contrail/src/core/changes.ts @@ -306,7 +306,8 @@ async function releaseLease( await db .prepare( `UPDATE change_consumers - SET lease_owner = NULL, lease_expires_at = NULL, updated_at = ? + SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ?`, ) @@ -427,7 +428,8 @@ async function claimChangeRange( const leased = await db .prepare( `UPDATE change_consumers - SET lease_owner = ?, lease_expires_at = ?, updated_at = ? + SET lease_owner = ?, lease_expires_at = ?, + lease_through_position = NULL, updated_at = ? WHERE consumer_id = ? AND bootstrap_state = '${requiredState}' AND generation_id = (SELECT generation_id FROM change_log_state WHERE id = 1) @@ -508,7 +510,8 @@ async function claimChangeRange( .prepare( `UPDATE change_consumers SET bootstrap_state = 'activating', lease_owner = NULL, - lease_expires_at = NULL, updated_at = ? + lease_expires_at = NULL, lease_through_position = NULL, + updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND bootstrap_state = 'catching-up' AND acknowledged_position = bootstrap_target_position @@ -585,6 +588,33 @@ async function claimChangeRange( } const through = String(selected.at(-1)!.position); + // Persist the exact selected upper bound before exposing the lease. Ack, + // renew, and fail all compare it so a mutated claim cannot skip delivery. + const bounded = await db + .prepare( + `UPDATE change_consumers + SET lease_through_position = ?, updated_at = ? + WHERE consumer_id = ? AND generation_id = ? + AND acknowledged_position = ? AND lease_owner = ? + AND lease_expires_at > ? + RETURNING consumer_id`, + ) + .bind( + through, + limits.now, + consumerId, + generation, + from, + owner, + limits.now, + ) + .first<{ consumer_id: string }>(); + if (!bounded) { + throw new ChangeLeaseLostError( + `Change claim for ${consumerId} expired while its range was being bound`, + ); + } + const rows = await db .prepare( `SELECT position, phase, change_count, encoded_bytes, changes_json @@ -759,12 +789,13 @@ export async function acknowledgeChanges( AND ? = bootstrap_target_position THEN 'activating' ELSE bootstrap_state END, lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, attempts = 0, next_attempt_at = NULL, last_success_at = ?, last_error_code = NULL, last_error_at = NULL, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? - AND lease_expires_at > ? + AND lease_expires_at > ? AND lease_through_position = ? AND ? > acknowledged_position AND ? <= ( SELECT head_position FROM change_log_state @@ -784,6 +815,7 @@ export async function acknowledgeChanges( now, claim.through, claim.through, + claim.through, claim.generation, ) .first<{ consumer_id: string }>(); @@ -814,7 +846,7 @@ export async function renewChangeClaim( SET lease_expires_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? - AND lease_expires_at > ? + AND lease_expires_at > ? AND lease_through_position = ? RETURNING consumer_id`, ) .bind( @@ -825,6 +857,7 @@ export async function renewChangeClaim( claim.from, claim.leaseOwner, now, + claim.through, ) .first<{ consumer_id: string }>(); if (!renewed) { @@ -860,10 +893,12 @@ export async function failChanges( .prepare( `UPDATE change_consumers SET lease_owner = NULL, lease_expires_at = NULL, + lease_through_position = NULL, attempts = attempts + 1, next_attempt_at = ?, last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? AND lease_owner = ? + AND lease_through_position = ? RETURNING attempts, next_attempt_at`, ) .bind( @@ -875,6 +910,7 @@ export async function failChanges( claim.generation, claim.from, claim.leaseOwner, + claim.through, ) .first<{ attempts: number | string; next_attempt_at: number | string | null }>(); if (!failed) { @@ -1117,7 +1153,8 @@ export async function skipChangeConsumer( .prepare( `UPDATE change_consumers SET acknowledged_position = ?, lease_owner = NULL, - lease_expires_at = NULL, attempts = 0, next_attempt_at = NULL, + lease_expires_at = NULL, lease_through_position = NULL, + attempts = 0, next_attempt_at = NULL, last_error_code = ?, last_error_at = ?, updated_at = ? WHERE consumer_id = ? AND generation_id = ? AND acknowledged_position = ? @@ -1230,25 +1267,63 @@ export async function pruneChanges( }; } + // created_at is captured before serialized position allocation, so age and + // position order may differ under overlapping projectors. Inspect only the + // bounded leading range and stop at the first age blocker; deleting every + // independently old row would create holes that retainedFloor cannot express. + const candidates = await db + .prepare( + `SELECT position, created_at FROM change_batches + WHERE generation_id = ? AND position > ? AND position <= ? + ORDER BY position LIMIT ?`, + ) + .bind(state.generation, state.retainedFloor, safeThrough, maxBatches) + .all<{ position: number | string; created_at: number | string }>(); + + let expected = BigInt(state.retainedFloor) + 1n; + let deleteThrough = state.retainedFloor; + for (const candidate of candidates.results) { + const position = BigInt(String(candidate.position)); + if (position !== expected) { + throw new ChangeHistoryGapError( + `Change log has a gap after retained floor ${state.retainedFloor}`, + ); + } + if ( + options.olderThan !== undefined && + Number(candidate.created_at) >= options.olderThan + ) { + break; + } + deleteThrough = String(candidate.position); + expected++; + } + + if (deleteThrough === state.retainedFloor) { + return { + pruned: 0, + retainedFloor: state.retainedFloor, + safeThrough, + done: true, + }; + } + const results = await db.batch([ db .prepare( `DELETE FROM change_batches - WHERE generation_id = ? AND position IN ( - SELECT position FROM change_batches - WHERE generation_id = ? AND position > ? AND position <= ? - AND (? IS NULL OR created_at < ?) - ORDER BY position LIMIT ? - )`, + WHERE generation_id = ? AND position > ? AND position <= ? + AND NOT EXISTS ( + SELECT 1 FROM change_consumers + WHERE generation_id = ? AND acknowledged_position < ? + )`, ) .bind( - state.generation, state.generation, state.retainedFloor, - safeThrough, - options.olderThan ?? null, - options.olderThan ?? null, - maxBatches, + deleteThrough, + state.generation, + deleteThrough, ), db .prepare( diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index d412723..8e89aea 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -789,6 +789,12 @@ export async function initSchema( "encoded_bytes", "INTEGER", ); + await addColumnIfNotExists( + db, + "change_consumers", + "lease_through_position", + dialect.bigintType, + ); await db .prepare( "UPDATE change_batches SET encoded_bytes = LENGTH(changes_json) WHERE encoded_bytes IS NULL", diff --git a/packages/contrail/tests/change-bootstrap.test.ts b/packages/contrail/tests/change-bootstrap.test.ts index c4733aa..73b2fea 100644 --- a/packages/contrail/tests/change-bootstrap.test.ts +++ b/packages/contrail/tests/change-bootstrap.test.ts @@ -356,4 +356,44 @@ describe("consumer-aware change pruning", () => { (await getChangesStatus(db)).consumers.find((item) => item.id === "late"), ).toMatchObject({ position: "5", backlogBatches: 0 }); }); + + it("keeps age-limited pruning to a contiguous retained prefix", async () => { + const db = createSqliteDatabase(":memory:"); + const resolved = config({ + keeper: { collections: [EVENT], initial: "history" }, + }); + await initSchema(db, resolved); + for (let position = 1; position <= 3; position++) { + await apply(db, resolved, event({ rkey: String(position), time: position })); + } + const claim = await claimChanges(db, "keeper", { now: 100 }); + await acknowledgeChanges(db, claim!, { now: 101 }); + + // Allocation order is serialized, but created_at is captured before that + // lock and can therefore be non-monotonic under overlapping projectors. + await db + .prepare( + `UPDATE change_batches SET created_at = CASE position + WHEN 1 THEN 100 WHEN 2 THEN 300 ELSE 100 END`, + ) + .run(); + + const pruned = await pruneChanges(db, { + maxBatches: 10, + olderThan: 200, + }); + expect(pruned).toEqual({ + pruned: 1, + retainedFloor: "1", + safeThrough: "3", + done: true, + }); + expect( + ( + await db + .prepare("SELECT position FROM change_batches ORDER BY position") + .all<{ position: number }>() + ).results.map((row) => row.position), + ).toEqual([2, 3]); + }); }); diff --git a/packages/contrail/tests/change-consumers.test.ts b/packages/contrail/tests/change-consumers.test.ts index f1d1528..3ec818a 100644 --- a/packages/contrail/tests/change-consumers.test.ts +++ b/packages/contrail/tests/change-consumers.test.ts @@ -343,16 +343,34 @@ describe("durable change consumers", () => { expect(claim?.changes).toHaveLength(2); }); - it("rejects forged generation cursors and never regresses a checkpoint", async () => { + it("rejects forged generation and range cursors without skipping delivery", async () => { const db = createSqliteDatabase(":memory:"); const config = readyEventConsumer(); await initSchema(db, config); - await apply(db, config, [mutation({ rkey: "one", sourceTime: 1 })]); - const claim = await claimChanges(db, "search", { now: 100 }); - const forged: ChangeClaim = { ...claim!, generation: crypto.randomUUID() }; + for (let position = 1; position <= 3; position++) { + await apply(db, config, [ + mutation({ rkey: String(position), sourceTime: position }), + ]); + } + const claim = await claimChanges(db, "search", { + now: 100, + maxBatches: 1, + }); + expect(claim?.through).toBe("1"); + + const wrongGeneration: ChangeClaim = { + ...claim!, + generation: crypto.randomUUID(), + }; await expect( - acknowledgeChanges(db, forged, { now: 101 }), + acknowledgeChanges(db, wrongGeneration, { now: 101 }), ).rejects.toBeInstanceOf(ChangeLeaseLostError); + + const beyondClaimedRange: ChangeClaim = { ...claim!, through: "3" }; + await expect( + acknowledgeChanges(db, beyondClaimedRange, { now: 101 }), + ).rejects.toBeInstanceOf(ChangeLeaseLostError); + await acknowledgeChanges(db, claim!, { now: 101 }); expect((await getChangesStatus(db)).consumers[0].position).toBe("1"); }); diff --git a/packages/contrail/tests/change-log.test.ts b/packages/contrail/tests/change-log.test.ts index 17f3303..851a83c 100644 --- a/packages/contrail/tests/change-log.test.ts +++ b/packages/contrail/tests/change-log.test.ts @@ -479,7 +479,7 @@ describe("transactional projection change log", () => { ); }); - it("upgrades retained pre-byte-count change batches without resetting consumers", async () => { + it("upgrades retained change-log columns without resetting consumers", async () => { const db = createSqliteDatabase(":memory:"); const resolved = loggedConfig(); await initSchema(db, resolved); @@ -487,6 +487,11 @@ describe("transactional projection change log", () => { await db .prepare("ALTER TABLE change_batches DROP COLUMN encoded_bytes") .run(); + await db + .prepare( + "ALTER TABLE change_consumers DROP COLUMN lease_through_position", + ) + .run(); await db .prepare( "UPDATE _contrail_meta SET value = 'old-change-schema' WHERE key = 'schema_fingerprint'", @@ -500,6 +505,13 @@ describe("transactional projection change log", () => { expect(row?.encoded_bytes).toBe( new TextEncoder().encode(row!.changes_json).byteLength, ); + expect( + ( + await db + .prepare("PRAGMA table_info(change_consumers)") + .all<{ name: string }>() + ).results.map((column) => column.name), + ).toContain("lease_through_position"); expect((await getChangeLogState(db))?.head).toBe("1"); });