diff --git a/.changeset/lexicon-tooling.md b/.changeset/lexicon-tooling.md new file mode 100644 index 0000000..4d56fd9 --- /dev/null +++ b/.changeset/lexicon-tooling.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Restore deterministic query-Lexicon generation from Contrail config, add drift checking, and orchestrate source pulling and TypeScript generation through Atcute. diff --git a/README.md b/README.md index 3e8ec5c..ad214d5 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,16 @@ GET /status The JSON status response reports live cursor lag, indexed records, known backfill progress, and mutually exclusive pending/retrying/failed account counts. Failed PDS work is retried automatically in small scheduled slices with backoff up to 48 hours. -For ordinary Lexicon parsing, validation, pulling, and TypeScript generation, use [Atcute](https://github.com/mary-ext/atcute) directly. Contrail no longer ships a separate Lexicon toolchain. +## Lexicons + +Generate query Lexicons from the Contrail config and detect checked-in drift: + +```bash +pnpm contrail lexicons generate +pnpm contrail lexicons check +``` + +Use `contrail lexicons all` to generate Contrail methods, pull referenced source Lexicons, and generate TypeScript types in one pass. The `pull` and `types` actions are also available separately. Contrail owns its config-specific query generation while delegating generic pulling and TypeScript generation to [Atcute](https://github.com/mary-ext/atcute). ## Other databases diff --git a/packages/contrail/README.md b/packages/contrail/README.md index dd3d69d..d3520a8 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -80,6 +80,17 @@ Jetstream generation replay requires an operator-owned continuity epoch and rete A separate control database can use `DatabaseGenerationRegistry` to store immutable `(code, definition, database, generation)` tuples. `activate(candidate, expectedActive)` switches one singleton pointer with compare-and-swap, retaining the previous ready tuple for rollback. There is intentionally no percentage traffic-split API; platform routing must resolve the one active tuple. +## Generate Lexicons + +Generate XRPC query Lexicons from the Contrail config and check them for drift: + +```bash +pnpm contrail lexicons generate +pnpm contrail lexicons check +``` + +`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only the collection methods intended for a public read surface. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. + ## Runtime record validation Pass the record Lexicons for every configured collection and their transitive references to enable shared strict validation and CID verification: diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 4898089..5ab9994 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -38,6 +38,10 @@ "./cli-config": { "types": "./dist/cli-config.d.ts", "import": "./dist/cli-config.js" + }, + "./lexicons": { + "types": "./dist/lexicons/index.d.ts", + "import": "./dist/lexicons/index.js" } }, "bin": { @@ -61,7 +65,7 @@ "clean": "rm -rf dist", "typecheck": "tsc --noEmit", "test": "vitest run", - "test:built": "node tests/built-sqlite.mjs", + "test:built": "node tests/built-sqlite.mjs && node tests/built-lexicons.mjs", "test:watch": "vitest" }, "dependencies": { @@ -71,6 +75,7 @@ "@atcute/client": "^5.1.1", "@atcute/identity-resolver": "^2.0.1", "@atcute/jetstream": "^2.0.2", + "@atcute/lex-cli": "^3.2.1", "@atcute/lexicon-doc": "3.0.2", "@atcute/lexicons": "^2.0.3", "@atcute/tid": "1.1.4", diff --git a/packages/contrail/src/cli.ts b/packages/contrail/src/cli.ts index 4dfffb6..892a83d 100644 --- a/packages/contrail/src/cli.ts +++ b/packages/contrail/src/cli.ts @@ -9,12 +9,14 @@ import { cac } from "cac"; import { registerBackfill } from "./cli/commands/backfill.js"; import { registerDev } from "./cli/commands/dev.js"; import { registerAppendScheduled } from "./cli/commands/append-scheduled.js"; +import { registerLexicons } from "./cli/commands/lexicons.js"; const cli = cac("contrail"); registerBackfill(cli); registerDev(cli); registerAppendScheduled(cli); +registerLexicons(cli); cli.help(); diff --git a/packages/contrail/src/cli/atcute.ts b/packages/contrail/src/cli/atcute.ts new file mode 100644 index 0000000..c4786ef --- /dev/null +++ b/packages/contrail/src/cli/atcute.ts @@ -0,0 +1,26 @@ +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; + +const require = createRequire(import.meta.url); + +function runAtcute(action: "pull" | "generate", root: string): void { + const entry = require.resolve("@atcute/lex-cli"); + const cli = join(dirname(entry), "..", "cli.mjs"); + const result = spawnSync(process.execPath, [cli, action], { + cwd: root, + stdio: "inherit", + }); + if (result.error) throw result.error; + if ((result.status ?? 1) !== 0) { + throw new Error(`Atcute lex-cli ${action} failed`); + } +} + +export function pullLexiconsWithAtcute(root: string): void { + runAtcute("pull", root); +} + +export function generateLexiconTypesWithAtcute(root: string): void { + runAtcute("generate", root); +} diff --git a/packages/contrail/src/cli/commands/lexicons.ts b/packages/contrail/src/cli/commands/lexicons.ts new file mode 100644 index 0000000..588fc0d --- /dev/null +++ b/packages/contrail/src/cli/commands/lexicons.ts @@ -0,0 +1,92 @@ +import { join, resolve } from "node:path"; +import type { CAC } from "cac"; +import { + checkLexicons, + generateLexicons, + type LexiconSurface, +} from "../../lexicons/generate.js"; +import { + generateLexiconTypesWithAtcute, + pullLexiconsWithAtcute, +} from "../atcute.js"; +import { resolveAndLoadConfig } from "../shared.js"; + +interface LexiconOptions { + config?: string; + root: string; + output: string; + public?: boolean; +} + +function surface(options: LexiconOptions): LexiconSurface { + return options.public ? "public" : "full"; +} + +async function generate(options: LexiconOptions) { + const config = await resolveAndLoadConfig(options); + return generateLexicons({ + config, + rootDir: resolve(options.root), + outputDir: resolve(options.root, options.output), + surface: surface(options), + }); +} + +export function registerLexicons(cli: CAC): void { + cli + .command( + "lexicons ", + "Generate Contrail Lexicons or delegate pulling/typegen to Atcute", + ) + .option("--config ", "Path to Contrail config file") + .option("--root ", "Project root", { default: process.cwd() }) + .option("--output ", "Generated output relative to root", { + default: join("lexicons", "generated"), + }) + .option("--public", "Generate only methods exposed by public read mode") + .action(async (action: string, options: LexiconOptions) => { + const root = resolve(options.root); + if (action === "generate") { + await generate(options); + return; + } + if (action === "check") { + const config = await resolveAndLoadConfig(options); + checkLexicons({ + config, + rootDir: root, + outputDir: resolve(root, options.output), + surface: surface(options), + }); + console.log("Contrail Lexicons are current."); + return; + } + if (action === "pull") { + pullLexiconsWithAtcute(root); + return; + } + if (action === "types") { + generateLexiconTypesWithAtcute(root); + return; + } + if (action === "all") { + let previous: string | undefined; + for (let pass = 0; pass < 5; pass++) { + const result = await generate(options); + const current = JSON.stringify(result.pullNsids); + pullLexiconsWithAtcute(root); + if (current === previous) break; + previous = current; + if (pass === 4) { + throw new Error("Lexicon reference discovery did not converge"); + } + } + await generate(options); + generateLexiconTypesWithAtcute(root); + return; + } + throw new Error( + `unknown lexicons action ${JSON.stringify(action)}; expected generate, check, pull, types, or all`, + ); + }); +} diff --git a/packages/contrail/src/core/router/hydrate.ts b/packages/contrail/src/core/router/hydrate.ts index 7763a5d..19a6857 100644 --- a/packages/contrail/src/core/router/hydrate.ts +++ b/packages/contrail/src/core/router/hydrate.ts @@ -93,9 +93,18 @@ export async function resolveHydrates( .filter((record) => record.did === matchedValue) .map((record) => record.uri) : [matchedValue]; - const groupValue = relation.groupBy + const rawGroupValue = relation.groupBy ? String(getNestedValue(value, relation.groupBy) ?? "other") : "_flat"; + const configuredGroup = relation.groups + ? Object.entries(relation.groups).find( + ([name, token]) => + name === rawGroupValue || token === rawGroupValue, + )?.[0] + : undefined; + const groupValue = relation.groupBy + ? (configuredGroup ?? (relation.groups ? "other" : rawGroupValue)) + : "_flat"; for (const parentUri of parentUris) { grouped[parentUri] ??= {}; diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts new file mode 100644 index 0000000..0804a4f --- /dev/null +++ b/packages/contrail/src/lexicons/generate.ts @@ -0,0 +1,985 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import type { + CollectionConfig, + ContrailConfig, + RelationConfig, +} from "../core/types.js"; +import { + getCollectionMethods, + normalizeFeedTarget, + resolveConfig, +} from "../core/types.js"; + +export type LexiconSurface = "full" | "public"; + +export interface GenerateLexiconsOptions { + config: ContrailConfig; + rootDir: string; + outputDir?: string; + sourceDirs?: string[]; + surface?: LexiconSurface; + writeAtcuteConfig?: boolean; + quiet?: boolean; +} + +export interface GenerateLexiconsResult { + generated: Record; + methods: string[]; + pullNsids: string[]; +} + +interface RelationDef { + name: string; + collection: string; + groupBy?: string; + groups: Record; + count: boolean; +} + +interface ReferenceDef { + name: string; + collection: string; +} + +function* walkJson(directory: string): Generator { + if (!existsSync(directory)) return; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) yield* walkJson(path); + else if (entry.isFile() && entry.name.endsWith(".json")) yield path; + } +} + +function fieldToParam(field: string): string { + return field.replace(/\.(\w)/g, (_, character: string) => + character.toUpperCase(), + ); +} + +function capitalize(value: string): string { + return value.charAt(0).toUpperCase() + value.slice(1); +} + +function tokenShortName(token: string): string { + const hash = token.indexOf("#"); + return hash === -1 ? token : token.slice(hash + 1); +} + +function readLexicon(path: string): any | null { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +function findLexicon(sourceDirs: string[], nsid: string): string | null { + const suffix = `${nsid.split(".").join("/")}.json`; + for (const directory of sourceDirs) { + const path = join(directory, suffix); + if (existsSync(path)) return path; + } + return null; +} + +function collectionReference( + sourceDirs: string[], + nsid: string, +): string | null { + const path = findLexicon(sourceDirs, nsid); + const document = path ? readLexicon(path) : null; + return document?.defs?.main ? `${nsid}#main` : null; +} + +function recordObjectSchema(sourceDirs: string[], nsid: string): any | null { + const path = findLexicon(sourceDirs, nsid); + const document = path ? readLexicon(path) : null; + const main = document?.defs?.main; + return main?.type === "record" && main.record ? main.record : null; +} + +function groupsForRelation( + sourceDirs: string[], + config: ContrailConfig, + relation: RelationConfig, +): Record { + if (relation.groups && Object.keys(relation.groups).length > 0) { + return Object.fromEntries(Object.entries(relation.groups).sort()); + } + if (!relation.groupBy) return {}; + const nsid = + config.collections[relation.collection]?.collection ?? relation.collection; + const path = findLexicon(sourceDirs, nsid); + const document = path ? readLexicon(path) : null; + const field = document?.defs?.main?.record?.properties?.[relation.groupBy]; + const values: unknown[] = Array.isArray(field?.knownValues) + ? field.knownValues + : []; + return Object.fromEntries( + values + .filter((value: unknown): value is string => typeof value === "string") + .map((value: string): [string, string] => [tokenShortName(value), value]) + .sort(([left], [right]) => left.localeCompare(right)), + ); +} + +function profileDefinitions(config: ContrailConfig, sourceDirs: string[]) { + const profiles = config.profiles ?? []; + if (profiles.length === 0) return {}; + const definitions: Record = {}; + const refs: string[] = []; + for (const profile of profiles) { + const collection = + typeof profile === "string" ? profile : profile.collection; + const schema = recordObjectSchema(sourceDirs, collection); + if (!schema) continue; + const name = collection + .split(".") + .map((part, index) => (index === 0 ? part : capitalize(part))) + .join(""); + definitions[name] = schema; + refs.push(`#${name}`); + } + const value = + refs.length === 1 + ? { type: "ref", ref: refs[0] } + : refs.length > 1 + ? { type: "union", refs } + : { type: "unknown" }; + return { + profileEntry: { + type: "object", + required: ["did"], + properties: { + did: { type: "string", format: "did" }, + handle: { type: "string" }, + uri: { type: "string", format: "at-uri" }, + cid: { type: "string", format: "cid" }, + value, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + }, + }, + ...definitions, + }; +} + +function recordDefinition( + sourceDirs: string[], + collection: string, + relations: RelationDef[], + references: ReferenceDef[], +) { + const properties: Record = { + uri: { type: "string", format: "at-uri" }, + cid: { type: "string", format: "cid" }, + value: collectionReference(sourceDirs, collection) + ? { type: "ref", ref: `${collection}#main` } + : { type: "unknown" }, + did: { type: "string", format: "did" }, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + time_us: { type: "integer" }, + }; + for (const relation of relations) { + if (relation.count) { + properties[`${relation.name}Count`] = { + type: "integer", + description: `Total ${relation.name} count`, + }; + for (const shortName of Object.keys(relation.groups)) { + properties[`${relation.name}${capitalize(shortName)}Count`] = { + type: "integer", + description: `${relation.name} count where ${relation.groupBy} = ${shortName}`, + }; + } + } + properties[relation.name] = + relation.groupBy && Object.keys(relation.groups).length > 0 + ? { type: "ref", ref: `#hydrate${capitalize(relation.name)}` } + : { + type: "array", + items: { + type: "ref", + ref: `#hydrate${capitalize(relation.name)}Record`, + }, + }; + } + for (const reference of references) { + properties[reference.name] = { + type: "ref", + ref: `#ref${capitalize(reference.name)}Record`, + }; + } + return { + type: "object", + required: ["uri", "cid", "value", "did", "collection", "rkey", "time_us"], + properties, + }; +} + +function relatedRecordDefinition(sourceDirs: string[], collection: string) { + const reference = collectionReference(sourceDirs, collection); + return { + type: "object", + required: ["uri", "value", "did", "collection", "rkey", "time_us"], + properties: { + uri: { type: "string", format: "at-uri" }, + cid: { type: "string", format: "cid" }, + value: reference ? { type: "ref", ref: reference } : { type: "unknown" }, + did: { type: "string", format: "did" }, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + time_us: { type: "integer" }, + }, + }; +} + +function hydrationDefinitions( + sourceDirs: string[], + relations: RelationDef[], + references: ReferenceDef[], +): Record { + const definitions: Record = {}; + for (const relation of relations) { + const recordName = `hydrate${capitalize(relation.name)}Record`; + definitions[recordName] = relatedRecordDefinition( + sourceDirs, + relation.collection, + ); + if (relation.groupBy && Object.keys(relation.groups).length > 0) { + definitions[`hydrate${capitalize(relation.name)}`] = { + type: "object", + properties: Object.fromEntries([ + ...Object.keys(relation.groups).map((name) => [ + name, + { type: "array", items: { type: "ref", ref: `#${recordName}` } }, + ]), + [ + "other", + { type: "array", items: { type: "ref", ref: `#${recordName}` } }, + ], + ]), + }; + } + } + for (const reference of references) { + definitions[`ref${capitalize(reference.name)}Record`] = + relatedRecordDefinition(sourceDirs, reference.collection); + } + return definitions; +} + +function relationDefinitions( + sourceDirs: string[], + config: ContrailConfig, + collection: CollectionConfig, +): RelationDef[] { + return Object.entries(collection.relations ?? {}) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, relation]) => ({ + name, + collection: + config.collections[relation.collection]?.collection ?? + relation.collection, + groupBy: relation.groupBy, + groups: groupsForRelation(sourceDirs, config, relation), + count: relation.count !== false, + })); +} + +function referenceDefinitions( + config: ContrailConfig, + collection: CollectionConfig, +): ReferenceDef[] { + return Object.entries(collection.references ?? {}) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, reference]) => ({ + name, + collection: + config.collections[reference.collection]?.collection ?? + reference.collection, + })); +} + +function listParameters( + config: ContrailConfig, + collection: CollectionConfig, + relations: RelationDef[], + references: ReferenceDef[], + surface: LexiconSurface, +) { + const properties: Record = { + limit: { type: "integer", minimum: 1, maximum: 200, default: 50 }, + cursor: { type: "string" }, + actor: { + type: "string", + format: "at-identifier", + description: + surface === "public" + ? "Filter by an indexed DID or cached handle" + : "Filter by DID or handle", + }, + }; + if ((config.profiles?.length ?? 0) > 0) { + properties.profiles = { + type: "boolean", + description: "Include indexed profile and identity information", + }; + } + if ( + Array.isArray(collection.searchable) && + collection.searchable.length > 0 + ) { + properties.search = { + type: "string", + description: `Full-text search across: ${collection.searchable.join(", ")}`, + }; + } + for (const [field, queryable] of Object.entries( + collection.queryable ?? {}, + ).sort(([left], [right]) => left.localeCompare(right))) { + const parameter = fieldToParam(field); + if (queryable.type === "range") { + properties[`${parameter}Min`] = { + type: "string", + description: `Minimum value for ${field}`, + }; + properties[`${parameter}Max`] = { + type: "string", + description: `Maximum value for ${field}`, + }; + } else { + properties[parameter] = { + type: "string", + description: `Filter by ${field}`, + }; + } + } + const sortable = Object.keys(collection.queryable ?? {}).map(fieldToParam); + for (const relation of relations) { + if (relation.count) { + properties[`${relation.name}CountMin`] = { + type: "integer", + description: `Minimum total ${relation.name} count`, + }; + sortable.push(`${relation.name}Count`); + for (const shortName of Object.keys(relation.groups)) { + const base = `${relation.name}${capitalize(shortName)}Count`; + properties[`${base}Min`] = { + type: "integer", + description: `Minimum ${relation.name} count where ${relation.groupBy} = ${shortName}`, + }; + sortable.push(base); + } + } + properties[`hydrate${capitalize(relation.name)}`] = { + type: "integer", + minimum: 1, + maximum: 50, + description: `Number of ${relation.name} records to embed`, + }; + } + for (const reference of references) { + properties[`hydrate${capitalize(reference.name)}`] = { + type: "boolean", + description: `Embed the referenced ${reference.name} record`, + }; + } + if (sortable.length > 0) { + properties.sort = { + type: "string", + knownValues: [...new Set(sortable)].sort(), + description: "Field to sort by (default: time_us)", + }; + properties.order = { + type: "string", + knownValues: ["asc", "desc"], + description: "Sort direction", + }; + } + return properties; +} + +function getParameters( + config: ContrailConfig, + relations: RelationDef[], + references: ReferenceDef[], +) { + const properties: Record = { + uri: { + type: "string", + format: "at-uri", + description: "AT URI of the record", + }, + }; + if ((config.profiles?.length ?? 0) > 0) { + properties.profiles = { + type: "boolean", + description: "Include indexed profile and identity information", + }; + } + for (const relation of relations) { + properties[`hydrate${capitalize(relation.name)}`] = { + type: "integer", + minimum: 1, + maximum: 50, + description: `Number of ${relation.name} records to embed`, + }; + } + for (const reference of references) { + properties[`hydrate${capitalize(reference.name)}`] = { + type: "boolean", + description: `Embed the referenced ${reference.name} record`, + }; + } + return properties; +} + +function collectReferencedNsids(path: string): string[] { + const document = readLexicon(path); + const values = new Set(); + const visit = (value: unknown, key?: string) => { + if (typeof value === "string" && (key === "ref" || key === "refs")) { + const nsid = value.split("#", 1)[0]; + if (nsid?.includes(".")) values.add(nsid); + return; + } + if (Array.isArray(value)) { + for (const child of value) + visit(child, key === "refs" ? "refs" : undefined); + } else if (value && typeof value === "object") { + for (const [childKey, child] of Object.entries(value)) + visit(child, childKey); + } + }; + visit(document); + return [...values]; +} + +function calculatePullNsids( + config: ContrailConfig, + sourceDirs: string[], +): string[] { + const values = new Set( + Object.values(config.collections) + .map((collection) => collection.collection) + .filter((value): value is string => typeof value === "string"), + ); + for (const directory of sourceDirs) { + for (const path of walkJson(directory)) { + for (const nsid of collectReferencedNsids(path)) values.add(nsid); + } + } + return [...values].sort(); +} + +function atcuteConfiguration(pullNsids: string[]): string { + return `import { defineLexiconConfig } from "@atcute/lex-cli";\n\nexport default defineLexiconConfig({\n generate: {\n files: [\n "lexicons/custom/**/*.json",\n "lexicons/pulled/**/*.json",\n "lexicons/generated/**/*.json",\n ],\n outdir: "src/lexicon-types/",\n },\n pull: {\n outdir: "lexicons/pulled/",\n clean: true,\n sources: [\n {\n type: "atproto",\n mode: "nsids",\n nsids: ${JSON.stringify(pullNsids, null, 2).replace(/^/gm, " ").trim()},\n },\n ],\n },\n});\n`; +} + +function writeAtcuteConfiguration(rootDir: string, pullNsids: string[]) { + writeFileSync(join(rootDir, "lex.config.js"), atcuteConfiguration(pullNsids)); +} + +function writeBundle( + outputDir: string, + generated: Record, + sourceDirs: string[], +) { + const paths = new Set(); + for (const nsid of Object.keys(generated)) { + paths.add(`./${nsid.split(".").join("/")}.json`); + } + for (const directory of sourceDirs) { + for (const path of walkJson(directory)) { + let rel = relative(outputDir, path); + if (!rel.startsWith(".")) rel = `./${rel}`; + paths.add(rel); + } + } + const sorted = [...paths].sort(); + const imports = sorted + .map((path, index) => `import _${index} from "${path}";`) + .join("\n"); + const values = sorted.map((_, index) => `_${index}`).join(", "); + writeFileSync( + join(outputDir, "index.ts"), + `// Auto-generated by @atmo-dev/contrail. Do not edit.\n` + + `// Regenerate with \`contrail lexicons generate\`.\n\n` + + `${imports}\n\nexport const lexicons: object[] = [${values}];\n`, + ); +} + +function feedLexicon( + config: ContrailConfig, + sourceDirs: string[], +): object | null { + if (!config.feeds || Object.keys(config.feeds).length === 0) return null; + const targets = [ + ...new Set( + Object.values(config.feeds).flatMap((feed) => + feed.targets.map((target) => normalizeFeedTarget(target).collection), + ), + ), + ].sort(); + const targetNsids = targets + .map((target) => config.collections[target]?.collection) + .filter((value): value is string => typeof value === "string"); + const parameters: Record = { + feed: { + type: "string", + knownValues: Object.keys(config.feeds).sort(), + }, + actor: { + type: "string", + format: "at-identifier", + description: "DID or handle whose feed should be queried", + }, + collection: { + type: "string", + knownValues: targetNsids, + }, + limit: { type: "integer", minimum: 1, maximum: 200, default: 50 }, + cursor: { type: "string" }, + }; + if ((config.profiles?.length ?? 0) > 0) { + parameters.profiles = { type: "boolean" }; + } + const recordDefinitions: Record = {}; + const recordRefs: string[] = []; + const hydrateDefinitions: Record = {}; + for (const target of targets) { + const collection = config.collections[target]; + if (!collection) continue; + const relations = relationDefinitions(sourceDirs, config, collection); + const references = referenceDefinitions(config, collection); + const targetParams = listParameters( + config, + collection, + relations, + references, + "full", + ); + for (const [name, value] of Object.entries(targetParams)) { + if (["limit", "cursor", "actor", "profiles"].includes(name)) continue; + if (name === "sort" && parameters.sort) { + parameters.sort.knownValues = [ + ...new Set([ + ...(parameters.sort.knownValues ?? []), + ...((value as any).knownValues ?? []), + ]), + ].sort(); + } else { + parameters[name] ??= value; + } + } + const definition = `feedRecord${capitalize(target.replace(/[^a-zA-Z0-9]/g, "_"))}`; + recordDefinitions[definition] = recordDefinition( + sourceDirs, + collection.collection ?? target, + relations, + references, + ); + recordRefs.push(`#${definition}`); + Object.assign( + hydrateDefinitions, + hydrationDefinitions(sourceDirs, relations, references), + ); + } + if (recordRefs.length === 0) { + throw new Error( + "configured feeds require at least one valid target collection", + ); + } + const recordItems = + recordRefs.length === 1 + ? { type: "ref", ref: recordRefs[0] } + : { type: "union", refs: recordRefs }; + const outputProperties: Record = { + records: { type: "array", items: recordItems }, + cursor: { type: "string" }, + }; + if ((config.profiles?.length ?? 0) > 0) { + outputProperties.profiles = { + type: "array", + items: { type: "ref", ref: "#profileEntry" }, + }; + } + return { + lexicon: 1, + id: `${config.namespace}.getFeed`, + defs: { + main: { + type: "query", + description: "Get a configured personalized feed", + parameters: { + type: "params", + required: ["feed", "actor"], + properties: parameters, + }, + output: { + encoding: "application/json", + schema: { + type: "object", + required: ["records"], + properties: outputProperties, + }, + }, + }, + ...recordDefinitions, + ...hydrateDefinitions, + ...profileDefinitions(config, sourceDirs), + }, + }; +} + +function customMethodIds(config: ContrailConfig): string[] { + const values: string[] = []; + for (const [alias, collection] of Object.entries(config.collections)) { + for (const name of new Set([ + ...Object.keys(collection.queries ?? {}), + ...Object.keys(collection.pipelineQueries ?? {}), + ])) { + values.push(`${config.namespace}.${alias}.${name}`); + } + } + return values.sort(); +} + +export function extractXrpcMethods( + documents: Record, +): string[] { + return Object.entries(documents) + .filter(([, document]) => { + const type = (document as any)?.defs?.main?.type; + return type === "query" || type === "procedure"; + }) + .map(([nsid]) => nsid) + .sort(); +} + +export function generateLexicons( + options: GenerateLexiconsOptions, +): GenerateLexiconsResult { + const config = resolveConfig(options.config); + const surface = options.surface ?? "full"; + const rootDir = resolve(options.rootDir); + const outputDir = options.outputDir + ? resolve(rootDir, options.outputDir) + : join(rootDir, "lexicons", "generated"); + const outputRelative = relative(rootDir, outputDir); + if ( + outputRelative === "" || + outputRelative === ".." || + outputRelative.startsWith( + `..${process.platform === "win32" ? "\\" : "/"}`, + ) || + isAbsolute(outputRelative) + ) { + throw new Error( + "generated Lexicon output must stay inside the project root", + ); + } + const sourceDirs = options.sourceDirs ?? [ + join(rootDir, "lexicons", "custom"), + join(rootDir, "lexicons", "pulled"), + ]; + const log = options.quiet ? () => {} : console.log; + const generated: Record = {}; + + for (const method of customMethodIds(config)) { + if (!findLexicon(sourceDirs, method)) { + throw new Error( + `custom query ${method} requires a matching Lexicon under lexicons/custom`, + ); + } + } + + rmSync(outputDir, { recursive: true, force: true }); + mkdirSync(outputDir, { recursive: true }); + + const emit = (nsid: string, document: object) => { + const path = join(outputDir, ...nsid.split(".")) + ".json"; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(document, null, 2)}\n`); + generated[nsid] = document; + log(` ${nsid}`); + }; + + if (surface === "full") { + emit(`${config.namespace}.getCursor`, { + lexicon: 1, + id: `${config.namespace}.getCursor`, + defs: { + main: { + type: "query", + description: "Get the current ingestion observation time", + output: { + encoding: "application/json", + schema: { + type: "object", + properties: { + time_us: { type: "integer" }, + date: { type: "string" }, + seconds_ago: { type: "integer" }, + }, + }, + }, + }, + }, + }); + emit(`${config.namespace}.getOverview`, { + lexicon: 1, + id: `${config.namespace}.getOverview`, + defs: { + main: { + type: "query", + description: "Get aggregate projection status", + output: { encoding: "application/json", schema: { type: "unknown" } }, + }, + }, + }); + if ((config.profiles?.length ?? 0) > 0) { + emit(`${config.namespace}.getProfile`, { + lexicon: 1, + id: `${config.namespace}.getProfile`, + defs: { + main: { + type: "query", + parameters: { + type: "params", + required: ["actor"], + properties: { + actor: { type: "string", format: "at-identifier" }, + }, + }, + output: { + encoding: "application/json", + schema: { + type: "object", + required: ["profiles"], + properties: { + profiles: { + type: "array", + items: { type: "ref", ref: "#profileEntry" }, + }, + }, + }, + }, + }, + ...profileDefinitions(config, sourceDirs), + }, + }); + } + if (config.notify) { + emit(`${config.namespace}.notifyOfUpdate`, { + lexicon: 1, + id: `${config.namespace}.notifyOfUpdate`, + defs: { + main: { + type: "procedure", + input: { + encoding: "application/json", + schema: { + type: "object", + properties: { + uri: { type: "string", format: "at-uri" }, + uris: { + type: "array", + items: { type: "string", format: "at-uri" }, + maxLength: 25, + }, + }, + }, + }, + output: { + encoding: "application/json", + schema: { type: "unknown" }, + }, + }, + }, + }); + } + const feed = feedLexicon(config, sourceDirs); + if (feed) emit(`${config.namespace}.getFeed`, feed); + } + + for (const [alias, collectionConfig] of Object.entries( + config.collections, + ).sort(([left], [right]) => left.localeCompare(right))) { + const collection = collectionConfig.collection ?? alias; + const relations = relationDefinitions(sourceDirs, config, collectionConfig); + const references = referenceDefinitions(config, collectionConfig); + const hydrations = hydrationDefinitions(sourceDirs, relations, references); + const profiles = profileDefinitions(config, sourceDirs); + const methods = getCollectionMethods(collectionConfig); + if (methods.includes("listRecords")) { + const properties: Record = { + records: { + type: "array", + items: { type: "ref", ref: "#record" }, + }, + cursor: { type: "string" }, + }; + if ((config.profiles?.length ?? 0) > 0) { + properties.profiles = { + type: "array", + items: { type: "ref", ref: "#profileEntry" }, + }; + } + emit(`${config.namespace}.${alias}.listRecords`, { + lexicon: 1, + id: `${config.namespace}.${alias}.listRecords`, + defs: { + main: { + type: "query", + description: `Query ${collection} records`, + parameters: { + type: "params", + properties: listParameters( + config, + collectionConfig, + relations, + references, + surface, + ), + }, + output: { + encoding: "application/json", + schema: { + type: "object", + required: ["records"], + properties, + }, + }, + }, + record: recordDefinition( + sourceDirs, + collection, + relations, + references, + ), + ...hydrations, + ...profiles, + }, + }); + } + if (methods.includes("getRecord")) { + const properties = { + ...recordDefinition(sourceDirs, collection, relations, references) + .properties, + ...((config.profiles?.length ?? 0) > 0 + ? { + profiles: { + type: "array", + items: { type: "ref", ref: "#profileEntry" }, + }, + } + : {}), + }; + emit(`${config.namespace}.${alias}.getRecord`, { + lexicon: 1, + id: `${config.namespace}.${alias}.getRecord`, + defs: { + main: { + type: "query", + description: `Get a ${collection} record by AT URI`, + parameters: { + type: "params", + required: ["uri"], + properties: getParameters(config, relations, references), + }, + output: { + encoding: "application/json", + schema: { + type: "object", + required: [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us", + ], + properties, + }, + }, + }, + ...hydrations, + ...profiles, + }, + }); + } + } + + const pullNsids = calculatePullNsids(config, sourceDirs); + if (options.writeAtcuteConfig !== false) { + writeAtcuteConfiguration(rootDir, pullNsids); + } + writeBundle(outputDir, generated, sourceDirs); + const methods = [ + ...extractXrpcMethods(generated), + ...customMethodIds(config), + ].sort(); + log(`Generated ${Object.keys(generated).length} Contrail Lexicons.`); + return { generated, methods, pullNsids }; +} + +function directoryContents(directory: string): Map { + const files = new Map(); + for (const path of walkJson(directory)) { + files.set(relative(directory, path), readFileSync(path, "utf8")); + } + const index = join(directory, "index.ts"); + if (existsSync(index)) files.set("index.ts", readFileSync(index, "utf8")); + return files; +} + +export function checkLexicons(options: GenerateLexiconsOptions): void { + const rootDir = resolve(options.rootDir); + const outputDir = options.outputDir + ? resolve(rootDir, options.outputDir) + : join(rootDir, "lexicons", "generated"); + mkdirSync(dirname(outputDir), { recursive: true }); + const temporary = mkdtempSync( + join(dirname(outputDir), ".contrail-lexicons-"), + ); + try { + const result = generateLexicons({ + ...options, + outputDir: temporary, + writeAtcuteConfig: false, + quiet: true, + }); + const expected = directoryContents(temporary); + const actual = directoryContents(outputDir); + const paths = [...new Set([...expected.keys(), ...actual.keys()])].sort(); + const changed = paths.filter( + (path) => expected.get(path) !== actual.get(path), + ); + const configPath = join(rootDir, "lex.config.js"); + const actualConfig = existsSync(configPath) + ? readFileSync(configPath, "utf8") + : undefined; + if (actualConfig !== atcuteConfiguration(result.pullNsids)) { + changed.push("lex.config.js"); + } + if (changed.length > 0) { + throw new Error( + `generated Contrail Lexicons are stale:\n${changed.map((path) => ` ${path}`).join("\n")}\nRun \`contrail lexicons generate\`.`, + ); + } + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +} diff --git a/packages/contrail/src/lexicons/index.ts b/packages/contrail/src/lexicons/index.ts new file mode 100644 index 0000000..a52e045 --- /dev/null +++ b/packages/contrail/src/lexicons/index.ts @@ -0,0 +1 @@ +export * from "./generate.js"; diff --git a/packages/contrail/tests/built-lexicons.mjs b/packages/contrail/tests/built-lexicons.mjs new file mode 100644 index 0000000..ae77d5f --- /dev/null +++ b/packages/contrail/tests/built-lexicons.mjs @@ -0,0 +1,167 @@ +import { spawnSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); +const cli = join(packageRoot, "dist", "cli.js"); +const root = mkdtempSync(join(packageRoot, ".built-lexicons-")); + +function run(command, args, cwd = packageRoot) { + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + env: { ...process.env, ESBUILD_BINARY_PATH: undefined }, + }); + if (result.status !== 0) { + process.stderr.write(result.stdout ?? ""); + process.stderr.write(result.stderr ?? ""); + throw new Error(`${command} ${args.join(" ")} failed`); + } + return result; +} + +try { + const sourcePath = join( + root, + "lexicons", + "pulled", + "community", + "example", + "event.json", + ); + mkdirSync(dirname(sourcePath), { recursive: true }); + writeFileSync( + sourcePath, + `${JSON.stringify( + { + lexicon: 1, + id: "community.example.event", + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + required: ["name"], + properties: { + name: { type: "string" }, + }, + }, + }, + }, + }, + null, + 2, + )}\n`, + ); + const configPath = join(root, "contrail.config.mjs"); + writeFileSync( + configPath, + `export default { + namespace: "com.example", + profiles: [], + collections: { + event: { + collection: "community.example.event", + queryable: { name: {} }, + }, + }, +}; +`, + ); + + run(process.execPath, [ + cli, + "lexicons", + "generate", + "--root", + root, + "--config", + configPath, + "--public", + ]); + run(process.execPath, [ + cli, + "lexicons", + "check", + "--root", + root, + "--config", + configPath, + "--public", + ]); + + const generated = JSON.parse( + readFileSync( + join( + root, + "lexicons", + "generated", + "com", + "example", + "event", + "listRecords.json", + ), + "utf8", + ), + ); + if (generated.defs.main.parameters.properties.name?.type !== "string") { + throw new Error("generated CLI fixture is missing its query parameter"); + } + + run(process.execPath, [cli, "lexicons", "types", "--root", root]); + const generatedTypes = join( + root, + "src", + "lexicon-types", + "types", + "com", + "example", + "event", + "listRecords.ts", + ); + readFileSync(generatedTypes, "utf8"); + + writeFileSync( + join(root, "tsconfig.json"), + `${JSON.stringify( + { + compilerOptions: { + target: "ES2022", + module: "NodeNext", + moduleResolution: "NodeNext", + strict: true, + noEmit: true, + skipLibCheck: true, + resolveJsonModule: true, + }, + include: ["src/lexicon-types/**/*.ts"], + }, + null, + 2, + )}\n`, + ); + const npmExecPath = process.env.npm_execpath; + if (npmExecPath) { + run(process.execPath, [ + npmExecPath, + "exec", + "tsc", + "--project", + join(root, "tsconfig.json"), + ]); + } else { + run("pnpm", ["exec", "tsc", "--project", join(root, "tsconfig.json")]); + } + + console.log("built Lexicon CLI and Atcute type generation passed"); +} finally { + rmSync(root, { recursive: true, force: true }); +} diff --git a/packages/contrail/tests/hydrate.test.ts b/packages/contrail/tests/hydrate.test.ts index b225213..187f670 100644 --- a/packages/contrail/tests/hydrate.test.ts +++ b/packages/contrail/tests/hydrate.test.ts @@ -79,13 +79,13 @@ describe("resolveHydrates", () => { expect(result).toEqual({}); }); - it("hydrates related records", async () => { + it("maps grouped Lexicon tokens to their configured short names", async () => { const eventUri = "at://did:plc:test/community.lexicon.calendar.event/evt1"; // Insert event await ingestRecords(db, [makeEvent({ uri: eventUri, rkey: "evt1", time_us: 1000 })]); - // Insert RSVPs + // Insert RSVPs using the full token stored in real Lexicon records. for (let i = 0; i < 3; i++) { await ingestRecords(db, [ makeEvent({ @@ -93,7 +93,10 @@ describe("resolveHydrates", () => { did: `did:plc:user${i}`, collection: "rsvp", rkey: `r${i}`, - record: { subject: { uri: eventUri }, status: "going" }, + record: { + subject: { uri: eventUri }, + status: "community.lexicon.calendar.rsvp#going", + }, time_us: 2000 + i, }), ]); diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts new file mode 100644 index 0000000..81e4e9c --- /dev/null +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -0,0 +1,286 @@ +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ContrailConfig } from "../src/core/types"; +import { checkLexicons, generateLexicons } from "../src/lexicons/generate"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "contrail-lexicons-")); + roots.push(root); + const pulled = join(root, "lexicons", "pulled"); + const write = (nsid: string, document: object) => { + const path = join(pulled, ...nsid.split(".")) + ".json"; + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, `${JSON.stringify(document, null, 2)}\n`); + }; + write("community.example.event", { + lexicon: 1, + id: "community.example.event", + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + properties: { + name: { type: "string" }, + startsAt: { type: "string", format: "datetime" }, + }, + }, + }, + }, + }); + write("community.example.rsvp", { + lexicon: 1, + id: "community.example.rsvp", + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + properties: { + status: { + type: "string", + knownValues: [ + "community.example.rsvp#going", + "community.example.rsvp#interested", + ], + }, + }, + }, + }, + }, + }); + const config: ContrailConfig = { + namespace: "example.public", + profiles: [], + collections: { + event: { + collection: "community.example.event", + queryable: { + name: {}, + startsAt: { type: "range" }, + }, + searchable: ["name"], + relations: { + rsvps: { + collection: "rsvp", + groupBy: "status", + groups: { + going: "community.example.rsvp#going", + interested: "community.example.rsvp#interested", + }, + }, + }, + }, + rsvp: { + collection: "community.example.rsvp", + queryable: { status: {} }, + references: { + event: { collection: "event", field: "subject.uri" }, + }, + }, + }, + }; + return { root, config, pulled }; +} + +function parameters(document: any): Record { + return document.defs.main.parameters.properties; +} + +describe("Contrail Lexicon generation", () => { + it("generates the public collection API from projection config", () => { + const { root, config } = fixture(); + const result = generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + + expect(result.methods).toEqual([ + "example.public.event.getRecord", + "example.public.event.listRecords", + "example.public.rsvp.getRecord", + "example.public.rsvp.listRecords", + ]); + expect(result.generated["example.public.getCursor"]).toBeUndefined(); + + const event = result.generated["example.public.event.listRecords"] as any; + const params = parameters(event); + expect(params).toMatchObject({ + search: { type: "string" }, + name: { type: "string" }, + startsAtMin: { type: "string" }, + startsAtMax: { type: "string" }, + rsvpsCountMin: { type: "integer" }, + hydrateRsvps: { type: "integer" }, + }); + expect(params.profiles).toBeUndefined(); + expect(params.sort.knownValues).toContain("rsvpsGoingCount"); + expect(event.defs.record.properties.value.ref).toBe( + "community.example.event#main", + ); + expect(event.defs.hydrateRsvpsRecord.properties.value.ref).toBe( + "community.example.rsvp#main", + ); + + const rsvp = result.generated["example.public.rsvp.listRecords"] as any; + expect(parameters(rsvp).hydrateEvent).toMatchObject({ type: "boolean" }); + expect(rsvp.defs.refEventRecord.properties.value.ref).toBe( + "community.example.event#main", + ); + }); + + it("respects disabled standard methods", () => { + const { root, config } = fixture(); + config.collections.event!.methods = ["listRecords"]; + const result = generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + expect(result.generated["example.public.event.listRecords"]).toBeDefined(); + expect(result.generated["example.public.event.getRecord"]).toBeUndefined(); + }); + + it("generates full-only operational methods when configured", () => { + const { root, config } = fixture(); + config.notify = "private-secret"; + config.feeds = { + network: { targets: ["event", "rsvp"] }, + }; + const result = generateLexicons({ config, rootDir: root, quiet: true }); + expect(result.methods).toContain("example.public.getCursor"); + expect(result.methods).toContain("example.public.getOverview"); + expect(result.methods).toContain("example.public.notifyOfUpdate"); + expect(result.methods).toContain("example.public.getFeed"); + const feed = result.generated["example.public.getFeed"] as any; + expect(parameters(feed)).toMatchObject({ + feed: { knownValues: ["network"] }, + actor: { format: "at-identifier" }, + collection: { + knownValues: ["community.example.event", "community.example.rsvp"], + }, + search: { type: "string" }, + startsAtMin: { type: "string" }, + status: { type: "string" }, + }); + }); + + it("writes a deterministic bundle and current Atcute configuration", () => { + const { root, config } = fixture(); + generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + const bundle = readFileSync( + join(root, "lexicons", "generated", "index.ts"), + "utf8", + ); + expect(bundle).toContain( + 'import _0 from "../pulled/community/example/event.json";', + ); + expect(bundle).toContain("Regenerate with `contrail lexicons generate`"); + + const atcute = readFileSync(join(root, "lex.config.js"), "utf8"); + expect(atcute).toContain("generate: {"); + expect(atcute).toContain("pull: {"); + expect(atcute).toContain('"community.example.event"'); + expect(atcute).toContain('"community.example.rsvp"'); + checkLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + writeFileSync(join(root, "lex.config.js"), "// stale\n"); + expect(() => + checkLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }), + ).toThrow("lex.config.js"); + }); + + it("detects checked-in drift", () => { + const { root, config } = fixture(); + generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + const path = join( + root, + "lexicons", + "generated", + "example", + "public", + "event", + "listRecords.json", + ); + writeFileSync(path, "{}\n"); + expect(() => + checkLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }), + ).toThrow("generated Contrail Lexicons are stale"); + }); + + it("never cleans the project root as generated output", () => { + const { root, config } = fixture(); + const sentinel = join(root, "keep.txt"); + writeFileSync(sentinel, "keep"); + expect(() => + generateLexicons({ + config, + rootDir: root, + outputDir: ".", + surface: "public", + quiet: true, + }), + ).toThrow("must stay inside the project root"); + expect(existsSync(sentinel)).toBe(true); + }); + + it("requires authored Lexicons for custom query handlers before cleaning output", () => { + const { root, config } = fixture(); + const output = join(root, "lexicons", "generated"); + mkdirSync(output, { recursive: true }); + const sentinel = join(output, "keep.txt"); + writeFileSync(sentinel, "keep"); + config.collections.event!.queries = { + featured: async () => Response.json({}), + }; + expect(() => + generateLexicons({ config, rootDir: root, quiet: true }), + ).toThrow("requires a matching Lexicon"); + expect(existsSync(sentinel)).toBe(true); + }); +}); diff --git a/packages/contrail/tsup.config.ts b/packages/contrail/tsup.config.ts index e127492..813e522 100644 --- a/packages/contrail/tsup.config.ts +++ b/packages/contrail/tsup.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ "src/worker/index.ts", "src/cli.ts", "src/cli-config.ts", + "src/lexicons/index.ts", ], format: ["esm"], dts: true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 229e75e..a8b90d6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -216,6 +216,9 @@ importers: '@atcute/jetstream': specifier: ^2.0.2 version: 2.0.2(@atcute/lexicons@2.0.3)(typescript@6.0.3) + '@atcute/lex-cli': + specifier: ^3.2.1 + version: 3.2.1(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(prettier@3.9.6)(typescript@6.0.3) '@atcute/lexicon-doc': specifier: 3.0.2 version: 3.0.2(@atcute/lexicons@2.0.3)(typescript@6.0.3) -- 2.51.2 From c2aef7d8ac6dd63ba5d5f6018776244bee7654da Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:43:13 +0200 Subject: [PATCH 02/15] Preserve consumer Lexicon configuration --- README.md | 2 +- packages/contrail/README.md | 2 +- .../contrail/src/cli/commands/lexicons.ts | 7 ++ packages/contrail/src/lexicons/generate.ts | 56 +++++++++++---- .../contrail/tests/lexicon-generation.test.ts | 70 ++++++++++++++++++- 5 files changed, 121 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ad214d5..6afad71 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ pnpm contrail lexicons generate pnpm contrail lexicons check ``` -Use `contrail lexicons all` to generate Contrail methods, pull referenced source Lexicons, and generate TypeScript types in one pass. The `pull` and `types` actions are also available separately. Contrail owns its config-specific query generation while delegating generic pulling and TypeScript generation to [Atcute](https://github.com/mary-ext/atcute). +Use `contrail lexicons all` to generate Contrail methods, pull referenced source Lexicons, and generate TypeScript types in one pass. The `pull` and `types` actions are also available separately. Contrail updates `lex.config.js` only when the file carries its generated marker; user-owned Atcute configuration is preserved. Pass `--no-atcute-config` to skip creating or checking that generated file. Contrail owns its config-specific query generation while delegating generic pulling and TypeScript generation to [Atcute](https://github.com/mary-ext/atcute). ## Other databases diff --git a/packages/contrail/README.md b/packages/contrail/README.md index d3520a8..b312637 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -89,7 +89,7 @@ pnpm contrail lexicons generate pnpm contrail lexicons check ``` -`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only the collection methods intended for a public read surface. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. +`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only the collection methods intended for a public read surface. Generated `lex.config.js` files carry an ownership marker; existing user-owned Atcute configuration is never replaced. Pass `--no-atcute-config` to manage that file yourself. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. ## Runtime record validation diff --git a/packages/contrail/src/cli/commands/lexicons.ts b/packages/contrail/src/cli/commands/lexicons.ts index 588fc0d..bad978a 100644 --- a/packages/contrail/src/cli/commands/lexicons.ts +++ b/packages/contrail/src/cli/commands/lexicons.ts @@ -16,6 +16,7 @@ interface LexiconOptions { root: string; output: string; public?: boolean; + atcuteConfig?: boolean; } function surface(options: LexiconOptions): LexiconSurface { @@ -29,6 +30,7 @@ async function generate(options: LexiconOptions) { rootDir: resolve(options.root), outputDir: resolve(options.root, options.output), surface: surface(options), + writeAtcuteConfig: options.atcuteConfig !== false, }); } @@ -44,6 +46,10 @@ export function registerLexicons(cli: CAC): void { default: join("lexicons", "generated"), }) .option("--public", "Generate only methods exposed by public read mode") + .option( + "--no-atcute-config", + "Do not create or update a generated lex.config.js", + ) .action(async (action: string, options: LexiconOptions) => { const root = resolve(options.root); if (action === "generate") { @@ -57,6 +63,7 @@ export function registerLexicons(cli: CAC): void { rootDir: root, outputDir: resolve(root, options.output), surface: surface(options), + writeAtcuteConfig: options.atcuteConfig !== false, }); console.log("Contrail Lexicons are current."); return; diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index 0804a4f..7458d6a 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -483,12 +483,28 @@ function calculatePullNsids( return [...values].sort(); } -function atcuteConfiguration(pullNsids: string[]): string { - return `import { defineLexiconConfig } from "@atcute/lex-cli";\n\nexport default defineLexiconConfig({\n generate: {\n files: [\n "lexicons/custom/**/*.json",\n "lexicons/pulled/**/*.json",\n "lexicons/generated/**/*.json",\n ],\n outdir: "src/lexicon-types/",\n },\n pull: {\n outdir: "lexicons/pulled/",\n clean: true,\n sources: [\n {\n type: "atproto",\n mode: "nsids",\n nsids: ${JSON.stringify(pullNsids, null, 2).replace(/^/gm, " ").trim()},\n },\n ],\n },\n});\n`; -} +const GENERATED_ATCUTE_CONFIG_HEADER = + "// Generated by `contrail lexicons generate`. Re-run the command to update; do not edit.\n"; -function writeAtcuteConfiguration(rootDir: string, pullNsids: string[]) { - writeFileSync(join(rootDir, "lex.config.js"), atcuteConfiguration(pullNsids)); +function atcuteConfiguration(pullNsids: string[]): string { + return `${GENERATED_ATCUTE_CONFIG_HEADER}import { defineLexiconConfig } from "@atcute/lex-cli";\n\nexport default defineLexiconConfig({\n generate: {\n files: [\n "lexicons/custom/**/*.json",\n "lexicons/pulled/**/*.json",\n "lexicons/generated/**/*.json",\n ],\n outdir: "src/lexicon-types/",\n },\n pull: {\n outdir: "lexicons/pulled/",\n clean: true,\n sources: [\n {\n type: "atproto",\n mode: "nsids",\n nsids: ${JSON.stringify(pullNsids, null, 2).replace(/^/gm, " ").trim()},\n },\n ],\n },\n});\n`; +} + +function writeAtcuteConfiguration(rootDir: string, pullNsids: string[]): boolean { + const path = join(rootDir, "lex.config.js"); + const source = atcuteConfiguration(pullNsids); + if (existsSync(path)) { + const current = readFileSync(path, "utf8"); + const legacyGeneratedSource = source.slice(GENERATED_ATCUTE_CONFIG_HEADER.length); + if ( + !current.startsWith(GENERATED_ATCUTE_CONFIG_HEADER) && + current !== legacyGeneratedSource + ) { + return false; + } + } + writeFileSync(path, source); + return true; } function writeBundle( @@ -923,8 +939,11 @@ export function generateLexicons( } const pullNsids = calculatePullNsids(config, sourceDirs); - if (options.writeAtcuteConfig !== false) { - writeAtcuteConfiguration(rootDir, pullNsids); + if ( + options.writeAtcuteConfig !== false && + !writeAtcuteConfiguration(rootDir, pullNsids) + ) { + log("Preserved user-owned lex.config.js; update its pull sources manually."); } writeBundle(outputDir, generated, sourceDirs); const methods = [ @@ -967,12 +986,23 @@ export function checkLexicons(options: GenerateLexiconsOptions): void { const changed = paths.filter( (path) => expected.get(path) !== actual.get(path), ); - const configPath = join(rootDir, "lex.config.js"); - const actualConfig = existsSync(configPath) - ? readFileSync(configPath, "utf8") - : undefined; - if (actualConfig !== atcuteConfiguration(result.pullNsids)) { - changed.push("lex.config.js"); + if (options.writeAtcuteConfig !== false) { + const configPath = join(rootDir, "lex.config.js"); + const expectedConfig = atcuteConfiguration(result.pullNsids); + const actualConfig = existsSync(configPath) + ? readFileSync(configPath, "utf8") + : undefined; + const legacyGeneratedConfig = expectedConfig.slice( + GENERATED_ATCUTE_CONFIG_HEADER.length, + ); + if ( + actualConfig === undefined || + ((actualConfig.startsWith(GENERATED_ATCUTE_CONFIG_HEADER) || + actualConfig === legacyGeneratedConfig) && + actualConfig !== expectedConfig) + ) { + changed.push("lex.config.js"); + } } if (changed.length > 0) { throw new Error( diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts index 81e4e9c..cbd998f 100644 --- a/packages/contrail/tests/lexicon-generation.test.ts +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -204,6 +204,7 @@ describe("Contrail Lexicon generation", () => { expect(bundle).toContain("Regenerate with `contrail lexicons generate`"); const atcute = readFileSync(join(root, "lex.config.js"), "utf8"); + expect(atcute).toContain("Generated by `contrail lexicons generate`"); expect(atcute).toContain("generate: {"); expect(atcute).toContain("pull: {"); expect(atcute).toContain('"community.example.event"'); @@ -214,7 +215,10 @@ describe("Contrail Lexicon generation", () => { surface: "public", quiet: true, }); - writeFileSync(join(root, "lex.config.js"), "// stale\n"); + writeFileSync( + join(root, "lex.config.js"), + "// Generated by `contrail lexicons generate`. Re-run the command to update; do not edit.\n// stale\n", + ); expect(() => checkLexicons({ config, @@ -225,6 +229,70 @@ describe("Contrail Lexicon generation", () => { ).toThrow("lex.config.js"); }); + it("preserves user-owned Atcute configuration and supports opting out", () => { + const { root, config } = fixture(); + const path = join(root, "lex.config.js"); + writeFileSync(path, "export default { mine: true };\n"); + + generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + expect(readFileSync(path, "utf8")).toBe( + "export default { mine: true };\n", + ); + expect(() => + checkLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }), + ).not.toThrow(); + + rmSync(path); + generateLexicons({ + config, + rootDir: root, + surface: "public", + writeAtcuteConfig: false, + quiet: true, + }); + expect(existsSync(path)).toBe(false); + expect(() => + checkLexicons({ + config, + rootDir: root, + surface: "public", + writeAtcuteConfig: false, + quiet: true, + }), + ).not.toThrow(); + }); + + it("adopts an exact legacy generated Atcute configuration", () => { + const { root, config } = fixture(); + generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + const path = join(root, "lex.config.js"); + const generated = readFileSync(path, "utf8"); + writeFileSync(path, generated.slice(generated.indexOf("import "))); + + generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + expect(readFileSync(path, "utf8")).toBe(generated); + }); + it("detects checked-in drift", () => { const { root, config } = fixture(); generateLexicons({ -- 2.51.2 From ca897f2f87653dc52805babb1d82d9c70dd9ad11 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 04:52:08 +0200 Subject: [PATCH 03/15] Add verified public read-through services --- packages/contrail/src/cli.ts | 2 + packages/contrail/src/cli/commands/connect.ts | 300 +++++++++++++++ packages/contrail/src/contrail.ts | 13 +- packages/contrail/src/core/backfill.ts | 10 +- packages/contrail/src/core/bootstrap.ts | 14 +- packages/contrail/src/core/db/index.ts | 4 +- packages/contrail/src/core/db/records.ts | 118 +++++- packages/contrail/src/core/db/schema.ts | 9 +- .../contrail/src/core/jetstream-source.ts | 10 + packages/contrail/src/core/jetstream.ts | 22 +- packages/contrail/src/core/persistent.ts | 20 +- .../core/router/{admin.ts => diagnostics.ts} | 20 +- packages/contrail/src/core/router/index.ts | 66 +++- packages/contrail/src/core/types.ts | 17 + packages/contrail/src/index.ts | 9 +- packages/contrail/src/lexicons/generate.ts | 158 ++++---- packages/contrail/src/public-service.ts | 358 ++++++++++++++++++ packages/contrail/src/server.ts | 14 +- packages/contrail/src/worker/index.ts | 12 + 19 files changed, 1049 insertions(+), 127 deletions(-) create mode 100644 packages/contrail/src/cli/commands/connect.ts rename packages/contrail/src/core/router/{admin.ts => diagnostics.ts} (80%) create mode 100644 packages/contrail/src/public-service.ts diff --git a/packages/contrail/src/cli.ts b/packages/contrail/src/cli.ts index 892a83d..7299c6b 100644 --- a/packages/contrail/src/cli.ts +++ b/packages/contrail/src/cli.ts @@ -9,6 +9,7 @@ import { cac } from "cac"; import { registerBackfill } from "./cli/commands/backfill.js"; import { registerDev } from "./cli/commands/dev.js"; import { registerAppendScheduled } from "./cli/commands/append-scheduled.js"; +import { registerConnect } from "./cli/commands/connect.js"; import { registerLexicons } from "./cli/commands/lexicons.js"; const cli = cac("contrail"); @@ -16,6 +17,7 @@ const cli = cac("contrail"); registerBackfill(cli); registerDev(cli); registerAppendScheduled(cli); +registerConnect(cli); registerLexicons(cli); cli.help(); diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts new file mode 100644 index 0000000..aebefc1 --- /dev/null +++ b/packages/contrail/src/cli/commands/connect.ts @@ -0,0 +1,300 @@ +import { + access, + mkdir, + mkdtemp, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, +} from "node:path"; +import { isNsid } from "@atcute/lexicons/syntax"; +import type { CAC } from "cac"; +import { + contractFromManifest, + digestLexiconDocuments, + digestPublicContract, + isPublicServiceManifest, + normalizePublicServiceEndpoint, + validateManifestContract, + type PublicServiceManifest, +} from "../../public-service.js"; +import { generateLexiconTypesWithAtcute } from "../atcute.js"; + +const MAX_DISCOVERY_BYTES = 10 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 15_000; + +interface ConnectOptions { + root: string; + out: string; + lock: string; + generate?: boolean; + update?: boolean; +} + +export interface ProviderLock { + format: "contrail.provider-lock"; + version: 1; + endpoint: string; + namespace: string; + contractDigest: string; + lexiconDigest: string; + methods: string[]; + lexiconRoot: string; +} + +async function readJson(response: Response, label: string): Promise { + if (!response.ok) { + throw new Error( + `${label} request failed: ${response.status} ${response.statusText}`, + ); + } + const declaredLength = Number(response.headers.get("content-length") ?? 0); + if (declaredLength > MAX_DISCOVERY_BYTES) { + throw new Error(`${label} exceeds ${MAX_DISCOVERY_BYTES} bytes`); + } + const text = await response.text(); + if (new TextEncoder().encode(text).byteLength > MAX_DISCOVERY_BYTES) { + throw new Error(`${label} exceeds ${MAX_DISCOVERY_BYTES} bytes`); + } + try { + return JSON.parse(text) as unknown; + } catch { + throw new Error(`${label} is not valid JSON`); + } +} + +async function fetchJson( + fetcher: typeof fetch, + url: string | URL, + label: string, + timeoutMs: number, +): Promise { + const requested = new URL(url); + const response = await fetcher(requested, { + signal: AbortSignal.timeout(timeoutMs), + }); + if (response.url && new URL(response.url).origin !== requested.origin) { + throw new Error(`${label} redirected to a different origin`); + } + return readJson(response, label); +} + +function resolveInsideRoot( + root: string, + value: string, + options: { allowRoot?: boolean } = {}, +): string { + const target = resolve(root, value); + const rel = relative(root, target); + if ( + (!options.allowRoot && rel === "") || + rel === ".." || + rel.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) || + isAbsolute(rel) + ) { + throw new Error(`path must stay inside the consumer project: ${value}`); + } + return target; +} + +function lexiconPath(root: string, id: string): string { + if (!isNsid(id)) throw new Error(`invalid Lexicon NSID: ${id}`); + return resolve(root, ...id.split(".")) + ".json"; +} + +async function exists(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + +export async function connectPublicService(options: { + endpoint: string; + root: string; + out: string; + lock: string; + fetcher?: typeof fetch; + update?: boolean; + /** @internal Shorter timeout for deterministic connection tests. */ + timeoutMs?: number; +}): Promise<{ + manifest: PublicServiceManifest; + lock: ProviderLock; + written: number; +}> { + const endpoint = normalizePublicServiceEndpoint(options.endpoint); + const projectRoot = resolve(options.root); + const lockPath = resolveInsideRoot(projectRoot, options.lock); + if (!options.update) { + try { + await readFile(lockPath, "utf8"); + throw new Error( + "a Contrail provider lock already exists; rerun with --update", + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + const fetcher = options.fetcher ?? fetch; + const manifestValue = await fetchJson( + fetcher, + `${endpoint}/.well-known/contrail`, + "Contrail manifest", + options.timeoutMs ?? REQUEST_TIMEOUT_MS, + ); + if (!isPublicServiceManifest(manifestValue)) { + throw new Error("response is not a supported Contrail service manifest"); + } + const manifest = manifestValue; + if (normalizePublicServiceEndpoint(manifest.endpoint) !== endpoint) { + throw new Error( + `manifest endpoint mismatch: expected ${endpoint}, received ${manifest.endpoint}`, + ); + } + const lexiconUrl = new URL(manifest.lexicons.url); + const statusUrl = new URL(manifest.status.url); + if ( + lexiconUrl.origin !== endpoint || + lexiconUrl.username !== "" || + lexiconUrl.password !== "" || + statusUrl.origin !== endpoint || + statusUrl.username !== "" || + statusUrl.password !== "" + ) { + throw new Error("manifest resource URLs must use the service origin"); + } + const lexiconValue = await fetchJson( + fetcher, + lexiconUrl, + "Contrail Lexicons", + options.timeoutMs ?? REQUEST_TIMEOUT_MS, + ); + const values = (lexiconValue as { lexicons?: unknown })?.lexicons; + if (!Array.isArray(values)) { + throw new Error("Lexicon response must contain a lexicons array"); + } + const { lexicons, digest } = await digestLexiconDocuments(values as object[]); + if (digest !== manifest.lexicons.digest) { + throw new Error( + `Lexicon digest mismatch: manifest=${manifest.lexicons.digest}, fetched=${digest}`, + ); + } + validateManifestContract(manifest, lexicons); + const contractDigest = await digestPublicContract( + contractFromManifest(manifest), + ); + if (contractDigest !== manifest.contract.digest) { + throw new Error( + `Contract digest mismatch: manifest=${manifest.contract.digest}, computed=${contractDigest}`, + ); + } + + const outputRoot = resolveInsideRoot(projectRoot, options.out); + const providerKey = new URL(endpoint).host.replace(/[^a-zA-Z0-9.-]/g, "_"); + const providerRoot = resolveInsideRoot(outputRoot, providerKey); + await mkdir(outputRoot, { recursive: true }); + const stagedProvider = await mkdtemp(join(outputRoot, `.${providerKey}-`)); + for (const document of lexicons) { + const path = lexiconPath(stagedProvider, document.id); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(document, null, 2)}\n`); + } + + const lock: ProviderLock = { + format: "contrail.provider-lock", + version: 1, + endpoint, + namespace: manifest.namespace, + contractDigest: manifest.contract.digest, + lexiconDigest: manifest.lexicons.digest, + methods: [...manifest.methods].sort(), + lexiconRoot: relative(projectRoot, providerRoot), + }; + const lockDirectory = dirname(lockPath); + await mkdir(lockDirectory, { recursive: true }); + const stagedLockDirectory = await mkdtemp( + join(lockDirectory, `.${basename(lockPath)}-`), + ); + const stagedLock = join(stagedLockDirectory, basename(lockPath)); + await writeFile(stagedLock, `${JSON.stringify(lock, null, 2)}\n`); + + const backupRoot = join( + outputRoot, + `.${providerKey}.backup-${process.pid}-${Date.now()}`, + ); + const backupLock = join( + lockDirectory, + `.${basename(lockPath)}.backup-${process.pid}-${Date.now()}`, + ); + const hadProvider = await exists(providerRoot); + const hadLock = await exists(lockPath); + try { + if (hadProvider) await rename(providerRoot, backupRoot); + if (hadLock) await rename(lockPath, backupLock); + await rename(stagedProvider, providerRoot); + await rename(stagedLock, lockPath); + await rm(backupRoot, { recursive: true, force: true }); + await rm(backupLock, { force: true }); + } catch (error) { + await rm(providerRoot, { recursive: true, force: true }); + await rm(lockPath, { force: true }); + if (hadProvider && (await exists(backupRoot))) { + await rename(backupRoot, providerRoot); + } + if (hadLock && (await exists(backupLock))) { + await rename(backupLock, lockPath); + } + throw error; + } finally { + await rm(stagedProvider, { recursive: true, force: true }); + await rm(stagedLockDirectory, { recursive: true, force: true }); + await rm(backupRoot, { recursive: true, force: true }); + await rm(backupLock, { force: true }); + } + return { manifest, lock, written: lexicons.length }; +} + +export function registerConnect(cli: CAC): void { + cli + .command( + "connect ", + "Discover a public Contrail, lock its API, pull Lexicons, and generate types", + ) + .option("--root ", "Consumer project root", { + default: process.cwd(), + }) + .option("--out ", "Provider-owned Lexicon storage relative to root", { + default: "lexicons/pulled", + }) + .option("--lock ", "Provider lock file relative to root", { + default: "contrail.lock.json", + }) + .option("--update", "Replace an existing provider lock and owned Lexicons") + .option("--no-generate", "Pull and lock without running Atcute lex-cli") + .action(async (endpoint: string, options: ConnectOptions) => { + const result = await connectPublicService({ + endpoint, + root: options.root, + out: options.out, + lock: options.lock, + update: options.update, + }); + console.log( + `connected ${result.lock.endpoint}: ${result.written} Lexicons, contract ${result.lock.contractDigest}`, + ); + if (options.generate !== false) { + generateLexiconTypesWithAtcute(resolve(options.root)); + } + }); +} diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index 181a80d..a567e26 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -12,7 +12,11 @@ import { initSchema } from "./core/db/schema"; import { prepareRecordValidation } from "./core/validation"; import { getIngestDiagnostics } from "./core/diagnostics"; import { optimizeDatabase } from "./core/db/optimize"; -import { queryRecords, type QueryOptions } from "./core/db/records"; +import { + assertServingSourceCompatibility, + queryRecords, + type QueryOptions, +} from "./core/db/records"; import { createIngestState, runIngestCycle, @@ -69,7 +73,12 @@ export class Contrail { /** Initialize the database schema. */ async init(db?: Database): Promise { - await initSchema(this.getDb(db), this.config); + const database = this.getDb(db); + await initSchema(database, this.config); + await assertServingSourceCompatibility( + database, + this.config.orderedSource, + ); } /** Refresh the SQLite query-planner statistics (bounded `PRAGMA optimize`) so diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index 9873c7b..f4d799f 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -41,11 +41,15 @@ const DERIVED_PROJECTIONS_DIRTY_KEY = "backfill_derived_projections_dirty"; * and cursor. */ const INITIAL_CAPTURE_OVERLAP_US = 10_000_000; -async function ensureInitialReplayBoundary(db: Database): Promise { +async function ensureInitialReplayBoundary( + db: Database, + config: ContrailConfig, +): Promise { if ((await getLastCursor(db)) !== null) return; await saveCursor( db, Math.max(0, Date.now() * 1000 - INITIAL_CAPTURE_OVERLAP_US), + config.orderedSource, ); } @@ -567,7 +571,7 @@ async function backfillPendingWork( // Direct callers may begin with already-discovered work. Capture before the // first PDS request so changes racing the sampled scan remain replayable. - await ensureInitialReplayBoundary(db); + await ensureInitialReplayBoundary(db, config); // Mark the set-based catch-up dirty before canonical writes. A crash can leave // search/count projections stale, so status stays incomplete and the next @@ -1054,7 +1058,7 @@ export async function discoverDIDs( // repository created after its relay page was scanned but before the later // PDS phase could fall before the live cursor and disappear from both paths. if (options.captureReplayBoundary !== false) { - await ensureInitialReplayBoundary(db); + await ensureInitialReplayBoundary(db, config); } const discovered: string[] = []; diff --git a/packages/contrail/src/core/bootstrap.ts b/packages/contrail/src/core/bootstrap.ts index df39997..a5b69a9 100644 --- a/packages/contrail/src/core/bootstrap.ts +++ b/packages/contrail/src/core/bootstrap.ts @@ -1,7 +1,10 @@ import type { ContrailConfig, Database, IngestEvent, Statement } from "./types"; import { recordTimeUs, createIngestEvent, ingestRecords } from "./ingest"; import { getDependentNsids, recordsTableName } from "./types"; -import { rebuildDerivedProjections } from "./db/records"; +import { + rebuildDerivedProjections, + saveServingSourcePositionStatement, +} from "./db/records"; import { BOOTSTRAP_VERIFICATION_META_KEY, BootstrapVerificationError, @@ -403,7 +406,14 @@ export class DatabaseBootstrapTarget implements BootstrapTarget { Date.now(), BOOTSTRAP_STATE_ID, ); - await this.apply(events, [checkpoint], false); + await this.apply( + events, + [ + checkpoint, + saveServingSourcePositionStatement(this.db, batch.checkpoint), + ], + false, + ); } async complete(): Promise { diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index a592667..c772276 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,6 +1,6 @@ export { initSchema, CONTRAIL_SCHEMA_VERSION } from "./schema"; export { getMeta, setMeta, getMetaNumber } from "./meta"; export { optimizeDatabase } from "./optimize"; -export { getLastCursor, saveCursor, saveCursorStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; -export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult } from "./records"; +export { assertServingSourceCompatibility, getLastCursor, getServingSourcePosition, orderedSourcePosition, saveCursor, saveCursorStatement, saveOrderedSourcePositionStatement, saveServingSourcePositionStatement, lookupExistingRecords, queryRecords, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; +export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult, ServingSourcePosition } from "./records"; export type { RecordSource } from "../types"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index 0f60317..2b34a39 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -7,6 +7,7 @@ import type { IngestEvent, RecordRow, RecordSource, + OrderedSourceConfig, } from "../types"; import { getNestedValue, @@ -24,6 +25,7 @@ import { } from "../types"; import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; import { ftsQueryClause, getDialect } from "../dialect"; +import type { SourcePosition } from "../sources"; // --- Counts --- @@ -508,7 +510,112 @@ export async function saveFeedPruneCursor( .run(); } -// --- Cursor --- +// --- Cursor and ordered source position --- + +export interface ServingSourcePosition { + position: SourcePosition; + updatedAt: number; +} + +export async function getServingSourcePosition( + db: Database, +): Promise { + const row = await db + .prepare( + "SELECT source, epoch, cursor, updated_at FROM source_position WHERE id = 1", + ) + .first<{ + source: string; + epoch: string; + cursor: string; + updated_at: number; + }>(); + return row + ? { + position: { + source: row.source, + epoch: row.epoch, + cursor: row.cursor, + }, + updatedAt: Number(row.updated_at), + } + : null; +} + +export function saveServingSourcePositionStatement( + db: Database, + position: SourcePosition, + updatedAt = Date.now(), +): Statement { + return db + .prepare( + `INSERT INTO source_position (id, source, epoch, cursor, updated_at) + VALUES (1, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + source = excluded.source, + epoch = excluded.epoch, + cursor = excluded.cursor, + updated_at = excluded.updated_at`, + ) + .bind(position.source, position.epoch, position.cursor, updatedAt); +} + +export async function assertServingSourceCompatibility( + db: Database, + orderedSource?: OrderedSourceConfig, +): Promise { + if (!orderedSource) return; + const existing = await getServingSourcePosition(db); + if ( + existing && + (existing.position.source !== orderedSource.source || + existing.position.epoch !== orderedSource.epoch) + ) { + throw new Error( + `configured ordered source ${orderedSource.source}/${orderedSource.epoch} ` + + `does not match durable source position ` + + `${existing.position.source}/${existing.position.epoch}`, + ); + } +} + +export function orderedSourcePosition( + orderedSource: OrderedSourceConfig, + cursor: number | string, +): SourcePosition { + return { + source: orderedSource.source, + epoch: orderedSource.epoch, + cursor: String(cursor), + }; +} + +export function saveOrderedSourcePositionStatement( + db: Database, + orderedSource: OrderedSourceConfig, + timeUs: number, + updatedAt = Date.now(), +): Statement { + const position = orderedSourcePosition(orderedSource, timeUs); + return db + .prepare( + `INSERT INTO source_position (id, source, epoch, cursor, updated_at) + SELECT 1, ?, ?, ?, ? + WHERE (SELECT time_us FROM cursor WHERE id = 1) = ? + ON CONFLICT(id) DO UPDATE SET + source = excluded.source, + epoch = excluded.epoch, + cursor = excluded.cursor, + updated_at = excluded.updated_at`, + ) + .bind( + position.source, + position.epoch, + position.cursor, + updatedAt, + timeUs, + ); +} export async function getLastCursor(db: Database): Promise { const row = await db @@ -531,8 +638,15 @@ export function saveCursorStatement( export async function saveCursor( db: Database, timeUs: number, + orderedSource?: OrderedSourceConfig, ): Promise { - await saveCursorStatement(db, timeUs).run(); + const statements = [saveCursorStatement(db, timeUs)]; + if (orderedSource) { + statements.push( + saveOrderedSourcePositionStatement(db, orderedSource, timeUs), + ); + } + await db.batch(statements); } // --- Existing record lookup --- diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index e7116d3..fde2f3e 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -17,7 +17,7 @@ import { getSearchableFields } from "../search"; import { buildLabelsSchema } from "../labels/schema"; import { getMeta, setMeta } from "./meta"; -export const CONTRAIL_SCHEMA_VERSION = 9; +export const CONTRAIL_SCHEMA_VERSION = 10; const SCHEMA_FINGERPRINT_KEY = "schema_fingerprint"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -60,6 +60,13 @@ CREATE TABLE IF NOT EXISTS cursor ( id INTEGER PRIMARY KEY CHECK (id = 1), time_us ${dialect.bigintType} NOT NULL ); +CREATE TABLE IF NOT EXISTS source_position ( + id INTEGER PRIMARY KEY CHECK (id = 1), + source TEXT NOT NULL, + epoch TEXT NOT NULL, + cursor TEXT NOT NULL, + updated_at ${dialect.bigintType} NOT NULL +); CREATE TABLE IF NOT EXISTS backfill_state ( id INTEGER PRIMARY KEY CHECK (id = 1), run_id TEXT, diff --git a/packages/contrail/src/core/jetstream-source.ts b/packages/contrail/src/core/jetstream-source.ts index c7cdaad..7449628 100644 --- a/packages/contrail/src/core/jetstream-source.ts +++ b/packages/contrail/src/core/jetstream-source.ts @@ -158,6 +158,16 @@ export class JetstreamChangeSource implements ChangeSource { private readonly options: JetstreamChangeSourceOptions, ) { if (!options.epoch) throw new TypeError("Jetstream source epoch is required"); + if ( + config.orderedSource && + (config.orderedSource.source !== this.id || + config.orderedSource.epoch !== options.epoch) + ) { + throw new TypeError( + `Jetstream source ${this.id}/${options.epoch} does not match ` + + `configured ordered source ${config.orderedSource.source}/${config.orderedSource.epoch}`, + ); + } if (!Number.isSafeInteger(options.retentionUs) || options.retentionUs <= 0) { throw new TypeError("Jetstream retentionUs must be a positive safe integer"); } diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index d6bacaf..b29c8d1 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -10,7 +10,7 @@ import { optimizeIntervalMs, optimizeAnalysisLimit, } from "./types"; -import { initSchema, getLastCursor, saveCursor, saveCursorStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; +import { initSchema, getLastCursor, saveCursor, saveCursorStatement, saveOrderedSourcePositionStatement, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor, getMetaNumber, setMeta, optimizeDatabase } from "./db"; import { createIngestEvent, ingestRecords, recordTimeUs } from "./ingest"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; @@ -290,7 +290,10 @@ export async function ingestEvents( cid: commit.operation === "delete" ? null : commit.cid, value: commit.operation === "delete" ? undefined : commit.record, source: { - id: "jetstream", + id: config.orderedSource?.source ?? "jetstream", + ...(config.orderedSource + ? { epoch: config.orderedSource.epoch } + : {}), time_us: event.time_us, revision: commit.rev, cursor: String(event.time_us), @@ -480,7 +483,18 @@ export async function runIngestCycle( // them safely; the final batch atomically commits the exact source cursor. trailingStatements: isFinalBatch && lastCursor !== null - ? [saveCursorStatement(db, lastCursor)] + ? [ + saveCursorStatement(db, lastCursor), + ...(config.orderedSource + ? [ + saveOrderedSourcePositionStatement( + db, + config.orderedSource, + lastCursor, + ), + ] + : []), + ] : undefined, }); accepted.push(...result.accepted); @@ -509,7 +523,7 @@ export async function runIngestCycle( // An identity-only or fully filtered stream has no canonical record batch, so // its cursor can advance independently after best-effort identity handling. if (lastCursor !== null && events.length === 0) { - await saveCursor(db, lastCursor); + await saveCursor(db, lastCursor, config.orderedSource); } if (lastCursor !== null) { log.log( diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index f5f1586..3b96cc6 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -8,7 +8,7 @@ import { jetstreamUrlOption, resolveConfig, } from "./types"; -import { initSchema, getLastCursor, saveCursorStatement } from "./db"; +import { initSchema, getLastCursor, saveCursorStatement, saveOrderedSourcePositionStatement } from "./db"; import { createIngestEvent, ingestRecords, recordTimeUs } from "./ingest"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; @@ -162,7 +162,18 @@ async function streamAndFlush( try { ingestResult = await ingestRecords(db, batch, config, { knownDids, - trailingStatements: [saveCursorStatement(db, lastTimeUs)], + trailingStatements: [ + saveCursorStatement(db, lastTimeUs), + ...(config.orderedSource + ? [ + saveOrderedSourcePositionStatement( + db, + config.orderedSource, + lastTimeUs, + ), + ] + : []), + ], }); } catch (error) { // The cursor transaction failed, so keep this exact batch at the front @@ -313,7 +324,10 @@ async function streamAndFlush( cid: commit.operation === "delete" ? null : commit.cid, value: commit.operation === "delete" ? undefined : commit.record, source: { - id: "jetstream", + id: config.orderedSource?.source ?? "jetstream", + ...(config.orderedSource + ? { epoch: config.orderedSource.epoch } + : {}), time_us: event.time_us, revision: commit.rev, cursor: String(event.time_us), diff --git a/packages/contrail/src/core/router/admin.ts b/packages/contrail/src/core/router/diagnostics.ts similarity index 80% rename from packages/contrail/src/core/router/admin.ts rename to packages/contrail/src/core/router/diagnostics.ts index 9438834..72538c4 100644 --- a/packages/contrail/src/core/router/admin.ts +++ b/packages/contrail/src/core/router/diagnostics.ts @@ -1,7 +1,7 @@ import type { Hono } from "hono"; import type { ContrailConfig, Database } from "../types"; import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; -import { getLastCursor } from "../db"; +import { getLastCursor, getServingSourcePosition } from "../db"; import { getBackfillStatus } from "../status"; export interface CursorStatus { @@ -30,7 +30,7 @@ export async function getCursorStatus(db: Database): Promise { }; } -export async function getOverview(db: Database, config: ContrailConfig) { +export async function getStatusOverview(db: Database, config: ContrailConfig) { const collections: CollectionOverview[] = []; for (const short of getCollectionShortNames(config)) { @@ -62,7 +62,7 @@ export async function getOverview(db: Database, config: ContrailConfig) { }; } -export function registerAdminRoutes( +export function registerCursorRoute( app: Hono, db: Database, config: ContrailConfig @@ -70,16 +70,12 @@ export function registerAdminRoutes( const ns = config.namespace; app.get(`/xrpc/${ns}.getCursor`, async (c) => { - const cursor = await getCursorStatus(db); - if (cursor.cursor === null) return c.json({ cursor: null }); + const current = await getServingSourcePosition(db); + if (!current) return c.json({}); return c.json({ - time_us: cursor.cursor, - date: cursor.date, - seconds_ago: cursor.seconds_ago, + position: current.position, + updatedAt: current.updatedAt, + updatedAtDate: new Date(current.updatedAt).toISOString(), }); }); - - app.get(`/xrpc/${ns}.getOverview`, async (c) => - c.json(await getOverview(db, config)) - ); } diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 052ec8b..4ae1c84 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -6,15 +6,23 @@ import { backfillUser } from "../backfill"; import { hydrateLabels } from "../labels/hydrate"; import { selectAcceptedLabelers } from "../labels/select"; import { resolveActor } from "../identity"; -import { getOverview, registerAdminRoutes } from "./admin"; +import { getStatusOverview, registerCursorRoute } from "./diagnostics"; import { registerCollectionRoutes } from "./collection"; import { registerFeedRoutes } from "./feed"; import { registerNotifyRoute } from "./notify"; import { resolveProfiles } from "./profiles"; +import { + describePublicService, + normalizeLexiconDocuments, + normalizePublicServiceEndpoint, + type PublicServiceOptions, +} from "../../public-service"; export interface CreateAppOptions { /** Lexicon JSON documents to expose from the deployment. */ lexicons?: object[]; + /** Enable stable discovery for anonymous read-through clients. */ + publicService?: PublicServiceOptions; } export function createApp( @@ -26,13 +34,61 @@ export function createApp( app.use("*", cors()); app.get("/", (c) => c.json({ status: "ok" })); - app.get("/status", async (c) => c.json(await getOverview(db, config))); + app.get("/status", async (c) => { + const overview = await getStatusOverview(db, config); + if (!options.publicService) return c.json(overview); + c.header("cache-control", "public, max-age=15, stale-while-revalidate=45"); + return c.json({ + status: overview.status, + serving: "ready", + total_records: overview.total_records, + collections: overview.collections, + freshness: { + last_event_at: overview.ingestion.date, + seconds_ago: overview.ingestion.seconds_ago, + }, + backfill: overview.backfill, + }); + }); app.get("/health", (c) => c.json({ status: "ok" })); app.get("/xrpc/_health", (c) => c.json({ status: "ok" })); const ns = config.namespace; - if (options.lexicons && options.lexicons.length > 0) { - const lexicons = options.lexicons; + const lexicons = options.publicService + ? normalizeLexiconDocuments(options.lexicons ?? []) + : (options.lexicons ?? []); + if (options.publicService) { + normalizePublicServiceEndpoint(options.publicService.endpoint); + const description = describePublicService( + config, + options.publicService, + lexicons, + ); + app.get("/.well-known/contrail", async (c) => { + const { manifest } = await description; + c.header("cache-control", "no-cache"); + c.header("etag", `\"${manifest.contract.digest}\"`); + return c.json(manifest); + }); + app.get("/lexicons", async (c) => { + const service = await description; + c.header("content-type", "application/json; charset=UTF-8"); + c.header("cache-control", "no-cache"); + c.header("etag", `\"${service.manifest.lexicons.digest}\"`); + return c.body(service.canonicalLexicons); + }); + app.get("/lexicons/:digest", async (c) => { + const service = await description; + if (c.req.param("digest") !== service.manifest.lexicons.digest) { + return c.json({ error: "Lexicon bundle not found" }, 404); + } + c.header("content-type", "application/json; charset=UTF-8"); + c.header("cache-control", "public, max-age=31536000, immutable"); + c.header("etag", `\"${service.manifest.lexicons.digest}\"`); + return c.body(service.canonicalLexicons); + }); + } + if (lexicons.length > 0) { app.get(`/xrpc/${ns}.lexicons`, (c) => c.json({ lexicons })); } @@ -80,7 +136,7 @@ export function createApp( return c.json({ profiles }); }); - registerAdminRoutes(app, db, config); + registerCursorRoute(app, db, config); registerCollectionRoutes(app, db, config); registerFeedRoutes(app, db, config); registerNotifyRoute(app, db, config); diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index f81c162..e71cdb5 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -238,6 +238,13 @@ export interface IngestValidationConfig { allowCidlessSources?: string[]; } +export interface OrderedSourceConfig { + /** Stable logical identifier of the primary ordered change source. */ + source: string; + /** Operator-owned continuity epoch. Change it whenever cursor continuity changes. */ + epoch: string; +} + export interface ContrailConfig { namespace: string; /** Collections to index, keyed by short name. Short names become endpoint URL segments @@ -256,6 +263,10 @@ export interface ContrailConfig { * connection (`runPersistent`), where the per-switch 10s skew rollback fires * about once rather than every cycle. */ jetstreams?: string[]; + /** Identity of the ordered source consumed by live ingestion. Its opaque + * cursor is persisted atomically with projected mutations and may be exposed + * to clients as a cache invalidation coordinate. */ + orderedSource?: OrderedSourceConfig; feeds?: Record; logger?: Logger; /** Expose the notifyOfUpdate HTTP endpoint. Off by default. @@ -382,6 +393,12 @@ export interface ResolvedContrailConfig extends ContrailConfig { * Resolve config: apply defaults, auto-add profile collections, compute queryable maps. */ export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig { + if ( + config.orderedSource && + (!config.orderedSource.source.trim() || !config.orderedSource.epoch.trim()) + ) { + throw new TypeError("orderedSource requires non-empty source and epoch values"); + } const profiles = (config.profiles ?? DEFAULT_PROFILES).map( normalizeProfileConfig ); diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 874ea50..bd254c2 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -2,6 +2,7 @@ export type { LexiconDoc } from "@atcute/lexicon-doc"; export { Contrail } from "./contrail"; export type { AppOptions, ContrailOptions } from "./contrail"; +export * from "./public-service"; // Configuration, storage, identity, and dialects. export * from "./core/types"; @@ -35,14 +36,19 @@ export * from "./core/constellation"; // Database. export * from "./core/db/schema"; export { + assertServingSourceCompatibility, getFeedPruneCursor, getLastCursor, + getServingSourcePosition, lookupExistingRecords, pruneActorFeed, pruneFeedItems, queryRecords, saveCursor, saveCursorStatement, + saveOrderedSourcePositionStatement, + saveServingSourcePositionStatement, + orderedSourcePosition, saveFeedPruneCursor, sweepFeedItems, } from "./core/db/records"; @@ -51,6 +57,7 @@ export type { FeedSweepResult, QueryOptions, SortOption, + ServingSourcePosition, } from "./core/db/records"; export * from "./core/db/meta"; export * from "./core/db/optimize"; @@ -60,7 +67,7 @@ export * from "./core/router"; export * from "./core/router/notify"; export * from "./core/router/profiles"; export * from "./core/router/feed"; -export * from "./core/router/admin"; +export * from "./core/router/diagnostics"; export * from "./core/router/collection"; export * from "./core/router/hydrate"; export * from "./core/router/helpers"; diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index 7458d6a..1aa628e 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -100,13 +100,6 @@ function collectionReference( return document?.defs?.main ? `${nsid}#main` : null; } -function recordObjectSchema(sourceDirs: string[], nsid: string): any | null { - const path = findLexicon(sourceDirs, nsid); - const document = path ? readLexicon(path) : null; - const main = document?.defs?.main; - return main?.type === "record" && main.record ? main.record : null; -} - function groupsForRelation( sourceDirs: string[], config: ContrailConfig, @@ -135,19 +128,13 @@ function groupsForRelation( function profileDefinitions(config: ContrailConfig, sourceDirs: string[]) { const profiles = config.profiles ?? []; if (profiles.length === 0) return {}; - const definitions: Record = {}; const refs: string[] = []; for (const profile of profiles) { const collection = typeof profile === "string" ? profile : profile.collection; - const schema = recordObjectSchema(sourceDirs, collection); - if (!schema) continue; - const name = collection - .split(".") - .map((part, index) => (index === 0 ? part : capitalize(part))) - .join(""); - definitions[name] = schema; - refs.push(`#${name}`); + if (collectionReference(sourceDirs, collection)) { + refs.push(`${collection}#main`); + } } const value = refs.length === 1 @@ -169,7 +156,6 @@ function profileDefinitions(config: ContrailConfig, sourceDirs: string[]) { rkey: { type: "string" }, }, }, - ...definitions, }; } @@ -732,102 +718,98 @@ export function generateLexicons( log(` ${nsid}`); }; - if (surface === "full") { - emit(`${config.namespace}.getCursor`, { + emit(`${config.namespace}.getCursor`, { + lexicon: 1, + id: `${config.namespace}.getCursor`, + defs: { + main: { + type: "query", + description: "Get the committed primary ordered-source position", + output: { + encoding: "application/json", + schema: { + type: "object", + properties: { + position: { type: "ref", ref: "#sourcePosition" }, + updatedAt: { type: "integer" }, + updatedAtDate: { type: "string", format: "datetime" }, + }, + }, + }, + }, + sourcePosition: { + type: "object", + required: ["source", "epoch", "cursor"], + properties: { + source: { type: "string" }, + epoch: { type: "string" }, + cursor: { type: "string" }, + }, + }, + }, + }); + if ((config.profiles?.length ?? 0) > 0) { + emit(`${config.namespace}.getProfile`, { lexicon: 1, - id: `${config.namespace}.getCursor`, + id: `${config.namespace}.getProfile`, defs: { main: { type: "query", - description: "Get the current ingestion observation time", + parameters: { + type: "params", + required: ["actor"], + properties: { + actor: { type: "string", format: "at-identifier" }, + }, + }, output: { encoding: "application/json", schema: { type: "object", + required: ["profiles"], properties: { - time_us: { type: "integer" }, - date: { type: "string" }, - seconds_ago: { type: "integer" }, + profiles: { + type: "array", + items: { type: "ref", ref: "#profileEntry" }, + }, }, }, }, }, + ...profileDefinitions(config, sourceDirs), }, }); - emit(`${config.namespace}.getOverview`, { + } + const feed = feedLexicon(config, sourceDirs); + if (feed) emit(`${config.namespace}.getFeed`, feed); + if (surface === "full" && config.notify) { + emit(`${config.namespace}.notifyOfUpdate`, { lexicon: 1, - id: `${config.namespace}.getOverview`, + id: `${config.namespace}.notifyOfUpdate`, defs: { main: { - type: "query", - description: "Get aggregate projection status", - output: { encoding: "application/json", schema: { type: "unknown" } }, - }, - }, - }); - if ((config.profiles?.length ?? 0) > 0) { - emit(`${config.namespace}.getProfile`, { - lexicon: 1, - id: `${config.namespace}.getProfile`, - defs: { - main: { - type: "query", - parameters: { - type: "params", - required: ["actor"], + type: "procedure", + input: { + encoding: "application/json", + schema: { + type: "object", properties: { - actor: { type: "string", format: "at-identifier" }, - }, - }, - output: { - encoding: "application/json", - schema: { - type: "object", - required: ["profiles"], - properties: { - profiles: { - type: "array", - items: { type: "ref", ref: "#profileEntry" }, - }, + uri: { type: "string", format: "at-uri" }, + uris: { + type: "array", + items: { type: "string", format: "at-uri" }, + maxLength: 25, }, }, }, }, - ...profileDefinitions(config, sourceDirs), - }, - }); - } - if (config.notify) { - emit(`${config.namespace}.notifyOfUpdate`, { - lexicon: 1, - id: `${config.namespace}.notifyOfUpdate`, - defs: { - main: { - type: "procedure", - input: { - encoding: "application/json", - schema: { - type: "object", - properties: { - uri: { type: "string", format: "at-uri" }, - uris: { - type: "array", - items: { type: "string", format: "at-uri" }, - maxLength: 25, - }, - }, - }, - }, - output: { - encoding: "application/json", - schema: { type: "unknown" }, - }, + output: { + encoding: "application/json", + schema: { type: "unknown" }, }, }, - }); - } - const feed = feedLexicon(config, sourceDirs); - if (feed) emit(`${config.namespace}.getFeed`, feed); + }, + }); } for (const [alias, collectionConfig] of Object.entries( diff --git a/packages/contrail/src/public-service.ts b/packages/contrail/src/public-service.ts new file mode 100644 index 0000000..118c8f6 --- /dev/null +++ b/packages/contrail/src/public-service.ts @@ -0,0 +1,358 @@ +import { isNsid } from "@atcute/lexicons/syntax"; +import type { ContrailConfig } from "./core/types.js"; +import { + getCollectionMethods, + nsidForShortName, + resolveConfig, +} from "./core/types.js"; + +export interface PublicServiceOptions { + /** Canonical public HTTPS origin, for example `https://api.example.com`. */ + endpoint: string; +} + +export interface PublicServiceCollection { + alias: string; + nsid: string; + methods: string[]; + queryable: string[]; + searchable: string[]; + relations: string[]; + references: string[]; +} + +export interface PublicContract { + format: "contrail.contract"; + version: 1; + namespace: string; + collections: PublicServiceCollection[]; + methods: string[]; + lexiconDigest: string; +} + +export interface PublicServiceManifest { + format: "contrail.service"; + version: 1; + endpoint: string; + namespace: string; + contract: { digest: string }; + lexicons: { url: string; digest: string }; + status: { url: string }; + collections: PublicServiceCollection[]; + methods: string[]; +} + +export interface LexiconDocument { + lexicon?: number; + id: string; + [key: string]: unknown; +} + +export interface PublicServiceDescription { + endpoint: string; + lexicons: LexiconDocument[]; + manifest: PublicServiceManifest; + canonicalLexicons: string; +} + +export function normalizePublicServiceEndpoint(value: string): string { + const url = new URL(value); + if (url.protocol !== "https:") { + throw new Error("public service endpoint must use HTTPS"); + } + if (url.username || url.password) { + throw new Error("public service endpoint must not contain credentials"); + } + if (url.pathname !== "/" || url.search || url.hash) { + throw new Error( + "public service endpoint must be an origin without a path, query, or fragment", + ); + } + return url.origin; +} + +function normalizeJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(normalizeJson); + if (value && typeof value === "object") { + const result: Record = {}; + for (const key of Object.keys(value).sort()) { + const child = (value as Record)[key]; + if (child !== undefined) result[key] = normalizeJson(child); + } + return result; + } + return value; +} + +export function canonicalJson(value: unknown): string { + return JSON.stringify(normalizeJson(value)); +} + +export async function sha256(value: string): Promise { + const bytes = new TextEncoder().encode(value); + const digest = await crypto.subtle.digest("SHA-256", bytes); + return `sha256:${Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join("")}`; +} + +export function normalizeLexiconDocuments( + values: readonly object[], +): LexiconDocument[] { + const byId = new Map(); + for (const value of values) { + const id = (value as { id?: unknown }).id; + if (typeof id !== "string" || !isNsid(id)) { + throw new Error("public service lexicons must each have a valid NSID id"); + } + if (byId.has(id)) { + throw new Error(`duplicate public service lexicon: ${id}`); + } + byId.set(id, value as LexiconDocument); + } + return [...byId.values()].sort((a, b) => a.id.localeCompare(b.id)); +} + +function publicCollections(config: ContrailConfig): PublicServiceCollection[] { + return Object.keys(config.collections) + .sort() + .map((alias) => { + const collection = config.collections[alias]!; + const standardMethods = getCollectionMethods(collection).map( + (method) => `${config.namespace}.${alias}.${method}`, + ); + const customMethods = [ + ...Object.keys(collection.queries ?? {}), + ...Object.keys(collection.pipelineQueries ?? {}), + ].map((method) => `${config.namespace}.${alias}.${method}`); + return { + alias, + nsid: nsidForShortName(config, alias) ?? alias, + methods: [...new Set([...standardMethods, ...customMethods])].sort(), + queryable: Object.keys(collection.queryable ?? {}).sort(), + searchable: + collection.searchable === false + ? [] + : [...(collection.searchable ?? [])].sort(), + relations: Object.keys(collection.relations ?? {}).sort(), + references: Object.keys(collection.references ?? {}).sort(), + }; + }); +} + +function publicTopLevelMethods(config: ContrailConfig): string[] { + const methods = [`${config.namespace}.getCursor`]; + if ((config.profiles?.length ?? 0) > 0) { + methods.push(`${config.namespace}.getProfile`); + } + if (config.feeds && Object.keys(config.feeds).length > 0) { + methods.push(`${config.namespace}.getFeed`); + } + return methods; +} + +export function createPublicContract( + config: ContrailConfig, + lexiconDigest: string, +): PublicContract { + const resolved = resolveConfig(config); + const collections = publicCollections(resolved); + const methods = [ + ...publicTopLevelMethods(resolved), + ...collections.flatMap((collection) => collection.methods), + ]; + return { + format: "contrail.contract", + version: 1, + namespace: resolved.namespace, + collections, + methods: [...new Set(methods)].sort(), + lexiconDigest, + }; +} + +export async function digestPublicContract( + contract: PublicContract, +): Promise { + return sha256(canonicalJson(contract)); +} + +export function validateContractLexicons( + contract: PublicContract, + values: readonly object[], +): LexiconDocument[] { + const lexicons = normalizeLexiconDocuments(values); + if (lexicons.length === 0) { + throw new Error("public service requires a non-empty Lexicon bundle"); + } + const byId = new Map(lexicons.map((document) => [document.id, document])); + for (const method of contract.methods) { + const document = byId.get(method) as + { defs?: { main?: { type?: unknown } } } | undefined; + if (document?.defs?.main?.type !== "query") { + throw new Error( + `public method requires a matching query Lexicon: ${method}`, + ); + } + } + return lexicons; +} + +export function validatePublicServiceLexicons( + config: ContrailConfig, + values: readonly object[], +): LexiconDocument[] { + const placeholderDigest = `sha256:${"0".repeat(64)}`; + return validateContractLexicons( + createPublicContract(config, placeholderDigest), + values, + ); +} + +export async function digestLexiconDocuments( + values: readonly object[], +): Promise<{ + lexicons: LexiconDocument[]; + canonicalLexicons: string; + digest: string; +}> { + const lexicons = normalizeLexiconDocuments(values); + const canonicalLexicons = canonicalJson({ lexicons }); + return { + lexicons, + canonicalLexicons, + digest: await sha256(canonicalLexicons), + }; +} + +export async function describePublicService( + config: ContrailConfig, + options: PublicServiceOptions, + values: readonly object[], +): Promise { + const endpoint = normalizePublicServiceEndpoint(options.endpoint); + const { + lexicons, + canonicalLexicons, + digest: lexiconDigest, + } = await digestLexiconDocuments(values); + const contract = createPublicContract(config, lexiconDigest); + validateContractLexicons(contract, lexicons); + const manifest: PublicServiceManifest = { + format: "contrail.service", + version: 1, + endpoint, + namespace: contract.namespace, + contract: { digest: await digestPublicContract(contract) }, + lexicons: { + url: `${endpoint}/lexicons/${lexiconDigest}`, + digest: lexiconDigest, + }, + status: { url: `${endpoint}/status` }, + collections: contract.collections, + methods: contract.methods, + }; + return { endpoint, lexicons, manifest, canonicalLexicons }; +} + +function uniqueStrings(values: string[]): boolean { + return new Set(values).size === values.length; +} + +export function contractFromManifest( + manifest: PublicServiceManifest, +): PublicContract { + return { + format: "contrail.contract", + version: 1, + namespace: manifest.namespace, + collections: manifest.collections, + methods: manifest.methods, + lexiconDigest: manifest.lexicons.digest, + }; +} + +export function validateManifestContract( + manifest: PublicServiceManifest, + values: readonly object[], +): LexiconDocument[] { + if (!uniqueStrings(manifest.methods)) { + throw new Error("service manifest contains duplicate methods"); + } + const aliases = manifest.collections.map((collection) => collection.alias); + if (!uniqueStrings(aliases)) { + throw new Error("service manifest contains duplicate collection aliases"); + } + const prefix = `${manifest.namespace}.`; + if (manifest.methods.some((method) => !method.startsWith(prefix))) { + throw new Error("service manifest method is outside its namespace"); + } + const advertised = new Set(manifest.methods); + for (const collection of manifest.collections) { + if (!uniqueStrings(collection.methods)) { + throw new Error( + `service manifest collection ${collection.alias} contains duplicate methods`, + ); + } + for (const method of collection.methods) { + if (!advertised.has(method)) { + throw new Error( + `collection ${collection.alias} advertises an unknown method: ${method}`, + ); + } + } + } + return validateContractLexicons(contractFromManifest(manifest), values); +} + +export function isPublicServiceManifest( + value: unknown, +): value is PublicServiceManifest { + if (!value || typeof value !== "object") return false; + const manifest = value as Partial; + const digest = /^sha256:[0-9a-f]{64}$/; + if ( + manifest.format !== "contrail.service" || + manifest.version !== 1 || + typeof manifest.endpoint !== "string" || + typeof manifest.namespace !== "string" || + !isNsid(`${manifest.namespace}.method`) || + typeof manifest.contract?.digest !== "string" || + !digest.test(manifest.contract.digest) || + typeof manifest.lexicons?.url !== "string" || + typeof manifest.lexicons?.digest !== "string" || + !digest.test(manifest.lexicons.digest) || + typeof manifest.status?.url !== "string" || + !Array.isArray(manifest.collections) || + !Array.isArray(manifest.methods) || + !manifest.methods.every( + (method) => typeof method === "string" && isNsid(method), + ) + ) { + return false; + } + return manifest.collections.every((collection) => { + if (!collection || typeof collection !== "object") return false; + const entry = collection as Partial; + return ( + typeof entry.alias === "string" && + entry.alias.length > 0 && + typeof entry.nsid === "string" && + isNsid(entry.nsid) && + Array.isArray(entry.methods) && + entry.methods.every( + (method) => typeof method === "string" && isNsid(method), + ) && + [ + entry.queryable, + entry.searchable, + entry.relations, + entry.references, + ].every( + (items) => + Array.isArray(items) && + items.every((item) => typeof item === "string"), + ) + ); + }); +} diff --git a/packages/contrail/src/server.ts b/packages/contrail/src/server.ts index faf4d14..38b424b 100644 --- a/packages/contrail/src/server.ts +++ b/packages/contrail/src/server.ts @@ -1,10 +1,13 @@ import { Client } from "@atcute/client"; import type { Database } from "./core/types"; import type { Contrail } from "./contrail"; +import type { PublicServiceOptions } from "./public-service"; export interface CreateHandlerOptions { /** Bundled Lexicon documents exposed by the HTTP app. */ lexicons?: object[]; + /** Enable stable discovery and Lexicon routes for anonymous remote clients. */ + publicService?: PublicServiceOptions; } /** Create a fetch handler, optionally accepting a per-request database binding. */ @@ -15,9 +18,16 @@ export function createHandler( let cached: ((request: Request) => Promise) | null = null; return (request: Request, db?: Database) => { if (db) { - return contrail.handler({ db, lexicons: options.lexicons })(request); + return contrail.handler({ + db, + lexicons: options.lexicons, + publicService: options.publicService, + })(request); } - cached ??= contrail.handler({ lexicons: options.lexicons }); + cached ??= contrail.handler({ + lexicons: options.lexicons, + publicService: options.publicService, + }); return cached(request); }; } diff --git a/packages/contrail/src/worker/index.ts b/packages/contrail/src/worker/index.ts index 2d01d7a..151cb7f 100644 --- a/packages/contrail/src/worker/index.ts +++ b/packages/contrail/src/worker/index.ts @@ -18,12 +18,19 @@ import { Contrail } from "../contrail.js"; import { createHandler } from "../server.js"; import type { ContrailConfig, Database } from "../core/types.js"; import type { BackfillRetryOptions } from "../core/backfill.js"; +import { + normalizePublicServiceEndpoint, + validatePublicServiceLexicons, + type PublicServiceOptions, +} from "../public-service.js"; export interface CreateWorkerOptions { /** D1 binding name in wrangler env. Default: `"DB"`. */ binding?: string; /** Bundled Lexicon documents to expose for application type generation. */ lexicons?: object[]; + /** Enable stable discovery and Lexicon routes for anonymous remote clients. */ + publicService?: PublicServiceOptions; /** Bounded pending-account retry slice after each scheduled ingest. Enabled * by default; pass `false` to disable or options to tune its budget. */ backfillRetries?: BackfillRetryOptions | false; @@ -39,9 +46,14 @@ export function createWorker( options: CreateWorkerOptions = {} ) { const binding = options.binding ?? "DB"; + if (options.publicService) { + normalizePublicServiceEndpoint(options.publicService.endpoint); + validatePublicServiceLexicons(config, options.lexicons ?? []); + } const contrail = new Contrail(config); const handle = createHandler(contrail, { lexicons: options.lexicons, + publicService: options.publicService, }); let ready = false; -- 2.51.2 From aad30e2f640999605973a84fa6c5471d7115cb2a Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 04:52:20 +0200 Subject: [PATCH 04/15] Test public service contracts and consumers --- .changeset/public-read-service.md | 5 + README.md | 21 ++ .../com/example/event/getRecord.json | 69 +--- .../com/example/event/listRecords.json | 73 +---- .../generated/com/example/getCursor.json | 35 ++- .../generated/com/example/getOverview.json | 51 --- .../generated/com/example/getProfile.json | 69 +--- .../generated/com/example/notifyOfUpdate.json | 59 ---- .../com/example/profile/getRecord.json | 110 +++++++ .../com/example/profile/listRecords.json | 135 ++++++++ .../lexicons/generated/index.ts | 11 +- .../lexicons/generated/index.ts | 17 +- .../generated/statusphere/app/getCursor.json | 35 ++- .../statusphere/app/getOverview.json | 51 --- .../generated/statusphere/app/getProfile.json | 69 +--- .../statusphere/app/notifyOfUpdate.json | 59 ---- .../statusphere/app/profile/getRecord.json | 110 +++++++ .../statusphere/app/profile/listRecords.json | 135 ++++++++ .../statusphere/app/status/getRecord.json | 69 +--- .../statusphere/app/status/listRecords.json | 85 +---- .../src/lib/lexicons/index.ts | 4 +- .../types/statusphere/app/getCursor.ts | 26 +- .../types/statusphere/app/getOverview.ts | 46 --- .../types/statusphere/app/getProfile.ts | 100 +----- .../types/statusphere/app/notifyOfUpdate.ts | 63 ---- .../statusphere/app/profile/getRecord.ts | 74 +++++ .../statusphere/app/profile/listRecords.ts | 102 ++++++ .../types/statusphere/app/status/getRecord.ts | 91 +----- .../statusphere/app/status/listRecords.ts | 95 +----- docs/02-querying.md | 6 +- packages/contrail/README.md | 29 +- .../contrail/tests/backfill-status.test.ts | 9 +- packages/contrail/tests/connect.test.ts | 296 ++++++++++++++++++ .../tests/database-bootstrap-target.test.ts | 58 ++++ .../tests/jetstream-change-source.test.ts | 14 + .../contrail/tests/lexicon-generation.test.ts | 39 ++- packages/contrail/tests/persistent.test.ts | 21 +- .../contrail/tests/public-service-e2e.test.ts | 179 +++++++++++ .../tests/serving-source-position.test.ts | 75 +++++ .../contrail/tests/source-ordering.test.ts | 5 + packages/contrail/tests/worker.test.ts | 260 ++++++++++++++- 41 files changed, 1798 insertions(+), 1062 deletions(-) create mode 100644 .changeset/public-read-service.md delete mode 100644 apps/cloudflare-workers/lexicons/generated/com/example/getOverview.json delete mode 100644 apps/cloudflare-workers/lexicons/generated/com/example/notifyOfUpdate.json create mode 100644 apps/cloudflare-workers/lexicons/generated/com/example/profile/getRecord.json create mode 100644 apps/cloudflare-workers/lexicons/generated/com/example/profile/listRecords.json delete mode 100644 apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getOverview.json delete mode 100644 apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/notifyOfUpdate.json create mode 100644 apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/getRecord.json create mode 100644 apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/listRecords.json delete mode 100644 apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getOverview.ts delete mode 100644 apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/notifyOfUpdate.ts create mode 100644 apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/getRecord.ts create mode 100644 apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/listRecords.ts create mode 100644 packages/contrail/tests/connect.test.ts create mode 100644 packages/contrail/tests/public-service-e2e.test.ts create mode 100644 packages/contrail/tests/serving-source-position.test.ts diff --git a/.changeset/public-read-service.md b/.changeset/public-read-service.md new file mode 100644 index 0000000..a6bb312 --- /dev/null +++ b/.changeset/public-read-service.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Add self-describing anonymous read-through services with verified contracts, durable ordered-source positions, cacheable Lexicon discovery, and a safe `contrail connect` workflow for typed independent clients. diff --git a/README.md b/README.md index 6afad71..294b515 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,27 @@ pnpm contrail lexicons check Use `contrail lexicons all` to generate Contrail methods, pull referenced source Lexicons, and generate TypeScript types in one pass. The `pull` and `types` actions are also available separately. Contrail updates `lex.config.js` only when the file carries its generated marker; user-owned Atcute configuration is preserved. Pass `--no-atcute-config` to skip creating or checking that generated file. Contrail owns its config-specific query generation while delegating generic pulling and TypeScript generation to [Atcute](https://github.com/mary-ext/atcute). +## Public read-through services + +A deployment can publish a verified contract and Lexicon bundle for independent typed clients: + +```ts +export default createWorker(config, { + lexicons, + publicService: { endpoint: "https://api.example.com" }, +}); +``` + +Contrail remains a read-through cache over public AT Protocol data: anonymous reads may resolve identities, fetch missing public records, and improve profile or feed projections. Custom query handlers are public when they have matching authored query Lexicons. Anonymous discovery uses the HTTPS origin directly and does not require a service DID. The optional `notifyOfUpdate` procedure is not advertised in the anonymous read contract. + +Consumers connect and generate Atcute types with one command: + +```bash +pnpm contrail connect https://api.example.com +``` + +`getCursor` returns the committed opaque `{ source, epoch, cursor }` position of the primary ordered source. Compare complete positions for equality only; a source or epoch change requires a full client refetch. To avoid racing ingestion, read a position before and after a query and accept the query snapshot only when both positions match. + ## Other databases ```ts diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/event/getRecord.json b/apps/cloudflare-workers/lexicons/generated/com/example/event/getRecord.json index 47c29e1..5c73d83 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/event/getRecord.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/event/getRecord.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Get a single community.lexicon.calendar.event record by AT URI", + "description": "Get a community.lexicon.calendar.event record by AT URI", "parameters": { "type": "params", "required": [ @@ -18,7 +18,7 @@ }, "profiles": { "type": "boolean", - "description": "Include profile + identity info keyed by DID" + "description": "Include indexed profile and identity information" } } }, @@ -95,7 +95,7 @@ }, "value": { "type": "ref", - "ref": "#appBskyActorProfile" + "ref": "app.bsky.actor.profile#main" }, "collection": { "type": "string", @@ -105,69 +105,6 @@ "type": "string" } } - }, - "appBskyActorProfile": { - "type": "object", - "properties": { - "avatar": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" - }, - "banner": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Larger horizontal image to display behind profile view." - }, - "labels": { - "refs": [ - "com.atproto.label.defs#selfLabels" - ], - "type": "union", - "description": "Self-label values, specific to the Bluesky application, on the overall account." - }, - "website": { - "type": "string", - "format": "uri" - }, - "pronouns": { - "type": "string", - "maxLength": 200, - "description": "Free-form pronouns text.", - "maxGraphemes": 20 - }, - "createdAt": { - "type": "string", - "format": "datetime" - }, - "pinnedPost": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - }, - "description": { - "type": "string", - "maxLength": 2560, - "description": "Free-form profile description text.", - "maxGraphemes": 256 - }, - "displayName": { - "type": "string", - "maxLength": 640, - "maxGraphemes": 64 - }, - "joinedViaStarterPack": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - } - } } } } diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/event/listRecords.json b/apps/cloudflare-workers/lexicons/generated/com/example/event/listRecords.json index 3191ac3..e0e3496 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/event/listRecords.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/event/listRecords.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Query community.lexicon.calendar.event records with filters", + "description": "Query community.lexicon.calendar.event records", "parameters": { "type": "params", "properties": { @@ -20,11 +20,11 @@ "actor": { "type": "string", "format": "at-identifier", - "description": "Filter by DID or handle (triggers on-demand backfill)" + "description": "Filter by DID or handle" }, "profiles": { "type": "boolean", - "description": "Include profile + identity info keyed by DID" + "description": "Include indexed profile and identity information" }, "search": { "type": "string", @@ -51,7 +51,7 @@ "asc", "desc" ], - "description": "Sort direction (default: desc for dates/numbers/counts, asc for strings)" + "description": "Sort direction" } } }, @@ -147,7 +147,7 @@ }, "value": { "type": "ref", - "ref": "#appBskyActorProfile" + "ref": "app.bsky.actor.profile#main" }, "collection": { "type": "string", @@ -157,69 +157,6 @@ "type": "string" } } - }, - "appBskyActorProfile": { - "type": "object", - "properties": { - "avatar": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" - }, - "banner": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Larger horizontal image to display behind profile view." - }, - "labels": { - "refs": [ - "com.atproto.label.defs#selfLabels" - ], - "type": "union", - "description": "Self-label values, specific to the Bluesky application, on the overall account." - }, - "website": { - "type": "string", - "format": "uri" - }, - "pronouns": { - "type": "string", - "maxLength": 200, - "description": "Free-form pronouns text.", - "maxGraphemes": 20 - }, - "createdAt": { - "type": "string", - "format": "datetime" - }, - "pinnedPost": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - }, - "description": { - "type": "string", - "maxLength": 2560, - "description": "Free-form profile description text.", - "maxGraphemes": 256 - }, - "displayName": { - "type": "string", - "maxLength": 640, - "maxGraphemes": 64 - }, - "joinedViaStarterPack": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - } - } } } } diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json index aa95e2b..a5238d7 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json @@ -4,24 +4,45 @@ "defs": { "main": { "type": "query", - "description": "Get the current cursor position", + "description": "Get the committed primary ordered-source position", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "time_us": { - "type": "integer" - }, - "date": { - "type": "string" + "position": { + "type": "ref", + "ref": "#sourcePosition" }, - "seconds_ago": { + "updatedAt": { "type": "integer" + }, + "updatedAtDate": { + "type": "string", + "format": "datetime" } } } } + }, + "sourcePosition": { + "type": "object", + "required": [ + "source", + "epoch", + "cursor" + ], + "properties": { + "source": { + "type": "string" + }, + "epoch": { + "type": "string" + }, + "cursor": { + "type": "string" + } + } } } } diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/getOverview.json b/apps/cloudflare-workers/lexicons/generated/com/example/getOverview.json deleted file mode 100644 index 0617a37..0000000 --- a/apps/cloudflare-workers/lexicons/generated/com/example/getOverview.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "lexicon": 1, - "id": "com.example.getOverview", - "defs": { - "main": { - "type": "query", - "description": "Get an overview of all indexed collections", - "output": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": [ - "total_records", - "collections" - ], - "properties": { - "total_records": { - "type": "integer" - }, - "collections": { - "type": "array", - "items": { - "type": "ref", - "ref": "#collectionStats" - } - } - } - } - } - }, - "collectionStats": { - "type": "object", - "required": [ - "collection", - "records", - "unique_users" - ], - "properties": { - "collection": { - "type": "string" - }, - "records": { - "type": "integer" - }, - "unique_users": { - "type": "integer" - } - } - } - } -} diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/getProfile.json b/apps/cloudflare-workers/lexicons/generated/com/example/getProfile.json index 97e7134..22c367e 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/getProfile.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/getProfile.json @@ -4,7 +4,6 @@ "defs": { "main": { "type": "query", - "description": "Get a user's profiles by DID or handle", "parameters": { "type": "params", "required": [ @@ -13,8 +12,7 @@ "properties": { "actor": { "type": "string", - "format": "at-identifier", - "description": "DID or handle of the user" + "format": "at-identifier" } } }, @@ -60,7 +58,7 @@ }, "value": { "type": "ref", - "ref": "#appBskyActorProfile" + "ref": "app.bsky.actor.profile#main" }, "collection": { "type": "string", @@ -70,69 +68,6 @@ "type": "string" } } - }, - "appBskyActorProfile": { - "type": "object", - "properties": { - "avatar": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" - }, - "banner": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Larger horizontal image to display behind profile view." - }, - "labels": { - "refs": [ - "com.atproto.label.defs#selfLabels" - ], - "type": "union", - "description": "Self-label values, specific to the Bluesky application, on the overall account." - }, - "website": { - "type": "string", - "format": "uri" - }, - "pronouns": { - "type": "string", - "maxLength": 200, - "description": "Free-form pronouns text.", - "maxGraphemes": 20 - }, - "createdAt": { - "type": "string", - "format": "datetime" - }, - "pinnedPost": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - }, - "description": { - "type": "string", - "maxLength": 2560, - "description": "Free-form profile description text.", - "maxGraphemes": 256 - }, - "displayName": { - "type": "string", - "maxLength": 640, - "maxGraphemes": 64 - }, - "joinedViaStarterPack": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - } - } } } } diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/notifyOfUpdate.json b/apps/cloudflare-workers/lexicons/generated/com/example/notifyOfUpdate.json deleted file mode 100644 index 67be4b8..0000000 --- a/apps/cloudflare-workers/lexicons/generated/com/example/notifyOfUpdate.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "lexicon": 1, - "id": "com.example.notifyOfUpdate", - "defs": { - "main": { - "type": "procedure", - "description": "Notify of a record change for immediate indexing. Fetches the record from the user's PDS and indexes (or deletes) it.", - "input": { - "encoding": "application/json", - "schema": { - "type": "object", - "properties": { - "uri": { - "type": "string", - "format": "at-uri", - "description": "Single AT URI to fetch and index" - }, - "uris": { - "type": "array", - "items": { - "type": "string", - "format": "at-uri" - }, - "maxLength": 25, - "description": "Batch of AT URIs to fetch and index (max 25)" - } - } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": [ - "indexed", - "deleted" - ], - "properties": { - "indexed": { - "type": "integer", - "description": "Number of records created or updated" - }, - "deleted": { - "type": "integer", - "description": "Number of records deleted (not found on PDS)" - }, - "errors": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Errors for individual URIs that could not be processed" - } - } - } - } - } - } -} diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/profile/getRecord.json b/apps/cloudflare-workers/lexicons/generated/com/example/profile/getRecord.json new file mode 100644 index 0000000..473d663 --- /dev/null +++ b/apps/cloudflare-workers/lexicons/generated/com/example/profile/getRecord.json @@ -0,0 +1,110 @@ +{ + "lexicon": 1, + "id": "com.example.profile.getRecord", + "defs": { + "main": { + "type": "query", + "description": "Get a app.bsky.actor.profile record by AT URI", + "parameters": { + "type": "params", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT URI of the record" + }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } + } + } +} diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/profile/listRecords.json b/apps/cloudflare-workers/lexicons/generated/com/example/profile/listRecords.json new file mode 100644 index 0000000..4deea49 --- /dev/null +++ b/apps/cloudflare-workers/lexicons/generated/com/example/profile/listRecords.json @@ -0,0 +1,135 @@ +{ + "lexicon": 1, + "id": "com.example.profile.listRecords", + "defs": { + "main": { + "type": "query", + "description": "Query app.bsky.actor.profile records", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 50 + }, + "cursor": { + "type": "string" + }, + "actor": { + "type": "string", + "format": "at-identifier", + "description": "Filter by DID or handle" + }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "type": "ref", + "ref": "#record" + } + }, + "cursor": { + "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "record": { + "type": "object", + "required": [ + "uri", + "cid", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } + } + } +} diff --git a/apps/cloudflare-workers/lexicons/generated/index.ts b/apps/cloudflare-workers/lexicons/generated/index.ts index 28c9d90..4a326d7 100644 --- a/apps/cloudflare-workers/lexicons/generated/index.ts +++ b/apps/cloudflare-workers/lexicons/generated/index.ts @@ -1,6 +1,5 @@ -// Checked-in Lexicon bundle. -// Pass `lexicons` to `createWorker(config, { lexicons })` to expose them -// at `/xrpc/.lexicons` for consumer apps to typegen against. +// Auto-generated by @atmo-dev/contrail. Do not edit. +// Regenerate with `contrail lexicons generate`. import _0 from "../pulled/app/bsky/actor/profile.json"; import _1 from "../pulled/community/lexicon/calendar/event.json"; @@ -11,8 +10,8 @@ import _5 from "../pulled/community/lexicon/location/hthree.json"; import _6 from "./com/example/event/getRecord.json"; import _7 from "./com/example/event/listRecords.json"; import _8 from "./com/example/getCursor.json"; -import _9 from "./com/example/getOverview.json"; -import _10 from "./com/example/getProfile.json"; -import _11 from "./com/example/notifyOfUpdate.json"; +import _9 from "./com/example/getProfile.json"; +import _10 from "./com/example/profile/getRecord.json"; +import _11 from "./com/example/profile/listRecords.json"; export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11]; diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts b/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts index 7ffad08..562f2bd 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts @@ -1,14 +1,13 @@ -// Checked-in Lexicon bundle. -// Pass `lexicons` to `createWorker(config, { lexicons })` to expose them -// at `/xrpc/.lexicons` for consumer apps to typegen against. +// Auto-generated by @atmo-dev/contrail. Do not edit. +// Regenerate with `contrail lexicons generate`. import _0 from "../pulled/app/bsky/actor/profile.json"; import _1 from "../pulled/xyz/statusphere/status.json"; import _2 from "./statusphere/app/getCursor.json"; -import _3 from "./statusphere/app/getOverview.json"; -import _4 from "./statusphere/app/getProfile.json"; -import _5 from "./statusphere/app/notifyOfUpdate.json"; -import _7 from "./statusphere/app/status/getRecord.json"; -import _8 from "./statusphere/app/status/listRecords.json"; +import _3 from "./statusphere/app/getProfile.json"; +import _4 from "./statusphere/app/profile/getRecord.json"; +import _5 from "./statusphere/app/profile/listRecords.json"; +import _6 from "./statusphere/app/status/getRecord.json"; +import _7 from "./statusphere/app/status/listRecords.json"; -export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _7, _8]; +export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7]; diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json index f85bdbf..520e385 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json @@ -4,24 +4,45 @@ "defs": { "main": { "type": "query", - "description": "Get the current cursor position", + "description": "Get the committed primary ordered-source position", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "time_us": { - "type": "integer" - }, - "date": { - "type": "string" + "position": { + "type": "ref", + "ref": "#sourcePosition" }, - "seconds_ago": { + "updatedAt": { "type": "integer" + }, + "updatedAtDate": { + "type": "string", + "format": "datetime" } } } } + }, + "sourcePosition": { + "type": "object", + "required": [ + "source", + "epoch", + "cursor" + ], + "properties": { + "source": { + "type": "string" + }, + "epoch": { + "type": "string" + }, + "cursor": { + "type": "string" + } + } } } } diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getOverview.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getOverview.json deleted file mode 100644 index 67dd2f5..0000000 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getOverview.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "lexicon": 1, - "id": "statusphere.app.getOverview", - "defs": { - "main": { - "type": "query", - "description": "Get an overview of all indexed collections", - "output": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": [ - "total_records", - "collections" - ], - "properties": { - "total_records": { - "type": "integer" - }, - "collections": { - "type": "array", - "items": { - "type": "ref", - "ref": "#collectionStats" - } - } - } - } - } - }, - "collectionStats": { - "type": "object", - "required": [ - "collection", - "records", - "unique_users" - ], - "properties": { - "collection": { - "type": "string" - }, - "records": { - "type": "integer" - }, - "unique_users": { - "type": "integer" - } - } - } - } -} diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getProfile.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getProfile.json index 82732a4..5dbd415 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getProfile.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getProfile.json @@ -4,7 +4,6 @@ "defs": { "main": { "type": "query", - "description": "Get a user's profiles by DID or handle", "parameters": { "type": "params", "required": [ @@ -13,8 +12,7 @@ "properties": { "actor": { "type": "string", - "format": "at-identifier", - "description": "DID or handle of the user" + "format": "at-identifier" } } }, @@ -60,7 +58,7 @@ }, "value": { "type": "ref", - "ref": "#appBskyActorProfile" + "ref": "app.bsky.actor.profile#main" }, "collection": { "type": "string", @@ -70,69 +68,6 @@ "type": "string" } } - }, - "appBskyActorProfile": { - "type": "object", - "properties": { - "avatar": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" - }, - "banner": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Larger horizontal image to display behind profile view." - }, - "labels": { - "refs": [ - "com.atproto.label.defs#selfLabels" - ], - "type": "union", - "description": "Self-label values, specific to the Bluesky application, on the overall account." - }, - "website": { - "type": "string", - "format": "uri" - }, - "pronouns": { - "type": "string", - "maxLength": 200, - "description": "Free-form pronouns text.", - "maxGraphemes": 20 - }, - "createdAt": { - "type": "string", - "format": "datetime" - }, - "pinnedPost": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - }, - "description": { - "type": "string", - "maxLength": 2560, - "description": "Free-form profile description text.", - "maxGraphemes": 256 - }, - "displayName": { - "type": "string", - "maxLength": 640, - "maxGraphemes": 64 - }, - "joinedViaStarterPack": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - } - } } } } diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/notifyOfUpdate.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/notifyOfUpdate.json deleted file mode 100644 index f34ded3..0000000 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/notifyOfUpdate.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "lexicon": 1, - "id": "statusphere.app.notifyOfUpdate", - "defs": { - "main": { - "type": "procedure", - "description": "Notify of a record change for immediate indexing. Fetches the record from the user's PDS and indexes (or deletes) it.", - "input": { - "encoding": "application/json", - "schema": { - "type": "object", - "properties": { - "uri": { - "type": "string", - "format": "at-uri", - "description": "Single AT URI to fetch and index" - }, - "uris": { - "type": "array", - "items": { - "type": "string", - "format": "at-uri" - }, - "maxLength": 25, - "description": "Batch of AT URIs to fetch and index (max 25)" - } - } - } - }, - "output": { - "encoding": "application/json", - "schema": { - "type": "object", - "required": [ - "indexed", - "deleted" - ], - "properties": { - "indexed": { - "type": "integer", - "description": "Number of records created or updated" - }, - "deleted": { - "type": "integer", - "description": "Number of records deleted (not found on PDS)" - }, - "errors": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Errors for individual URIs that could not be processed" - } - } - } - } - } - } -} diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/getRecord.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/getRecord.json new file mode 100644 index 0000000..20d799c --- /dev/null +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/getRecord.json @@ -0,0 +1,110 @@ +{ + "lexicon": 1, + "id": "statusphere.app.profile.getRecord", + "defs": { + "main": { + "type": "query", + "description": "Get a app.bsky.actor.profile record by AT URI", + "parameters": { + "type": "params", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT URI of the record" + }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } + } + } +} diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/listRecords.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/listRecords.json new file mode 100644 index 0000000..bae9f79 --- /dev/null +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/profile/listRecords.json @@ -0,0 +1,135 @@ +{ + "lexicon": 1, + "id": "statusphere.app.profile.listRecords", + "defs": { + "main": { + "type": "query", + "description": "Query app.bsky.actor.profile records", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 50 + }, + "cursor": { + "type": "string" + }, + "actor": { + "type": "string", + "format": "at-identifier", + "description": "Filter by DID or handle" + }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "type": "ref", + "ref": "#record" + } + }, + "cursor": { + "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "record": { + "type": "object", + "required": [ + "uri", + "cid", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } + } + } +} diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/getRecord.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/getRecord.json index c866e5e..99d2454 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/getRecord.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/getRecord.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Get a single xyz.statusphere.status record by AT URI", + "description": "Get a xyz.statusphere.status record by AT URI", "parameters": { "type": "params", "required": [ @@ -18,7 +18,7 @@ }, "profiles": { "type": "boolean", - "description": "Include profile + identity info keyed by DID" + "description": "Include indexed profile and identity information" } } }, @@ -95,7 +95,7 @@ }, "value": { "type": "ref", - "ref": "#appBskyActorProfile" + "ref": "app.bsky.actor.profile#main" }, "collection": { "type": "string", @@ -105,69 +105,6 @@ "type": "string" } } - }, - "appBskyActorProfile": { - "type": "object", - "properties": { - "avatar": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" - }, - "banner": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Larger horizontal image to display behind profile view." - }, - "labels": { - "refs": [ - "com.atproto.label.defs#selfLabels" - ], - "type": "union", - "description": "Self-label values, specific to the Bluesky application, on the overall account." - }, - "website": { - "type": "string", - "format": "uri" - }, - "pronouns": { - "type": "string", - "maxLength": 200, - "description": "Free-form pronouns text.", - "maxGraphemes": 20 - }, - "createdAt": { - "type": "string", - "format": "datetime" - }, - "pinnedPost": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - }, - "description": { - "type": "string", - "maxLength": 2560, - "description": "Free-form profile description text.", - "maxGraphemes": 256 - }, - "displayName": { - "type": "string", - "maxLength": 640, - "maxGraphemes": 64 - }, - "joinedViaStarterPack": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - } - } } } } diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/listRecords.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/listRecords.json index 762f83f..70eef40 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/listRecords.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/status/listRecords.json @@ -4,7 +4,7 @@ "defs": { "main": { "type": "query", - "description": "Query xyz.statusphere.status records with filters", + "description": "Query xyz.statusphere.status records", "parameters": { "type": "params", "properties": { @@ -20,15 +20,11 @@ "actor": { "type": "string", "format": "at-identifier", - "description": "Filter by DID or handle (triggers on-demand backfill)" + "description": "Filter by DID or handle" }, "profiles": { "type": "boolean", - "description": "Include profile + identity info keyed by DID" - }, - "status": { - "type": "string", - "description": "Filter by status" + "description": "Include indexed profile and identity information" }, "createdAtMin": { "type": "string", @@ -38,11 +34,15 @@ "type": "string", "description": "Maximum value for createdAt" }, + "status": { + "type": "string", + "description": "Filter by status" + }, "sort": { "type": "string", "knownValues": [ - "status", - "createdAt" + "createdAt", + "status" ], "description": "Field to sort by (default: time_us)" }, @@ -52,7 +52,7 @@ "asc", "desc" ], - "description": "Sort direction (default: desc for dates/numbers/counts, asc for strings)" + "description": "Sort direction" } } }, @@ -148,7 +148,7 @@ }, "value": { "type": "ref", - "ref": "#appBskyActorProfile" + "ref": "app.bsky.actor.profile#main" }, "collection": { "type": "string", @@ -158,69 +158,6 @@ "type": "string" } } - }, - "appBskyActorProfile": { - "type": "object", - "properties": { - "avatar": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" - }, - "banner": { - "type": "blob", - "accept": [ - "image/png", - "image/jpeg" - ], - "maxSize": 1000000, - "description": "Larger horizontal image to display behind profile view." - }, - "labels": { - "refs": [ - "com.atproto.label.defs#selfLabels" - ], - "type": "union", - "description": "Self-label values, specific to the Bluesky application, on the overall account." - }, - "website": { - "type": "string", - "format": "uri" - }, - "pronouns": { - "type": "string", - "maxLength": 200, - "description": "Free-form pronouns text.", - "maxGraphemes": 20 - }, - "createdAt": { - "type": "string", - "format": "datetime" - }, - "pinnedPost": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - }, - "description": { - "type": "string", - "maxLength": 2560, - "description": "Free-form profile description text.", - "maxGraphemes": 256 - }, - "displayName": { - "type": "string", - "maxLength": 640, - "maxGraphemes": 64 - }, - "joinedViaStarterPack": { - "ref": "com.atproto.repo.strongRef", - "type": "ref" - } - } } } } diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/index.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/index.ts index 48c9760..9f81202 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/index.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/index.ts @@ -1,7 +1,7 @@ export * as StatusphereAppGetCursor from "./types/statusphere/app/getCursor.js"; -export * as StatusphereAppGetOverview from "./types/statusphere/app/getOverview.js"; export * as StatusphereAppGetProfile from "./types/statusphere/app/getProfile.js"; -export * as StatusphereAppNotifyOfUpdate from "./types/statusphere/app/notifyOfUpdate.js"; +export * as StatusphereAppProfileGetRecord from "./types/statusphere/app/profile/getRecord.js"; +export * as StatusphereAppProfileListRecords from "./types/statusphere/app/profile/listRecords.js"; export * as StatusphereAppStatusGetRecord from "./types/statusphere/app/status/getRecord.js"; export * as StatusphereAppStatusListRecords from "./types/statusphere/app/status/listRecords.js"; export * as XyzStatusphereStatus from "./types/xyz/statusphere/status.js"; diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts index 05a42fc..c2255ab 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts @@ -8,18 +8,34 @@ const _mainSchema = /*#__PURE__*/ v.query( "params": null, "output": { "type": "lex", - "schema": /*#__PURE__*/ v.object({ - "date": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), - "seconds_ago": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), - "time_us": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), - }), + "schema": /*#__PURE__*/ v.object( + { + get "position"() { + return /*#__PURE__*/ v.optional(sourcePositionSchema) + }, + "updatedAt": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + "updatedAtDate": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.datetimeString()), + } + ), } } ); +const _sourcePositionSchema = /*#__PURE__*/ v.object({ + "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.getCursor#sourcePosition")), + "cursor": /*#__PURE__*/ v.string(), + "epoch": /*#__PURE__*/ v.string(), + "source": /*#__PURE__*/ v.string(), +}); type main$schematype = typeof _mainSchema; +type sourcePosition$schematype = typeof _sourcePositionSchema; export interface mainSchema extends main$schematype {} + +export interface sourcePositionSchema extends sourcePosition$schematype {} export const mainSchema = _mainSchema as mainSchema; +export const sourcePositionSchema = _sourcePositionSchema as sourcePositionSchema; + +export interface SourcePosition extends v.InferInput {} export interface $params {} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getOverview.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getOverview.ts deleted file mode 100644 index e058174..0000000 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getOverview.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type {} from '@atcute/lexicons'; -import * as v from '@atcute/lexicons/validations'; -import type {} from '@atcute/lexicons/ambient'; - -const _collectionStatsSchema = /*#__PURE__*/ v.object({ - "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.getOverview#collectionStats")), - "collection": /*#__PURE__*/ v.string(), - "records": /*#__PURE__*/ v.integer(), - "unique_users": /*#__PURE__*/ v.integer(), -}); -const _mainSchema = /*#__PURE__*/ v.query( - "statusphere.app.getOverview", - { - "params": null, - "output": { - "type": "lex", - "schema": /*#__PURE__*/ v.object( - { - get "collections"() { - return /*#__PURE__*/ v.array(collectionStatsSchema) - }, - "total_records": /*#__PURE__*/ v.integer(), - } - ), - } - } -); -type collectionStats$schematype = typeof _collectionStatsSchema; -type main$schematype = typeof _mainSchema; - -export interface collectionStatsSchema extends collectionStats$schematype {} - -export interface mainSchema extends main$schematype {} -export const collectionStatsSchema = _collectionStatsSchema as collectionStatsSchema; -export const mainSchema = _mainSchema as mainSchema; - -export interface CollectionStats extends v.InferInput {} - -export interface $params {} - -export interface $output extends v.InferXRPCBodyInput {} -declare module '@atcute/lexicons/ambient' { - interface XRPCQueries { - "statusphere.app.getOverview": mainSchema; - } -} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getProfile.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getProfile.ts index 7597288..4bcad8f 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getProfile.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getProfile.ts @@ -1,98 +1,14 @@ import type {} from '@atcute/lexicons'; import * as v from '@atcute/lexicons/validations'; import type {} from '@atcute/lexicons/ambient'; -import * as ComAtprotoLabelDefs from "@atcute/atproto/types/label/defs"; -import * as ComAtprotoRepoStrongRef from "@atcute/atproto/types/repo/strongRef"; +import * as AppBskyActorProfile from "@atcute/bluesky/types/app/actor/profile"; -const _appBskyActorProfileSchema = /*#__PURE__*/ v.object( - { - "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.getProfile#appBskyActorProfile")), - /** - * Small image to be displayed next to posts from account. AKA, 'profile picture' - * @accept image/png, image/jpeg - * @maxSize 1000000 - */ - "avatar": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.blob(), - [ - /*#__PURE__*/ v.blobSize(1000000), - /*#__PURE__*/ v.blobAccept(["image/png", "image/jpeg"]) - ] - )), - /** - * Larger horizontal image to display behind profile view. - * @accept image/png, image/jpeg - * @maxSize 1000000 - */ - "banner": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.blob(), - [ - /*#__PURE__*/ v.blobSize(1000000), - /*#__PURE__*/ v.blobAccept(["image/png", "image/jpeg"]) - ] - )), - "createdAt": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.datetimeString()), - /** - * Free-form profile description text. - * @maxLength 2560 - * @maxGraphemes 256 - */ - "description": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 2560), - /*#__PURE__*/ v.stringGraphemes(0, 256) - ] - )), - /** - * @maxLength 640 - * @maxGraphemes 64 - */ - "displayName": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 640), - /*#__PURE__*/ v.stringGraphemes(0, 64) - ] - )), - get "joinedViaStarterPack"() { - return /*#__PURE__*/ v.optional(ComAtprotoRepoStrongRef.mainSchema) - }, - /** - * Self-label values, specific to the Bluesky application, on the overall account. - */ - get "labels"() { - return /*#__PURE__*/ v.optional(/*#__PURE__*/ v.variant([ComAtprotoLabelDefs.selfLabelsSchema])) - }, - get "pinnedPost"() { - return /*#__PURE__*/ v.optional(ComAtprotoRepoStrongRef.mainSchema) - }, - /** - * Free-form pronouns text. - * @maxLength 200 - * @maxGraphemes 20 - */ - "pronouns": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 200), - /*#__PURE__*/ v.stringGraphemes(0, 20) - ] - )), - "website": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.genericUriString()), - } -); const _mainSchema = /*#__PURE__*/ v.query( "statusphere.app.getProfile", { - "params": /*#__PURE__*/ v.object( - { - /** - * DID or handle of the user - */ - "actor": /*#__PURE__*/ v.actorIdentifierString(), - } - ), + "params": /*#__PURE__*/ v.object({ + "actor": /*#__PURE__*/ v.actorIdentifierString(), + }), "output": { "type": "lex", "schema": /*#__PURE__*/ v.object( @@ -115,25 +31,19 @@ const _profileEntrySchema = /*#__PURE__*/ v.object( "rkey": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), "uri": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.resourceUriString()), get "value"() { - return /*#__PURE__*/ v.optional(appBskyActorProfileSchema) + return /*#__PURE__*/ v.optional(AppBskyActorProfile.mainSchema) }, } ); -type appBskyActorProfile$schematype = typeof _appBskyActorProfileSchema; type main$schematype = typeof _mainSchema; type profileEntry$schematype = typeof _profileEntrySchema; -export interface appBskyActorProfileSchema extends appBskyActorProfile$schematype {} - export interface mainSchema extends main$schematype {} export interface profileEntrySchema extends profileEntry$schematype {} -export const appBskyActorProfileSchema = _appBskyActorProfileSchema as appBskyActorProfileSchema; export const mainSchema = _mainSchema as mainSchema; export const profileEntrySchema = _profileEntrySchema as profileEntrySchema; -export interface AppBskyActorProfile extends v.InferInput {} - export interface ProfileEntry extends v.InferInput {} export interface $params extends v.InferInput {} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/notifyOfUpdate.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/notifyOfUpdate.ts deleted file mode 100644 index e7c784c..0000000 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/notifyOfUpdate.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type {} from '@atcute/lexicons'; -import * as v from '@atcute/lexicons/validations'; -import type {} from '@atcute/lexicons/ambient'; - -const _mainSchema = /*#__PURE__*/ v.procedure( - "statusphere.app.notifyOfUpdate", - { - "params": null, - "input": { - "type": "lex", - "schema": /*#__PURE__*/ v.object( - { - /** - * Single AT URI to fetch and index - */ - "uri": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.resourceUriString()), - /** - * Batch of AT URIs to fetch and index (max 25) - * @maxLength 25 - */ - "uris": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.array(/*#__PURE__*/ v.resourceUriString()), - [/*#__PURE__*/ v.arrayLength(0, 25)] - )), - } - ), - }, - "output": { - "type": "lex", - "schema": /*#__PURE__*/ v.object( - { - /** - * Number of records deleted (not found on PDS) - */ - "deleted": /*#__PURE__*/ v.integer(), - /** - * Errors for individual URIs that could not be processed - */ - "errors": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.array(/*#__PURE__*/ v.string())), - /** - * Number of records created or updated - */ - "indexed": /*#__PURE__*/ v.integer(), - } - ), - } - } -); -type main$schematype = typeof _mainSchema; - -export interface mainSchema extends main$schematype {} -export const mainSchema = _mainSchema as mainSchema; - -export interface $params {} - -export interface $input extends v.InferXRPCBodyInput {} - -export interface $output extends v.InferXRPCBodyInput {} -declare module '@atcute/lexicons/ambient' { - interface XRPCProcedures { - "statusphere.app.notifyOfUpdate": mainSchema; - } -} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/getRecord.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/getRecord.ts new file mode 100644 index 0000000..3ba2a7f --- /dev/null +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/getRecord.ts @@ -0,0 +1,74 @@ +import type {} from '@atcute/lexicons'; +import * as v from '@atcute/lexicons/validations'; +import type {} from '@atcute/lexicons/ambient'; +import * as AppBskyActorProfile from "@atcute/bluesky/types/app/actor/profile"; + +const _mainSchema = /*#__PURE__*/ v.query( + "statusphere.app.profile.getRecord", + { + "params": /*#__PURE__*/ v.object( + { + /** + * Include indexed profile and identity information + */ + "profiles": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.boolean()), + /** + * AT URI of the record + */ + "uri": /*#__PURE__*/ v.resourceUriString(), + } + ), + "output": { + "type": "lex", + "schema": /*#__PURE__*/ v.object( + { + "cid": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.cidString()), + "collection": /*#__PURE__*/ v.nsidString(), + "did": /*#__PURE__*/ v.didString(), + get "profiles"() { + return /*#__PURE__*/ v.optional(/*#__PURE__*/ v.array(profileEntrySchema)) + }, + "rkey": /*#__PURE__*/ v.string(), + "time_us": /*#__PURE__*/ v.integer(), + "uri": /*#__PURE__*/ v.resourceUriString(), + get "value"() { + return AppBskyActorProfile.mainSchema + }, + } + ), + } + } +); +const _profileEntrySchema = /*#__PURE__*/ v.object( + { + "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.profile.getRecord#profileEntry")), + "cid": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.cidString()), + "collection": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.nsidString()), + "did": /*#__PURE__*/ v.didString(), + "handle": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + "rkey": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + "uri": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.resourceUriString()), + get "value"() { + return /*#__PURE__*/ v.optional(AppBskyActorProfile.mainSchema) + }, + } +); +type main$schematype = typeof _mainSchema; +type profileEntry$schematype = typeof _profileEntrySchema; + +export interface mainSchema extends main$schematype {} + +export interface profileEntrySchema extends profileEntry$schematype {} +export const mainSchema = _mainSchema as mainSchema; +export const profileEntrySchema = _profileEntrySchema as profileEntrySchema; + +export interface ProfileEntry extends v.InferInput {} + +export interface $params extends v.InferInput {} + +export interface $output extends v.InferXRPCBodyInput {} +declare module '@atcute/lexicons/ambient' { + interface XRPCQueries { + "statusphere.app.profile.getRecord": mainSchema; + } +} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/listRecords.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/listRecords.ts new file mode 100644 index 0000000..df0f714 --- /dev/null +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/profile/listRecords.ts @@ -0,0 +1,102 @@ +import type {} from '@atcute/lexicons'; +import * as v from '@atcute/lexicons/validations'; +import type {} from '@atcute/lexicons/ambient'; +import * as AppBskyActorProfile from "@atcute/bluesky/types/app/actor/profile"; + +const _mainSchema = /*#__PURE__*/ v.query( + "statusphere.app.profile.listRecords", + { + "params": /*#__PURE__*/ v.object( + { + /** + * Filter by DID or handle + */ + "actor": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.actorIdentifierString()), + "cursor": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + /** + * @minimum 1 + * @maximum 200 + * @default 50 + */ + "limit": /*#__PURE__*/ v.optional( + /*#__PURE__*/ v.constrain( + /*#__PURE__*/ v.integer(), + [/*#__PURE__*/ v.integerRange(1, 200)] + ), + 50 + ), + /** + * Include indexed profile and identity information + */ + "profiles": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.boolean()), + } + ), + "output": { + "type": "lex", + "schema": /*#__PURE__*/ v.object( + { + "cursor": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + get "profiles"() { + return /*#__PURE__*/ v.optional(/*#__PURE__*/ v.array(profileEntrySchema)) + }, + get "records"() { + return /*#__PURE__*/ v.array(recordSchema) + }, + } + ), + } + } +); +const _profileEntrySchema = /*#__PURE__*/ v.object( + { + "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.profile.listRecords#profileEntry")), + "cid": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.cidString()), + "collection": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.nsidString()), + "did": /*#__PURE__*/ v.didString(), + "handle": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + "rkey": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + "uri": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.resourceUriString()), + get "value"() { + return /*#__PURE__*/ v.optional(AppBskyActorProfile.mainSchema) + }, + } +); +const _recordSchema = /*#__PURE__*/ v.object( + { + "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.profile.listRecords#record")), + "cid": /*#__PURE__*/ v.cidString(), + "collection": /*#__PURE__*/ v.nsidString(), + "did": /*#__PURE__*/ v.didString(), + "rkey": /*#__PURE__*/ v.string(), + "time_us": /*#__PURE__*/ v.integer(), + "uri": /*#__PURE__*/ v.resourceUriString(), + get "value"() { + return AppBskyActorProfile.mainSchema + }, + } +); +type main$schematype = typeof _mainSchema; +type profileEntry$schematype = typeof _profileEntrySchema; +type record$schematype = typeof _recordSchema; + +export interface mainSchema extends main$schematype {} + +export interface profileEntrySchema extends profileEntry$schematype {} + +export interface recordSchema extends record$schematype {} +export const mainSchema = _mainSchema as mainSchema; +export const profileEntrySchema = _profileEntrySchema as profileEntrySchema; +export const recordSchema = _recordSchema as recordSchema; + +export interface ProfileEntry extends v.InferInput {} + +export interface Record extends v.InferInput {} + +export interface $params extends v.InferInput {} + +export interface $output extends v.InferXRPCBodyInput {} +declare module '@atcute/lexicons/ambient' { + interface XRPCQueries { + "statusphere.app.profile.listRecords": mainSchema; + } +} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/getRecord.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/getRecord.ts index 836d45f..f923374 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/getRecord.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/getRecord.ts @@ -1,95 +1,16 @@ import type {} from '@atcute/lexicons'; import * as v from '@atcute/lexicons/validations'; import type {} from '@atcute/lexicons/ambient'; -import * as ComAtprotoLabelDefs from "@atcute/atproto/types/label/defs"; -import * as ComAtprotoRepoStrongRef from "@atcute/atproto/types/repo/strongRef"; +import * as AppBskyActorProfile from "@atcute/bluesky/types/app/actor/profile"; import * as XyzStatusphereStatus from "../../../xyz/statusphere/status.js"; -const _appBskyActorProfileSchema = /*#__PURE__*/ v.object( - { - "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.status.getRecord#appBskyActorProfile")), - /** - * Small image to be displayed next to posts from account. AKA, 'profile picture' - * @accept image/png, image/jpeg - * @maxSize 1000000 - */ - "avatar": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.blob(), - [ - /*#__PURE__*/ v.blobSize(1000000), - /*#__PURE__*/ v.blobAccept(["image/png", "image/jpeg"]) - ] - )), - /** - * Larger horizontal image to display behind profile view. - * @accept image/png, image/jpeg - * @maxSize 1000000 - */ - "banner": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.blob(), - [ - /*#__PURE__*/ v.blobSize(1000000), - /*#__PURE__*/ v.blobAccept(["image/png", "image/jpeg"]) - ] - )), - "createdAt": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.datetimeString()), - /** - * Free-form profile description text. - * @maxLength 2560 - * @maxGraphemes 256 - */ - "description": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 2560), - /*#__PURE__*/ v.stringGraphemes(0, 256) - ] - )), - /** - * @maxLength 640 - * @maxGraphemes 64 - */ - "displayName": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 640), - /*#__PURE__*/ v.stringGraphemes(0, 64) - ] - )), - get "joinedViaStarterPack"() { - return /*#__PURE__*/ v.optional(ComAtprotoRepoStrongRef.mainSchema) - }, - /** - * Self-label values, specific to the Bluesky application, on the overall account. - */ - get "labels"() { - return /*#__PURE__*/ v.optional(/*#__PURE__*/ v.variant([ComAtprotoLabelDefs.selfLabelsSchema])) - }, - get "pinnedPost"() { - return /*#__PURE__*/ v.optional(ComAtprotoRepoStrongRef.mainSchema) - }, - /** - * Free-form pronouns text. - * @maxLength 200 - * @maxGraphemes 20 - */ - "pronouns": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 200), - /*#__PURE__*/ v.stringGraphemes(0, 20) - ] - )), - "website": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.genericUriString()), - } -); const _mainSchema = /*#__PURE__*/ v.query( "statusphere.app.status.getRecord", { "params": /*#__PURE__*/ v.object( { /** - * Include profile + identity info keyed by DID + * Include indexed profile and identity information */ "profiles": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.boolean()), /** @@ -129,25 +50,19 @@ const _profileEntrySchema = /*#__PURE__*/ v.object( "rkey": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), "uri": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.resourceUriString()), get "value"() { - return /*#__PURE__*/ v.optional(appBskyActorProfileSchema) + return /*#__PURE__*/ v.optional(AppBskyActorProfile.mainSchema) }, } ); -type appBskyActorProfile$schematype = typeof _appBskyActorProfileSchema; type main$schematype = typeof _mainSchema; type profileEntry$schematype = typeof _profileEntrySchema; -export interface appBskyActorProfileSchema extends appBskyActorProfile$schematype {} - export interface mainSchema extends main$schematype {} export interface profileEntrySchema extends profileEntry$schematype {} -export const appBskyActorProfileSchema = _appBskyActorProfileSchema as appBskyActorProfileSchema; export const mainSchema = _mainSchema as mainSchema; export const profileEntrySchema = _profileEntrySchema as profileEntrySchema; -export interface AppBskyActorProfile extends v.InferInput {} - export interface ProfileEntry extends v.InferInput {} export interface $params extends v.InferInput {} diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/listRecords.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/listRecords.ts index 207312c..467fe66 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/listRecords.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/status/listRecords.ts @@ -1,95 +1,16 @@ import type {} from '@atcute/lexicons'; import * as v from '@atcute/lexicons/validations'; import type {} from '@atcute/lexicons/ambient'; -import * as ComAtprotoLabelDefs from "@atcute/atproto/types/label/defs"; -import * as ComAtprotoRepoStrongRef from "@atcute/atproto/types/repo/strongRef"; +import * as AppBskyActorProfile from "@atcute/bluesky/types/app/actor/profile"; import * as XyzStatusphereStatus from "../../../xyz/statusphere/status.js"; -const _appBskyActorProfileSchema = /*#__PURE__*/ v.object( - { - "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.status.listRecords#appBskyActorProfile")), - /** - * Small image to be displayed next to posts from account. AKA, 'profile picture' - * @accept image/png, image/jpeg - * @maxSize 1000000 - */ - "avatar": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.blob(), - [ - /*#__PURE__*/ v.blobSize(1000000), - /*#__PURE__*/ v.blobAccept(["image/png", "image/jpeg"]) - ] - )), - /** - * Larger horizontal image to display behind profile view. - * @accept image/png, image/jpeg - * @maxSize 1000000 - */ - "banner": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.blob(), - [ - /*#__PURE__*/ v.blobSize(1000000), - /*#__PURE__*/ v.blobAccept(["image/png", "image/jpeg"]) - ] - )), - "createdAt": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.datetimeString()), - /** - * Free-form profile description text. - * @maxLength 2560 - * @maxGraphemes 256 - */ - "description": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 2560), - /*#__PURE__*/ v.stringGraphemes(0, 256) - ] - )), - /** - * @maxLength 640 - * @maxGraphemes 64 - */ - "displayName": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 640), - /*#__PURE__*/ v.stringGraphemes(0, 64) - ] - )), - get "joinedViaStarterPack"() { - return /*#__PURE__*/ v.optional(ComAtprotoRepoStrongRef.mainSchema) - }, - /** - * Self-label values, specific to the Bluesky application, on the overall account. - */ - get "labels"() { - return /*#__PURE__*/ v.optional(/*#__PURE__*/ v.variant([ComAtprotoLabelDefs.selfLabelsSchema])) - }, - get "pinnedPost"() { - return /*#__PURE__*/ v.optional(ComAtprotoRepoStrongRef.mainSchema) - }, - /** - * Free-form pronouns text. - * @maxLength 200 - * @maxGraphemes 20 - */ - "pronouns": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.constrain( - /*#__PURE__*/ v.string(), - [ - /*#__PURE__*/ v.stringLength(0, 200), - /*#__PURE__*/ v.stringGraphemes(0, 20) - ] - )), - "website": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.genericUriString()), - } -); const _mainSchema = /*#__PURE__*/ v.query( "statusphere.app.status.listRecords", { "params": /*#__PURE__*/ v.object( { /** - * Filter by DID or handle (triggers on-demand backfill) + * Filter by DID or handle */ "actor": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.actorIdentifierString()), /** @@ -114,11 +35,11 @@ const _mainSchema = /*#__PURE__*/ v.query( 50 ), /** - * Sort direction (default: desc for dates/numbers/counts, asc for strings) + * Sort direction */ "order": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string<"asc" | "desc" | (string & {})>()), /** - * Include profile + identity info keyed by DID + * Include indexed profile and identity information */ "profiles": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.boolean()), /** @@ -157,7 +78,7 @@ const _profileEntrySchema = /*#__PURE__*/ v.object( "rkey": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), "uri": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.resourceUriString()), get "value"() { - return /*#__PURE__*/ v.optional(appBskyActorProfileSchema) + return /*#__PURE__*/ v.optional(AppBskyActorProfile.mainSchema) }, } ); @@ -175,25 +96,19 @@ const _recordSchema = /*#__PURE__*/ v.object( }, } ); -type appBskyActorProfile$schematype = typeof _appBskyActorProfileSchema; type main$schematype = typeof _mainSchema; type profileEntry$schematype = typeof _profileEntrySchema; type record$schematype = typeof _recordSchema; -export interface appBskyActorProfileSchema extends appBskyActorProfile$schematype {} - export interface mainSchema extends main$schematype {} export interface profileEntrySchema extends profileEntry$schematype {} export interface recordSchema extends record$schematype {} -export const appBskyActorProfileSchema = _appBskyActorProfileSchema as appBskyActorProfileSchema; export const mainSchema = _mainSchema as mainSchema; export const profileEntrySchema = _profileEntrySchema as profileEntrySchema; export const recordSchema = _recordSchema as recordSchema; -export interface AppBskyActorProfile extends v.InferInput {} - export interface ProfileEntry extends v.InferInput {} export interface Record extends v.InferInput {} diff --git a/docs/02-querying.md b/docs/02-querying.md index 6997372..46dac29 100644 --- a/docs/02-querying.md +++ b/docs/02-querying.md @@ -7,7 +7,7 @@ Once [indexing](./01-indexing.md) is set up, every collection you declared gets | `{namespace}.{short}.listRecords` | Paginated list with filters, sorts, hydration | | `{namespace}.{short}.getRecord?uri=…` | Single record by AT-URI | -Plus a few top-level ones: `{namespace}.getProfile`, `{namespace}.getCursor`, `{namespace}.getOverview`, `{namespace}.notifyOfUpdate`, and optionally `{namespace}.lexicons`. +Top-level methods include `{namespace}.getProfile`, `{namespace}.getCursor`, `{namespace}.notifyOfUpdate`, and optionally `{namespace}.getFeed` and `{namespace}.lexicons`. ## HTTP (what most callers use) @@ -30,7 +30,7 @@ Dotted field names become camelCase params — `queryable: { "subject.uri": {} } ## Operational status -`GET /status` and `GET /xrpc/{namespace}.getOverview` return the current JSON overview. It includes indexed record totals, the live-ingest cursor and lag, and durable backfill state: +`GET /status` returns the current JSON overview. It includes indexed record totals, live-ingest freshness, and durable backfill state: - discovery source progress; - mutually exclusive account totals for `complete`, `pending`, `retrying`, and `failed`; @@ -42,6 +42,8 @@ Dotted field names become camelCase params — `queryable: { "subject.uri": {} } "Known" is deliberate: while relay discovery is incomplete, Contrail cannot honestly claim how many accounts remain undiscovered. `/health` remains a lightweight liveness response and does not claim that historical backfill is complete. +`GET /xrpc/{namespace}.getCursor` returns the committed primary ordered-source position when `orderedSource` is configured. The `{ source, epoch, cursor }` tuple is opaque: compare complete tuples for equality only, and treat a source or epoch change as a full reset. A consumer that needs a stable query snapshot can read the position before and after its query and retry when the two positions differ. + ## Programmatic ```ts diff --git a/packages/contrail/README.md b/packages/contrail/README.md index b312637..2ff33a3 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -89,7 +89,34 @@ pnpm contrail lexicons generate pnpm contrail lexicons check ``` -`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only the collection methods intended for a public read surface. Generated `lex.config.js` files carry an ownership marker; existing user-owned Atcute configuration is never replaced. Pass `--no-atcute-config` to manage that file yourself. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. +`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only methods advertised by the anonymous read surface. Generated `lex.config.js` files carry an ownership marker; existing user-owned Atcute configuration is never replaced. Pass `--no-atcute-config` to manage that file yourself. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. + +## Public read-through service + +```ts +export default createWorker(config, { + lexicons, + publicService: { endpoint: "https://api.example.com" }, +}); +``` + +Discovery at `/.well-known/contrail` advertises a canonical contract digest and a content-addressed Lexicon bundle. Collection reads, profiles, feeds, and authored custom queries remain anonymous read-through operations: they may acquire public AT Protocol data and improve the cache behind the response. `notifyOfUpdate` remains separately controlled by `config.notify` and is not part of the anonymous contract. + +Configure the primary ordered source so `getCursor` can expose its committed position: + +```ts +const config = { + orderedSource: { + source: "jetstream", + epoch: "primary-2026", // change whenever cursor continuity changes + }, + // ... +}; +``` + +The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` value for equality; never order cursors from different epochs. Consumers can read the position before and after a query, retry if it changed, then poll it as a refetch/invalidation signal. + +Connect an independent consumer with `contrail connect `. A repeated connection requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. ## Runtime record validation diff --git a/packages/contrail/tests/backfill-status.test.ts b/packages/contrail/tests/backfill-status.test.ts index a431268..6549e76 100644 --- a/packages/contrail/tests/backfill-status.test.ts +++ b/packages/contrail/tests/backfill-status.test.ts @@ -881,7 +881,12 @@ describe("backfill status JSON", () => { const root = await app.fetch(new Request("http://localhost/")); expect(await root.json()).toEqual({ status: "ok" }); - const xrpc = await app.fetch(new Request("http://localhost/xrpc/com.example.getOverview")); - expect(((await xrpc.json()) as any).backfill).toEqual(overview.backfill); + const status = await app.fetch(new Request("http://localhost/status")); + expect(((await status.json()) as any).backfill).toEqual(overview.backfill); + + const removed = await app.fetch( + new Request("http://localhost/xrpc/com.example.getOverview"), + ); + expect(removed.status).toBe(404); }); }); diff --git a/packages/contrail/tests/connect.test.ts b/packages/contrail/tests/connect.test.ts new file mode 100644 index 0000000..bcc6b4b --- /dev/null +++ b/packages/contrail/tests/connect.test.ts @@ -0,0 +1,296 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { connectPublicService } from "../src/cli/commands/connect"; +import { + contractFromManifest, + digestLexiconDocuments, + digestPublicContract, + type PublicServiceManifest, +} from "../src/public-service"; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true })), + ); +}); + +const endpoint = "https://api.atmo.rsvp"; +const method = "atmo.rsvp.event.listRecords"; +const methodLexicon = { + lexicon: 1, + id: method, + defs: { main: { type: "query" } }, +}; +const sourceLexicon = { + lexicon: 1, + id: "community.lexicon.calendar.event", + defs: { main: { type: "record" } }, +}; + +async function serviceFixture(values = [methodLexicon, sourceLexicon]) { + const { digest } = await digestLexiconDocuments(values); + const manifest: PublicServiceManifest = { + format: "contrail.service", + version: 1, + endpoint, + namespace: "atmo.rsvp", + contract: { digest: "" }, + lexicons: { url: `${endpoint}/lexicons/${digest}`, digest }, + status: { url: `${endpoint}/status` }, + collections: [ + { + alias: "event", + nsid: "community.lexicon.calendar.event", + methods: [method], + queryable: [], + searchable: [], + relations: [], + references: [], + }, + ], + methods: [method], + }; + manifest.contract.digest = await digestPublicContract( + contractFromManifest(manifest), + ); + const fetcher = vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/.well-known/contrail")) { + return Response.json(manifest); + } + if (url.includes("/lexicons/")) return Response.json({ lexicons: values }); + return new Response("not found", { status: 404 }); + }); + return { fetcher, manifest, values }; +} + +async function temporaryRoot() { + const root = await mkdtemp(join(tmpdir(), "contrail-connect-")); + roots.push(root); + return root; +} + +describe("contrail connect", () => { + it("verifies and atomically locks a discovered service", async () => { + const root = await temporaryRoot(); + const { fetcher, manifest, values } = await serviceFixture(); + + const result = await connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher, + }); + + expect(result.written).toBe(2); + expect(result.lock).toMatchObject({ + endpoint, + namespace: "atmo.rsvp", + contractDigest: manifest.contract.digest, + lexiconRoot: "lexicons/pulled/api.atmo.rsvp", + }); + expect( + JSON.parse( + await readFile( + join( + root, + "lexicons/pulled/api.atmo.rsvp/atmo/rsvp/event/listRecords.json", + ), + "utf8", + ), + ), + ).toEqual(values[0]); + expect( + JSON.parse(await readFile(join(root, "contrail.lock.json"), "utf8")), + ).toEqual(result.lock); + + await writeFile(join(root, "lexicons/pulled/consumer-owned.json"), "keep"); + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher, + }), + ).rejects.toThrow("rerun with --update"); + await connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher, + update: true, + }); + expect( + await readFile(join(root, "lexicons/pulled/consumer-owned.json"), "utf8"), + ).toBe("keep"); + }); + + it("preserves the previous provider and lock when an update fails validation", async () => { + const root = await temporaryRoot(); + const fixture = await serviceFixture(); + await connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: fixture.fetcher, + }); + const lockPath = join(root, "contrail.lock.json"); + const documentPath = join( + root, + "lexicons/pulled/api.atmo.rsvp/atmo/rsvp/event/listRecords.json", + ); + const previousLock = await readFile(lockPath, "utf8"); + const previousDocument = await readFile(documentPath, "utf8"); + fixture.manifest.contract.digest = `sha256:${"f".repeat(64)}`; + + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: fixture.fetcher, + update: true, + }), + ).rejects.toThrow("Contract digest mismatch"); + expect(await readFile(lockPath, "utf8")).toBe(previousLock); + expect(await readFile(documentPath, "utf8")).toBe(previousDocument); + }); + + it("rejects Lexicon and contract digest mismatches", async () => { + const root = await temporaryRoot(); + const lexiconMismatch = await serviceFixture(); + lexiconMismatch.manifest.lexicons.digest = `sha256:${"0".repeat(64)}`; + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: lexiconMismatch.fetcher, + }), + ).rejects.toThrow("Lexicon digest mismatch"); + + const contractMismatch = await serviceFixture(); + contractMismatch.manifest.contract.digest = `sha256:${"1".repeat(64)}`; + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: contractMismatch.fetcher, + }), + ).rejects.toThrow("Contract digest mismatch"); + }); + + it("rejects advertised methods without matching query Lexicons", async () => { + const root = await temporaryRoot(); + const fixture = await serviceFixture([ + { ...methodLexicon, defs: { main: { type: "procedure" } } }, + sourceLexicon, + ]); + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: fixture.fetcher, + }), + ).rejects.toThrow("matching query Lexicon"); + }); + + it("rejects inconsistent method namespaces and collection capabilities", async () => { + const root = await temporaryRoot(); + const outside = await serviceFixture(); + outside.manifest.methods.push("other.example.read"); + outside.manifest.contract.digest = await digestPublicContract( + contractFromManifest(outside.manifest), + ); + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: outside.fetcher, + }), + ).rejects.toThrow("outside its namespace"); + + const unknown = await serviceFixture(); + unknown.manifest.collections[0]!.methods.push("atmo.rsvp.event.getRecord"); + unknown.manifest.contract.digest = await digestPublicContract( + contractFromManifest(unknown.manifest), + ); + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: unknown.fetcher, + }), + ).rejects.toThrow("advertises an unknown method"); + }); + + it("rejects cross-origin redirects and bounded request timeouts", async () => { + const root = await temporaryRoot(); + const redirected = vi.fn(async () => { + const response = Response.json({}); + Object.defineProperty(response, "url", { + value: "https://evil.example/", + }); + return response; + }); + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: redirected, + }), + ).rejects.toThrow("redirected to a different origin"); + + const hanging = vi.fn( + async (_input: RequestInfo | URL, init?: RequestInit) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => + reject(init.signal?.reason), + ); + }), + ); + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: hanging, + timeoutMs: 5, + }), + ).rejects.toThrow(); + }); + + it("never cleans an output path outside the consumer project", async () => { + const root = await temporaryRoot(); + const { fetcher } = await serviceFixture(); + await expect( + connectPublicService({ + endpoint, + root, + out: ".", + lock: "contrail.lock.json", + fetcher, + }), + ).rejects.toThrow("path must stay inside"); + }); +}); diff --git a/packages/contrail/tests/database-bootstrap-target.test.ts b/packages/contrail/tests/database-bootstrap-target.test.ts index 6caac93..bc7da5d 100644 --- a/packages/contrail/tests/database-bootstrap-target.test.ts +++ b/packages/contrail/tests/database-bootstrap-target.test.ts @@ -6,6 +6,7 @@ import { bootstrapFreshProjection, getBootstrapFailure, getBootstrapVerification, + getServingSourcePosition, initSchema, queryRecords, resolveConfig, @@ -236,6 +237,9 @@ describe("database bootstrap target", () => { source_cursor: "2", }); expect(await getBootstrapVerification(db)).toMatchObject({ ok: true }); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: sourcePosition(3), + }); }); it("blocks completion and persists aggregate verification failures", async () => { @@ -317,6 +321,60 @@ describe("database bootstrap target", () => { expect((await target.load())?.phase).toBe("complete"); }); + it("rolls projection and checkpoint back when source position cannot commit", async () => { + const resolved = config(); + const db = createSqliteDatabase(":memory:"); + await initSchema(db, resolved); + const target = new DatabaseBootstrapTarget(db, resolved); + const prepared = snapshot(); + await target.beginCapture(sourcePosition(1)); + await target.setSnapshot(prepared, sourcePosition(1)); + await target.applySnapshotBatch(prepared, { + records: [], + sourceTimeUs: 1, + progress: { partition: "pds", cursor: null, complete: true }, + done: true, + }); + await target.beginCatchup(sourcePosition(2)); + await db + .prepare( + `CREATE TRIGGER fail_source_position + BEFORE INSERT ON source_position + BEGIN SELECT RAISE(ABORT, 'injected source position failure'); END`, + ) + .run(); + const batch = { + mutations: [ + { + operation: "put" as const, + ...record("a", "atomic"), + sourceTimeUs: 2, + position: sourcePosition(2), + }, + ], + checkpoint: sourcePosition(2), + caughtUp: true, + }; + + await expect(target.applyMutationBatch(batch)).rejects.toThrow( + "injected source position failure", + ); + expect( + (await queryRecords(db, resolved, { collection: "event" })).records, + ).toHaveLength(0); + expect((await target.load())?.changeCheckpoint).toEqual(sourcePosition(1)); + expect(await getServingSourcePosition(db)).toBeNull(); + + await db.prepare("DROP TRIGGER fail_source_position").run(); + await target.applyMutationBatch(batch); + expect( + (await queryRecords(db, resolved, { collection: "event" })).records, + ).toHaveLength(1); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: sourcePosition(2), + }); + }); + it("rejects a mutation position from another epoch before advancing progress", async () => { const resolved = config(); const db = createSqliteDatabase(":memory:"); diff --git a/packages/contrail/tests/jetstream-change-source.test.ts b/packages/contrail/tests/jetstream-change-source.test.ts index 09e32dc..ab45cfb 100644 --- a/packages/contrail/tests/jetstream-change-source.test.ts +++ b/packages/contrail/tests/jetstream-change-source.test.ts @@ -62,6 +62,20 @@ function config() { } describe("Jetstream change source", () => { + it("requires the bootstrap epoch to match the configured live source", () => { + const configured = resolveConfig({ + ...config(), + orderedSource: { source: "jetstream", epoch: "live-epoch" }, + }); + expect( + () => + new JetstreamChangeSource(configured, { + epoch: "different-epoch", + retentionUs: 60_000_000, + }), + ).toThrow("does not match configured ordered source"); + }); + it("uses real stream events as marks and replays through the exact watermark", async () => { const nowUs = Date.now() * 1000; const start = nowUs - 20_000; diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts index cbd998f..7f02e9f 100644 --- a/packages/contrail/tests/lexicon-generation.test.ts +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -98,7 +98,7 @@ function fixture() { }, }, }; - return { root, config, pulled }; + return { root, config, pulled, write }; } function parameters(document: any): Record { @@ -118,10 +118,15 @@ describe("Contrail Lexicon generation", () => { expect(result.methods).toEqual([ "example.public.event.getRecord", "example.public.event.listRecords", + "example.public.getCursor", "example.public.rsvp.getRecord", "example.public.rsvp.listRecords", ]); - expect(result.generated["example.public.getCursor"]).toBeUndefined(); + expect(result.generated["example.public.getCursor"]).toBeDefined(); + expect( + (result.generated["example.public.getCursor"] as any).defs.sourcePosition + .required, + ).toEqual(["source", "epoch", "cursor"]); const event = result.generated["example.public.event.listRecords"] as any; const params = parameters(event); @@ -149,6 +154,34 @@ describe("Contrail Lexicon generation", () => { ); }); + it("references profile record schemas without duplicating their definitions", () => { + const { root, config, write } = fixture(); + write("community.example.profile", { + lexicon: 1, + id: "community.example.profile", + defs: { + main: { + type: "record", + key: "literal:self", + record: { + type: "object", + properties: { displayName: { type: "string" } }, + }, + }, + }, + }); + config.profiles = [ + { collection: "community.example.profile", shortName: "profile" }, + ]; + const result = generateLexicons({ config, rootDir: root, quiet: true }); + const profile = result.generated["example.public.getProfile"] as any; + expect(profile.defs.profileEntry.properties.value).toEqual({ + type: "ref", + ref: "community.example.profile#main", + }); + expect(profile.defs.communityExampleProfile).toBeUndefined(); + }); + it("respects disabled standard methods", () => { const { root, config } = fixture(); config.collections.event!.methods = ["listRecords"]; @@ -170,7 +203,7 @@ describe("Contrail Lexicon generation", () => { }; const result = generateLexicons({ config, rootDir: root, quiet: true }); expect(result.methods).toContain("example.public.getCursor"); - expect(result.methods).toContain("example.public.getOverview"); + expect(result.methods).not.toContain("example.public.getOverview"); expect(result.methods).toContain("example.public.notifyOfUpdate"); expect(result.methods).toContain("example.public.getFeed"); const feed = result.generated["example.public.getFeed"] as any; diff --git a/packages/contrail/tests/persistent.test.ts b/packages/contrail/tests/persistent.test.ts index 9c36b5a..47767fd 100644 --- a/packages/contrail/tests/persistent.test.ts +++ b/packages/contrail/tests/persistent.test.ts @@ -3,7 +3,11 @@ import type { ContrailConfig, Database } from "../src/index"; import { resolveConfig } from "../src/index"; import { createTestDb, createTestDbWithSchema, TEST_CONFIG } from "./helpers"; import { runPersistent } from "../src/index"; -import { getLastCursor, queryRecords } from "../src/index"; +import { + getLastCursor, + getServingSourcePosition, + queryRecords, +} from "../src/index"; import { initSchema } from "../src/index"; const applyIdentityEventMock = vi.fn().mockResolvedValue(undefined); @@ -48,7 +52,11 @@ function mockSubscription(events: Array<{ kind: string; did: string; time_us: nu } describe("runPersistent", () => { - it("flushes when batch size is reached", async () => { + it("flushes records and the ordered source position atomically", async () => { + const config = resolveConfig({ + ...TEST_CONFIG, + orderedSource: { source: "jetstream", epoch: "persistent-test" }, + }); const events = Array.from({ length: 50 }, (_, i) => ({ kind: "commit" as const, did: `did:plc:user${i}`, @@ -65,7 +73,7 @@ describe("runPersistent", () => { const controller = new AbortController(); // After yielding 50 events, the mock hangs. Give it time to flush, then abort. - const promise = runPersistent(db, TEST_CONFIG, { + const promise = runPersistent(db, config, { batchSize: 50, flushIntervalMs: 60_000, // high so only batch size triggers flush signal: controller.signal, @@ -85,6 +93,13 @@ describe("runPersistent", () => { const cursor = await getLastCursor(db); expect(cursor).toBe(1049); // last event's time_us + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "persistent-test", + cursor: "1049", + }, + }); }); it("keeps dependent events discovered in the same flush batch", async () => { diff --git a/packages/contrail/tests/public-service-e2e.test.ts b/packages/contrail/tests/public-service-e2e.test.ts new file mode 100644 index 0000000..84200f2 --- /dev/null +++ b/packages/contrail/tests/public-service-e2e.test.ts @@ -0,0 +1,179 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { connectPublicService } from "../src/cli/commands/connect"; +import { generateLexiconTypesWithAtcute } from "../src/cli/atcute"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { createApp } from "../src/core/router"; +import { + createIngestEvent, + ingestRecords, + initSchema, + resolveConfig, + type ContrailConfig, +} from "../src/index"; +import { generateLexicons } from "../src/lexicons/generate"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +function temporaryRoot(label: string): string { + const root = mkdtempSync(join(process.cwd(), `.${label}-`)); + roots.push(root); + return root; +} + +function writeJson(path: string, value: unknown): void { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); +} + +describe("public service consumer integration", () => { + it("discovers, connects, generates types, compiles, and queries", async () => { + const serviceRoot = temporaryRoot("public-service"); + const consumerRoot = temporaryRoot("public-consumer"); + const sourceLexicon = { + lexicon: 1, + id: "community.example.event", + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + required: ["name"], + properties: { name: { type: "string" } }, + }, + }, + }, + }; + writeJson( + join(serviceRoot, "lexicons/pulled/community/example/event.json"), + sourceLexicon, + ); + const config: ContrailConfig = { + namespace: "com.example", + profiles: [], + orderedSource: { source: "jetstream", epoch: "e2e" }, + collections: { + event: { + collection: "community.example.event", + queryable: { name: {} }, + }, + }, + }; + const generated = generateLexicons({ + config, + rootDir: serviceRoot, + surface: "public", + quiet: true, + }); + const lexicons = [sourceLexicon, ...Object.values(generated.generated)]; + const resolved = resolveConfig(config); + const db = createSqliteDatabase(":memory:"); + await initSchema(db, resolved); + await ingestRecords( + db, + [ + createIngestEvent({ + uri: "at://did:plc:test/community.example.event/1", + did: "did:plc:test", + collection: "community.example.event", + rkey: "1", + operation: "create", + cid: "bafyreievent", + value: { name: "Typed event" }, + timeUs: 1, + indexedAt: 1, + }), + ], + resolved, + ); + const app = createApp(db, resolved, { + lexicons, + publicService: { endpoint: "https://api.example.com" }, + }); + const fetcher: typeof fetch = (input, init) => + app.fetch(new Request(input, init)); + + writeFileSync( + join(consumerRoot, "lex.config.js"), + `import { defineLexiconConfig } from "@atcute/lex-cli"; +export default defineLexiconConfig({ + generate: { + files: ["lexicons/pulled/**/*.json"], + outdir: "src/lexicons/", + }, +}); +`, + ); + await connectPublicService({ + endpoint: "https://api.example.com", + root: consumerRoot, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher, + }); + generateLexiconTypesWithAtcute(consumerRoot); + + mkdirSync(join(consumerRoot, "src"), { recursive: true }); + writeFileSync( + join(consumerRoot, "src", "consumer.ts"), + `import { Client, simpleFetchHandler } from "@atcute/client"; +import "./lexicons/index.js"; +const client = new Client({ + handler: simpleFetchHandler({ service: "https://api.example.com" }), +}); +const response = await client.get("com.example.event.listRecords", { + params: { name: "Typed event", limit: 1 }, +}); +if (response.ok) { + const name: string = response.data.records[0]!.value.name; + console.log(name); +} +`, + ); + writeJson(join(consumerRoot, "tsconfig.json"), { + compilerOptions: { + target: "ES2022", + module: "NodeNext", + moduleResolution: "NodeNext", + strict: true, + noEmit: true, + skipLibCheck: true, + }, + include: ["src/**/*.ts"], + }); + const npmExecPath = process.env.npm_execpath; + const command = npmExecPath ? process.execPath : "pnpm"; + const args = npmExecPath + ? [ + npmExecPath, + "exec", + "tsc", + "--project", + join(consumerRoot, "tsconfig.json"), + ] + : ["exec", "tsc", "--project", join(consumerRoot, "tsconfig.json")]; + const checked = spawnSync(command, args, { + cwd: process.cwd(), + encoding: "utf8", + }); + expect(checked.status, `${checked.stdout}\n${checked.stderr}`).toBe(0); + + const queried = await app.fetch( + new Request( + "https://api.example.com/xrpc/com.example.event.listRecords?name=Typed%20event&limit=1", + ), + ); + expect(queried.status).toBe(200); + expect(await queried.json()).toMatchObject({ + records: [{ value: { name: "Typed event" } }], + }); + }); +}); diff --git a/packages/contrail/tests/serving-source-position.test.ts b/packages/contrail/tests/serving-source-position.test.ts new file mode 100644 index 0000000..9266732 --- /dev/null +++ b/packages/contrail/tests/serving-source-position.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { + Contrail, + assertServingSourceCompatibility, + getLastCursor, + getServingSourcePosition, + initSchema, + saveCursor, + type ContrailConfig, +} from "../src/index"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; + +const config: ContrailConfig = { + namespace: "com.example", + profiles: [], + orderedSource: { source: "jetstream", epoch: "primary-2026" }, + collections: { + event: { collection: "com.example.event" }, + }, +}; + +describe("serving source positions", () => { + it("commits the legacy replay cursor and opaque source position together", async () => { + const db = createSqliteDatabase(":memory:"); + await initSchema(db, config); + + await saveCursor(db, 123_456, config.orderedSource); + + expect(await getLastCursor(db)).toBe(123_456); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "primary-2026", + cursor: "123456", + }, + }); + + await saveCursor(db, 100, config.orderedSource); + expect(await getLastCursor(db)).toBe(123_456); + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { cursor: "123456" }, + }); + }); + + it("rejects a configured continuity epoch that differs from durable state", async () => { + const db = createSqliteDatabase(":memory:"); + await initSchema(db, config); + await saveCursor(db, 10, config.orderedSource); + + await expect( + assertServingSourceCompatibility(db, { + source: "jetstream", + epoch: "replacement-epoch", + }), + ).rejects.toThrow("does not match durable source position"); + + const contrail = new Contrail({ + ...config, + orderedSource: { source: "jetstream", epoch: "replacement-epoch" }, + }); + await expect(contrail.init(db)).rejects.toThrow( + "does not match durable source position", + ); + }); + + it("validates ordered source configuration", () => { + expect( + () => + new Contrail({ + ...config, + orderedSource: { source: "jetstream", epoch: "" }, + }), + ).toThrow("orderedSource requires non-empty source and epoch values"); + }); +}); diff --git a/packages/contrail/tests/source-ordering.test.ts b/packages/contrail/tests/source-ordering.test.ts index 08b363b..dd643ed 100644 --- a/packages/contrail/tests/source-ordering.test.ts +++ b/packages/contrail/tests/source-ordering.test.ts @@ -3,9 +3,11 @@ import { createIngestEvent, ingestRecords, initSchema, + getServingSourcePosition, queryRecords, resolveConfig, saveCursorStatement, + saveOrderedSourcePositionStatement, type ContrailConfig, type Database, type IngestEvent, @@ -427,6 +429,7 @@ describe("durable source ordering", () => { await db .prepare("INSERT INTO cursor_failure (value) VALUES ('duplicate')") .run(); + const orderedSource = { source: "jetstream", epoch: "atomic-test" }; const event = mutation({ operation: "create", sourceTime: 500, @@ -438,6 +441,7 @@ describe("durable source ordering", () => { ingestRecords(db, [event], resolved, { trailingStatements: [ saveCursorStatement(db, 500), + saveOrderedSourcePositionStatement(db, orderedSource, 500), db.prepare("INSERT INTO cursor_failure (value) VALUES ('duplicate')"), ], }), @@ -453,6 +457,7 @@ describe("durable source ordering", () => { expect( await db.prepare("SELECT time_us FROM cursor WHERE id = 1").first(), ).toBeNull(); + expect(await getServingSourcePosition(db)).toBeNull(); }); it("stores record time separately from source and local times", async () => { diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts index b595a82..b2561e2 100644 --- a/packages/contrail/tests/worker.test.ts +++ b/packages/contrail/tests/worker.test.ts @@ -2,10 +2,17 @@ import { describe, it, expect, vi } from "vitest"; import { createWorker } from "../src/worker"; import { Contrail } from "../src/contrail"; import { createSqliteDatabase } from "../src/adapters/sqlite"; -import type { ContrailConfig } from "../src/index"; +import { + contractFromManifest, + digestPublicContract, + saveCursor, + type ContrailConfig, +} from "../src/index"; const MINIMAL_CONFIG: ContrailConfig = { namespace: "com.example", + profiles: [], + orderedSource: { source: "jetstream", epoch: "worker-test" }, collections: { event: { collection: "community.lexicon.calendar.event", @@ -14,6 +21,20 @@ const MINIMAL_CONFIG: ContrailConfig = { }, }; +function queryLexicons(...ids: string[]) { + return ids.map((id) => ({ + lexicon: 1, + id, + defs: { main: { type: "query" } }, + })); +} + +const MINIMAL_PUBLIC_LEXICONS = queryLexicons( + "com.example.getCursor", + "com.example.event.getRecord", + "com.example.event.listRecords", +); + describe("createWorker", () => { it("returns an object with fetch + scheduled handlers", () => { const worker = createWorker(MINIMAL_CONFIG); @@ -28,7 +49,9 @@ describe("createWorker", () => { // Before first fetch: schema not present yet. const tables = await db - .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='cursor'") + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='cursor'", + ) .first<{ name: string }>(); expect(tables).toBeNull(); @@ -36,7 +59,9 @@ describe("createWorker", () => { // After first fetch: schema present. const after = await db - .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='cursor'") + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='cursor'", + ) .first<{ name: string }>(); expect(after?.name).toBe("cursor"); @@ -67,7 +92,7 @@ describe("createWorker", () => { const res = await worker.fetch( new Request("http://localhost/xrpc/com.example.lexicons"), - env + env, ); expect(res.status).toBe(200); expect(await res.json()).toEqual({ lexicons }); @@ -80,11 +105,236 @@ describe("createWorker", () => { const res = await worker.fetch( new Request("http://localhost/xrpc/com.example.lexicons"), - env + env, ); expect(res.status).toBe(404); }); + it("serves deterministic public discovery and stable Lexicons", async () => { + const db = createSqliteDatabase(":memory:"); + const lexicons = MINIMAL_PUBLIC_LEXICONS; + const worker = createWorker(MINIMAL_CONFIG, { + lexicons, + publicService: { endpoint: "https://api.example.com" }, + }); + const env = { DB: db }; + + const manifestResponse = await worker.fetch( + new Request("https://api.example.com/.well-known/contrail"), + env, + ); + expect(manifestResponse.status).toBe(200); + expect(manifestResponse.headers.get("access-control-allow-origin")).toBe( + "*", + ); + const manifest = await manifestResponse.json(); + expect(manifest).toMatchObject({ + format: "contrail.service", + version: 1, + endpoint: "https://api.example.com", + namespace: "com.example", + contract: { digest: expect.stringMatching(/^sha256:[0-9a-f]{64}$/) }, + lexicons: { + url: expect.stringMatching( + /^https:\/\/api\.example\.com\/lexicons\/sha256:[0-9a-f]{64}$/, + ), + digest: expect.stringMatching(/^sha256:[0-9a-f]{64}$/), + }, + methods: [ + "com.example.event.getRecord", + "com.example.event.listRecords", + "com.example.getCursor", + ], + collections: expect.arrayContaining([ + { + alias: "event", + nsid: "community.lexicon.calendar.event", + methods: [ + "com.example.event.getRecord", + "com.example.event.listRecords", + ], + queryable: ["startsAt"], + searchable: [], + relations: [], + references: [], + }, + ]), + }); + expect(manifest.contract.digest).not.toBe(manifest.lexicons.digest); + expect(await digestPublicContract(contractFromManifest(manifest))).toBe( + manifest.contract.digest, + ); + expect(manifest.lexicons.url).toBe( + `https://api.example.com/lexicons/${manifest.lexicons.digest}`, + ); + + const lexiconResponse = await worker.fetch( + new Request("https://api.example.com/lexicons"), + env, + ); + expect(lexiconResponse.status).toBe(200); + expect(await lexiconResponse.json()).toEqual({ + lexicons: [...lexicons].sort((left, right) => + left.id.localeCompare(right.id), + ), + }); + expect(lexiconResponse.headers.get("etag")).toBe( + `"${manifest.lexicons.digest}"`, + ); + const immutableLexicons = await worker.fetch( + new Request(manifest.lexicons.url), + env, + ); + expect(immutableLexicons.status).toBe(200); + expect(immutableLexicons.headers.get("cache-control")).toContain( + "immutable", + ); + + const statusResponse = await worker.fetch( + new Request("https://api.example.com/status"), + env, + ); + const status = await statusResponse.json(); + expect(status).toMatchObject({ + serving: "ready", + freshness: { last_event_at: null, seconds_ago: null }, + }); + expect(status.ingestion).toBeUndefined(); + expect(statusResponse.headers.get("cache-control")).toContain("max-age=15"); + expect(JSON.stringify(status)).not.toContain("cursor"); + + const emptyCursor = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.getCursor"), + env, + ); + expect(await emptyCursor.json()).toEqual({}); + + await saveCursor(db, 1234, MINIMAL_CONFIG.orderedSource); + const cursorResponse = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.getCursor"), + env, + ); + expect(cursorResponse.status).toBe(200); + expect(await cursorResponse.json()).toMatchObject({ + position: { + source: "jetstream", + epoch: "worker-test", + cursor: "1234", + }, + }); + expect( + ( + await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.getOverview"), + env, + ) + ).status, + ).toBe(404); + }); + + it("keeps profiles, feeds, custom queries, and configured notify routes", async () => { + const config: ContrailConfig = { + ...MINIMAL_CONFIG, + profiles: [{ collection: "com.example.profile", shortName: "profile" }], + feeds: { network: { targets: ["event"] } }, + notify: "secret", + collections: { + event: { + ...MINIMAL_CONFIG.collections.event, + queries: { + featured: async () => Response.json({ records: [] }), + }, + }, + }, + }; + const lexicons = queryLexicons( + "com.example.getCursor", + "com.example.getProfile", + "com.example.getFeed", + "com.example.event.getRecord", + "com.example.event.listRecords", + "com.example.event.featured", + "com.example.profile.getRecord", + "com.example.profile.listRecords", + "com.example.follow.getRecord", + "com.example.follow.listRecords", + ); + const worker = createWorker(config, { + lexicons, + publicService: { endpoint: "https://api.example.com" }, + }); + const env = { DB: createSqliteDatabase(":memory:") }; + + const profile = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.getProfile"), + env, + ); + expect(profile.status).toBe(400); + const feed = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.getFeed"), + env, + ); + expect(feed.status).toBe(400); + const custom = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.event.featured"), + env, + ); + expect(custom.status).toBe(200); + expect(await custom.json()).toEqual({ records: [] }); + const manifest = await ( + await worker.fetch( + new Request("https://api.example.com/.well-known/contrail"), + env, + ) + ).json(); + expect(manifest.methods).toEqual( + expect.arrayContaining([ + "com.example.getCursor", + "com.example.getProfile", + "com.example.getFeed", + "com.example.event.featured", + ]), + ); + expect(manifest.methods).not.toContain("com.example.notifyOfUpdate"); + + const notify = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.notifyOfUpdate", { + method: "POST", + body: JSON.stringify({ uri: "at://did:plc:test/com.example.event/1" }), + }), + env, + ); + expect(notify.status).toBe(401); + }); + + it("refuses public mode without an HTTPS origin and Lexicons", () => { + expect(() => + createWorker(MINIMAL_CONFIG, { + publicService: { endpoint: "https://api.example.com" }, + }), + ).toThrow("non-empty Lexicon bundle"); + + expect(() => + createWorker(MINIMAL_CONFIG, { + lexicons: [{ lexicon: 1, id: "com.example.foo" }], + publicService: { endpoint: "http://api.example.com" }, + }), + ).toThrow("must use HTTPS"); + + expect(() => + createWorker(MINIMAL_CONFIG, { + lexicons: [ + { + lexicon: 1, + id: "com.example.event.listRecords", + defs: { main: { type: "query" } }, + }, + ], + publicService: { endpoint: "https://api.example.com" }, + }), + ).toThrow("public method requires a matching query Lexicon"); + }); + it("scheduled handler runs live ingest then a bounded backfill retry slice", async () => { const order: string[] = []; const ingest = vi -- 2.51.2 From 24eb9feebaf62963f9c77f673d13be556f6acd8b Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 06:24:25 +0200 Subject: [PATCH 05/15] Adopt legacy serving cursors --- packages/contrail/src/core/db/records.ts | 16 +++++++++++++--- .../tests/serving-source-position.test.ts | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index 2b34a39..5f8decb 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -566,10 +566,20 @@ export async function assertServingSourceCompatibility( ): Promise { if (!orderedSource) return; const existing = await getServingSourcePosition(db); + if (!existing) { + const legacyCursor = await getLastCursor(db); + if (legacyCursor !== null) { + await saveOrderedSourcePositionStatement( + db, + orderedSource, + legacyCursor, + ).run(); + } + return; + } if ( - existing && - (existing.position.source !== orderedSource.source || - existing.position.epoch !== orderedSource.epoch) + existing.position.source !== orderedSource.source || + existing.position.epoch !== orderedSource.epoch ) { throw new Error( `configured ordered source ${orderedSource.source}/${orderedSource.epoch} ` + diff --git a/packages/contrail/tests/serving-source-position.test.ts b/packages/contrail/tests/serving-source-position.test.ts index 9266732..f6a00da 100644 --- a/packages/contrail/tests/serving-source-position.test.ts +++ b/packages/contrail/tests/serving-source-position.test.ts @@ -42,6 +42,22 @@ describe("serving source positions", () => { }); }); + it("adopts an existing legacy cursor when an ordered source is configured", async () => { + const db = createSqliteDatabase(":memory:"); + await initSchema(db, config); + await saveCursor(db, 777); + + await new Contrail(config).init(db); + + expect(await getServingSourcePosition(db)).toMatchObject({ + position: { + source: "jetstream", + epoch: "primary-2026", + cursor: "777", + }, + }); + }); + it("rejects a configured continuity epoch that differs from durable state", async () => { const db = createSqliteDatabase(":memory:"); await initSchema(db, config); -- 2.51.2 From 430e77d97ab0b33a1767ceddd78516260a62400a Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:47:56 +0200 Subject: [PATCH 06/15] Keep serving cursors compatible --- README.md | 2 +- .../generated/com/example/getCursor.json | 35 ++------ .../generated/statusphere/app/getCursor.json | 35 ++------ .../types/statusphere/app/getCursor.ts | 26 ++---- packages/contrail/README.md | 2 +- packages/contrail/src/core/db/records.ts | 21 ++--- .../contrail/src/core/router/diagnostics.ts | 10 +++ packages/contrail/src/core/router/index.ts | 2 + packages/contrail/src/lexicons/generate.ts | 85 ++++++++++++------- packages/contrail/src/public-service.ts | 10 +++ .../contrail/tests/lexicon-generation.test.ts | 19 +++++ packages/contrail/tests/worker.test.ts | 33 ++++++- 12 files changed, 160 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index 294b515..145ed35 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Consumers connect and generate Atcute types with one command: pnpm contrail connect https://api.example.com ``` -`getCursor` returns the committed opaque `{ source, epoch, cursor }` position of the primary ordered source. Compare complete positions for equality only; a source or epoch change requires a full client refetch. To avoid racing ingestion, read a position before and after a query and accept the query snapshot only when both positions match. +Public-service mode requires `orderedSource`; `getCursor` then returns the committed opaque `{ source, epoch, cursor }` position of that primary source. Compare complete positions for equality only; a source or epoch change requires a full client refetch. To avoid racing ingestion, read a position before and after a query and accept the query snapshot only when both positions match. Existing non-public deployments without `orderedSource` retain the legacy `time_us`, `date`, and `seconds_ago` response. ## Other databases diff --git a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json index a5238d7..93bb5dd 100644 --- a/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json +++ b/apps/cloudflare-workers/lexicons/generated/com/example/getCursor.json @@ -4,45 +4,24 @@ "defs": { "main": { "type": "query", - "description": "Get the committed primary ordered-source position", + "description": "Get the current ingestion observation time", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "position": { - "type": "ref", - "ref": "#sourcePosition" - }, - "updatedAt": { + "time_us": { "type": "integer" }, - "updatedAtDate": { - "type": "string", - "format": "datetime" + "date": { + "type": "string" + }, + "seconds_ago": { + "type": "integer" } } } } - }, - "sourcePosition": { - "type": "object", - "required": [ - "source", - "epoch", - "cursor" - ], - "properties": { - "source": { - "type": "string" - }, - "epoch": { - "type": "string" - }, - "cursor": { - "type": "string" - } - } } } } diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json index 520e385..32cd295 100644 --- a/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/statusphere/app/getCursor.json @@ -4,45 +4,24 @@ "defs": { "main": { "type": "query", - "description": "Get the committed primary ordered-source position", + "description": "Get the current ingestion observation time", "output": { "encoding": "application/json", "schema": { "type": "object", "properties": { - "position": { - "type": "ref", - "ref": "#sourcePosition" - }, - "updatedAt": { + "time_us": { "type": "integer" }, - "updatedAtDate": { - "type": "string", - "format": "datetime" + "date": { + "type": "string" + }, + "seconds_ago": { + "type": "integer" } } } } - }, - "sourcePosition": { - "type": "object", - "required": [ - "source", - "epoch", - "cursor" - ], - "properties": { - "source": { - "type": "string" - }, - "epoch": { - "type": "string" - }, - "cursor": { - "type": "string" - } - } } } } diff --git a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts index c2255ab..05a42fc 100644 --- a/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts +++ b/apps/sveltekit-cloudflare-workers/src/lib/lexicons/types/statusphere/app/getCursor.ts @@ -8,34 +8,18 @@ const _mainSchema = /*#__PURE__*/ v.query( "params": null, "output": { "type": "lex", - "schema": /*#__PURE__*/ v.object( - { - get "position"() { - return /*#__PURE__*/ v.optional(sourcePositionSchema) - }, - "updatedAt": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), - "updatedAtDate": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.datetimeString()), - } - ), + "schema": /*#__PURE__*/ v.object({ + "date": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.string()), + "seconds_ago": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + "time_us": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.integer()), + }), } } ); -const _sourcePositionSchema = /*#__PURE__*/ v.object({ - "$type": /*#__PURE__*/ v.optional(/*#__PURE__*/ v.literal("statusphere.app.getCursor#sourcePosition")), - "cursor": /*#__PURE__*/ v.string(), - "epoch": /*#__PURE__*/ v.string(), - "source": /*#__PURE__*/ v.string(), -}); type main$schematype = typeof _mainSchema; -type sourcePosition$schematype = typeof _sourcePositionSchema; export interface mainSchema extends main$schematype {} - -export interface sourcePositionSchema extends sourcePosition$schematype {} export const mainSchema = _mainSchema as mainSchema; -export const sourcePositionSchema = _sourcePositionSchema as sourcePositionSchema; - -export interface SourcePosition extends v.InferInput {} export interface $params {} diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 2ff33a3..3c90680 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -102,7 +102,7 @@ export default createWorker(config, { Discovery at `/.well-known/contrail` advertises a canonical contract digest and a content-addressed Lexicon bundle. Collection reads, profiles, feeds, and authored custom queries remain anonymous read-through operations: they may acquire public AT Protocol data and improve the cache behind the response. `notifyOfUpdate` remains separately controlled by `config.notify` and is not part of the anonymous contract. -Configure the primary ordered source so `getCursor` can expose its committed position: +Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Existing non-public deployments without one retain the legacy ingestion-time cursor response: ```ts const config = { diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index 5f8decb..eda15c1 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -565,17 +565,18 @@ export async function assertServingSourceCompatibility( orderedSource?: OrderedSourceConfig, ): Promise { if (!orderedSource) return; - const existing = await getServingSourcePosition(db); + let existing = await getServingSourcePosition(db); if (!existing) { - const legacyCursor = await getLastCursor(db); - if (legacyCursor !== null) { - await saveOrderedSourcePositionStatement( - db, - orderedSource, - legacyCursor, - ).run(); - } - return; + await db + .prepare( + `INSERT INTO source_position (id, source, epoch, cursor, updated_at) + SELECT 1, ?, ?, CAST(time_us AS TEXT), ? FROM cursor WHERE id = 1 + ON CONFLICT(id) DO NOTHING`, + ) + .bind(orderedSource.source, orderedSource.epoch, Date.now()) + .run(); + existing = await getServingSourcePosition(db); + if (!existing) return; } if ( existing.position.source !== orderedSource.source || diff --git a/packages/contrail/src/core/router/diagnostics.ts b/packages/contrail/src/core/router/diagnostics.ts index 72538c4..5a7927d 100644 --- a/packages/contrail/src/core/router/diagnostics.ts +++ b/packages/contrail/src/core/router/diagnostics.ts @@ -70,6 +70,16 @@ export function registerCursorRoute( const ns = config.namespace; app.get(`/xrpc/${ns}.getCursor`, async (c) => { + if (!config.orderedSource) { + const legacy = await getCursorStatus(db); + if (legacy.cursor === null) return c.json({}); + return c.json({ + time_us: legacy.cursor, + date: legacy.date, + seconds_ago: legacy.seconds_ago, + }); + } + const current = await getServingSourcePosition(db); if (!current) return c.json({}); return c.json({ diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 4ae1c84..a928954 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -15,6 +15,7 @@ import { describePublicService, normalizeLexiconDocuments, normalizePublicServiceEndpoint, + validatePublicServiceLexicons, type PublicServiceOptions, } from "../../public-service"; @@ -59,6 +60,7 @@ export function createApp( : (options.lexicons ?? []); if (options.publicService) { normalizePublicServiceEndpoint(options.publicService.endpoint); + validatePublicServiceLexicons(config, lexicons); const description = describePublicService( config, options.publicService, diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index 1aa628e..56574c8 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -670,6 +670,60 @@ export function extractXrpcMethods( .sort(); } +function cursorLexicon(config: ContrailConfig): object { + const legacyProperties = { + time_us: { type: "integer" }, + date: { type: "string" }, + seconds_ago: { type: "integer" }, + }; + if (!config.orderedSource) { + return { + lexicon: 1, + id: `${config.namespace}.getCursor`, + defs: { + main: { + type: "query", + description: "Get the current ingestion observation time", + output: { + encoding: "application/json", + schema: { type: "object", properties: legacyProperties }, + }, + }, + }, + }; + } + return { + lexicon: 1, + id: `${config.namespace}.getCursor`, + defs: { + main: { + type: "query", + description: "Get the committed primary ordered-source position", + output: { + encoding: "application/json", + schema: { + type: "object", + properties: { + position: { type: "ref", ref: "#sourcePosition" }, + updatedAt: { type: "integer" }, + updatedAtDate: { type: "string", format: "datetime" }, + }, + }, + }, + }, + sourcePosition: { + type: "object", + required: ["source", "epoch", "cursor"], + properties: { + source: { type: "string" }, + epoch: { type: "string" }, + cursor: { type: "string" }, + }, + }, + }, + }; +} + export function generateLexicons( options: GenerateLexiconsOptions, ): GenerateLexiconsResult { @@ -718,36 +772,7 @@ export function generateLexicons( log(` ${nsid}`); }; - emit(`${config.namespace}.getCursor`, { - lexicon: 1, - id: `${config.namespace}.getCursor`, - defs: { - main: { - type: "query", - description: "Get the committed primary ordered-source position", - output: { - encoding: "application/json", - schema: { - type: "object", - properties: { - position: { type: "ref", ref: "#sourcePosition" }, - updatedAt: { type: "integer" }, - updatedAtDate: { type: "string", format: "datetime" }, - }, - }, - }, - }, - sourcePosition: { - type: "object", - required: ["source", "epoch", "cursor"], - properties: { - source: { type: "string" }, - epoch: { type: "string" }, - cursor: { type: "string" }, - }, - }, - }, - }); + emit(`${config.namespace}.getCursor`, cursorLexicon(config)); if ((config.profiles?.length ?? 0) > 0) { emit(`${config.namespace}.getProfile`, { lexicon: 1, diff --git a/packages/contrail/src/public-service.ts b/packages/contrail/src/public-service.ts index 118c8f6..efc4085 100644 --- a/packages/contrail/src/public-service.ts +++ b/packages/contrail/src/public-service.ts @@ -198,10 +198,19 @@ export function validateContractLexicons( return lexicons; } +function assertPublicServiceSource(config: ContrailConfig): void { + if (!config.orderedSource) { + throw new Error( + "public service mode requires orderedSource so getCursor has a durable continuity identity", + ); + } +} + export function validatePublicServiceLexicons( config: ContrailConfig, values: readonly object[], ): LexiconDocument[] { + assertPublicServiceSource(config); const placeholderDigest = `sha256:${"0".repeat(64)}`; return validateContractLexicons( createPublicContract(config, placeholderDigest), @@ -230,6 +239,7 @@ export async function describePublicService( options: PublicServiceOptions, values: readonly object[], ): Promise { + assertPublicServiceSource(config); const endpoint = normalizePublicServiceEndpoint(options.endpoint); const { lexicons, diff --git a/packages/contrail/tests/lexicon-generation.test.ts b/packages/contrail/tests/lexicon-generation.test.ts index 7f02e9f..d4a2bf9 100644 --- a/packages/contrail/tests/lexicon-generation.test.ts +++ b/packages/contrail/tests/lexicon-generation.test.ts @@ -70,6 +70,7 @@ function fixture() { const config: ContrailConfig = { namespace: "example.public", profiles: [], + orderedSource: { source: "jetstream", epoch: "test-primary" }, collections: { event: { collection: "community.example.event", @@ -182,6 +183,24 @@ describe("Contrail Lexicon generation", () => { expect(profile.defs.communityExampleProfile).toBeUndefined(); }); + it("keeps the legacy cursor contract without an ordered source", () => { + const { root, config } = fixture(); + delete config.orderedSource; + const result = generateLexicons({ + config, + rootDir: root, + surface: "public", + quiet: true, + }); + const cursor = result.generated["example.public.getCursor"] as any; + expect(cursor.defs.sourcePosition).toBeUndefined(); + expect(cursor.defs.main.output.schema.properties).toEqual({ + time_us: { type: "integer" }, + date: { type: "string" }, + seconds_ago: { type: "integer" }, + }); + }); + it("respects disabled standard methods", () => { const { root, config } = fixture(); config.collections.event!.methods = ["listRecords"]; diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts index b2561e2..1937366 100644 --- a/packages/contrail/tests/worker.test.ts +++ b/packages/contrail/tests/worker.test.ts @@ -232,6 +232,27 @@ describe("createWorker", () => { ).toBe(404); }); + it("preserves the legacy cursor response without an ordered source", async () => { + const config: ContrailConfig = { + ...MINIMAL_CONFIG, + orderedSource: undefined, + }; + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(config); + const env = { DB: db }; + await worker.fetch(new Request("https://api.example.com/health"), env); + await saveCursor(db, 1_234_000); + + const response = await worker.fetch( + new Request("https://api.example.com/xrpc/com.example.getCursor"), + env, + ); + expect(await response.json()).toMatchObject({ + time_us: 1_234_000, + date: new Date(1234).toISOString(), + }); + }); + it("keeps profiles, feeds, custom queries, and configured notify routes", async () => { const config: ContrailConfig = { ...MINIMAL_CONFIG, @@ -307,7 +328,17 @@ describe("createWorker", () => { expect(notify.status).toBe(401); }); - it("refuses public mode without an HTTPS origin and Lexicons", () => { + it("refuses public mode without an ordered source, HTTPS origin, and Lexicons", () => { + expect(() => + createWorker( + { ...MINIMAL_CONFIG, orderedSource: undefined }, + { + lexicons: MINIMAL_PUBLIC_LEXICONS, + publicService: { endpoint: "https://api.example.com" }, + }, + ), + ).toThrow("requires orderedSource"); + expect(() => createWorker(MINIMAL_CONFIG, { publicService: { endpoint: "https://api.example.com" }, -- 2.51.2 From cf105cc9847495667dfc04bdb3e3ec069df87311 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:59:53 +0200 Subject: [PATCH 07/15] Keep provider updates within their lock --- packages/contrail/README.md | 2 +- packages/contrail/src/cli/commands/connect.ts | 56 ++++++++++++++++--- packages/contrail/tests/connect.test.ts | 56 ++++++++++++++++++- 3 files changed, 103 insertions(+), 11 deletions(-) diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 3c90680..6fd56c2 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -116,7 +116,7 @@ const config = { The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` value for equality; never order cursors from different epochs. Consumers can read the position before and after a query, retry if it changed, then poll it as a refetch/invalidation signal. -Connect an independent consumer with `contrail connect `. A repeated connection requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. +Connect an independent consumer with `contrail connect `. A repeated connection to the same endpoint and provider-owned output root requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. Switching providers or output roots requires removing the existing connection deliberately, so stale Lexicons cannot remain under a broad generator glob. ## Runtime record validation diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index aebefc1..b0d18b2 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -50,6 +50,34 @@ export interface ProviderLock { lexiconRoot: string; } +async function readProviderLock(path: string): Promise { + let source: string; + try { + source = await readFile(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + let value: unknown; + try { + value = JSON.parse(source) as unknown; + } catch { + throw new Error("existing Contrail provider lock is not valid JSON"); + } + const lock = value as Partial; + if ( + !value || + typeof value !== "object" || + lock.format !== "contrail.provider-lock" || + lock.version !== 1 || + typeof lock.endpoint !== "string" || + typeof lock.lexiconRoot !== "string" + ) { + throw new Error("existing Contrail provider lock is malformed"); + } + return lock as ProviderLock; +} + async function readJson(response: Response, label: string): Promise { if (!response.ok) { throw new Error( @@ -136,14 +164,27 @@ export async function connectPublicService(options: { const endpoint = normalizePublicServiceEndpoint(options.endpoint); const projectRoot = resolve(options.root); const lockPath = resolveInsideRoot(projectRoot, options.lock); - if (!options.update) { - try { - await readFile(lockPath, "utf8"); + const outputRoot = resolveInsideRoot(projectRoot, options.out); + const providerKey = new URL(endpoint).host.replace(/[^a-zA-Z0-9.-]/g, "_"); + const providerRoot = resolveInsideRoot(outputRoot, providerKey); + const existingLock = await readProviderLock(lockPath); + if (existingLock && !options.update) { + throw new Error( + "a Contrail provider lock already exists; rerun with --update", + ); + } + if (existingLock) { + if (normalizePublicServiceEndpoint(existingLock.endpoint) !== endpoint) { throw new Error( - "a Contrail provider lock already exists; rerun with --update", + `provider lock targets ${existingLock.endpoint}; remove the existing connection before switching endpoints`, + ); + } + if ( + resolveInsideRoot(projectRoot, existingLock.lexiconRoot) !== providerRoot + ) { + throw new Error( + `provider lock owns ${existingLock.lexiconRoot}; reuse its output path or remove the existing connection`, ); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } } const fetcher = options.fetcher ?? fetch; @@ -200,9 +241,6 @@ export async function connectPublicService(options: { ); } - const outputRoot = resolveInsideRoot(projectRoot, options.out); - const providerKey = new URL(endpoint).host.replace(/[^a-zA-Z0-9.-]/g, "_"); - const providerRoot = resolveInsideRoot(outputRoot, providerKey); await mkdir(outputRoot, { recursive: true }); const stagedProvider = await mkdtemp(join(outputRoot, `.${providerKey}-`)); for (const document of lexicons) { diff --git a/packages/contrail/tests/connect.test.ts b/packages/contrail/tests/connect.test.ts index bcc6b4b..58a4714 100644 --- a/packages/contrail/tests/connect.test.ts +++ b/packages/contrail/tests/connect.test.ts @@ -2,7 +2,10 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { connectPublicService } from "../src/cli/commands/connect"; +import { + connectPublicService, + type ProviderLock, +} from "../src/cli/commands/connect"; import { contractFromManifest, digestLexiconDocuments, @@ -30,6 +33,19 @@ const sourceLexicon = { defs: { main: { type: "record" } }, }; +function providerLock(): ProviderLock { + return { + format: "contrail.provider-lock", + version: 1, + endpoint, + namespace: "atmo.rsvp", + contractDigest: `sha256:${"a".repeat(64)}`, + lexiconDigest: `sha256:${"b".repeat(64)}`, + methods: [method], + lexiconRoot: "lexicons/pulled/api.atmo.rsvp", + }; +} + async function serviceFixture(values = [methodLexicon, sourceLexicon]) { const { digest } = await digestLexiconDocuments(values); const manifest: PublicServiceManifest = { @@ -131,6 +147,44 @@ describe("contrail connect", () => { ).toBe("keep"); }); + it("refuses to repoint an existing lock or abandon its owned output", async () => { + const root = await temporaryRoot(); + const lockPath = join(root, "contrail.lock.json"); + const fetcher = vi.fn(); + await writeFile( + lockPath, + `${JSON.stringify({ + ...providerLock(), + endpoint: "https://old.example.com", + })}\n`, + ); + + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher, + update: true, + }), + ).rejects.toThrow("remove the existing connection before switching endpoints"); + expect(fetcher).not.toHaveBeenCalled(); + + await writeFile(lockPath, `${JSON.stringify(providerLock())}\n`); + await expect( + connectPublicService({ + endpoint, + root, + out: "different-lexicons", + lock: "contrail.lock.json", + fetcher, + update: true, + }), + ).rejects.toThrow("reuse its output path"); + expect(fetcher).not.toHaveBeenCalled(); + }); + it("preserves the previous provider and lock when an update fails validation", async () => { const root = await temporaryRoot(); const fixture = await serviceFixture(); -- 2.51.2 From fff7dfaf44dc2f229d243d660036732c66565ef0 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 06:55:39 +0200 Subject: [PATCH 08/15] Add the public Atmo RSVP service --- apps/atmo-rsvp/.gitignore | 3 + apps/atmo-rsvp/README.md | 54 ++++ apps/atmo-rsvp/lex.config.js | 31 +++ apps/atmo-rsvp/lexicons/generated/index.ts | 17 ++ .../generated/rsvp/atmo/event/getRecord.json | 163 +++++++++++ .../rsvp/atmo/event/listRecords.json | 263 ++++++++++++++++++ .../generated/rsvp/atmo/getCursor.json | 48 ++++ .../generated/rsvp/atmo/rsvp/getRecord.json | 112 ++++++++ .../generated/rsvp/atmo/rsvp/listRecords.json | 170 +++++++++++ apps/atmo-rsvp/lexicons/pulled/README.md | 5 + .../pulled/com/atproto/repo/strongRef.json | 25 ++ .../community/lexicon/calendar/event.json | 147 ++++++++++ .../community/lexicon/calendar/rsvp.json | 46 +++ .../community/lexicon/location/address.json | 42 +++ .../community/lexicon/location/fsq.json | 30 ++ .../community/lexicon/location/geo.json | 30 ++ .../community/lexicon/location/hthree.json | 24 ++ apps/atmo-rsvp/package.json | 27 ++ apps/atmo-rsvp/src/contrail.config.ts | 58 ++++ apps/atmo-rsvp/src/worker.ts | 10 + apps/atmo-rsvp/tests/contract.test.ts | 50 ++++ apps/atmo-rsvp/tsconfig.json | 14 + apps/atmo-rsvp/wrangler.jsonc | 25 ++ pnpm-lock.yaml | 60 ++++ todo/other-stuff.md | 3 +- 25 files changed, 1456 insertions(+), 1 deletion(-) create mode 100644 apps/atmo-rsvp/.gitignore create mode 100644 apps/atmo-rsvp/README.md create mode 100644 apps/atmo-rsvp/lex.config.js create mode 100644 apps/atmo-rsvp/lexicons/generated/index.ts create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getCursor.json create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/README.md create mode 100644 apps/atmo-rsvp/lexicons/pulled/com/atproto/repo/strongRef.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/event.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/rsvp.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/address.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/fsq.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/geo.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/hthree.json create mode 100644 apps/atmo-rsvp/package.json create mode 100644 apps/atmo-rsvp/src/contrail.config.ts create mode 100644 apps/atmo-rsvp/src/worker.ts create mode 100644 apps/atmo-rsvp/tests/contract.test.ts create mode 100644 apps/atmo-rsvp/tsconfig.json create mode 100644 apps/atmo-rsvp/wrangler.jsonc diff --git a/apps/atmo-rsvp/.gitignore b/apps/atmo-rsvp/.gitignore new file mode 100644 index 0000000..e8c2e88 --- /dev/null +++ b/apps/atmo-rsvp/.gitignore @@ -0,0 +1,3 @@ +node_modules +.wrangler +dist diff --git a/apps/atmo-rsvp/README.md b/apps/atmo-rsvp/README.md new file mode 100644 index 0000000..6e14e4e --- /dev/null +++ b/apps/atmo-rsvp/README.md @@ -0,0 +1,54 @@ +# api.atmo.rsvp + +Anonymous public Contrail read-through service for calendar events and RSVPs. Records remain owned by their authors' PDSes; reads may acquire and cache missing public data. + +Public discovery: + +```text +https://api.atmo.rsvp/.well-known/contrail +https://api.atmo.rsvp/lexicons +https://api.atmo.rsvp/status +``` + +Typed XRPC methods: + +```text +rsvp.atmo.getCursor +rsvp.atmo.event.getRecord +rsvp.atmo.event.listRecords +rsvp.atmo.rsvp.getRecord +rsvp.atmo.rsvp.listRecords +``` + +`getCursor` returns the committed opaque `{ source, epoch, cursor }` position of the primary Jetstream source. Clients compare complete positions for equality and fully refetch when the source or epoch changes. + +The service has no user sessions, service DID, or write proxy. Applications authenticate and write through users' PDSes. + +## Development + +```bash +pnpm --dir apps/atmo-rsvp lexicons:all +pnpm --dir apps/atmo-rsvp typecheck +pnpm --dir apps/atmo-rsvp dev +pnpm --dir apps/atmo-rsvp backfill:dev +``` + +The development backfill uses the local Wrangler/Miniflare D1 binding. It remains resumable and uses the same validation, retry, and completion logic as production ingestion. + +## Deployment + +```bash +pnpm --dir apps/atmo-rsvp lexicons:check +pnpm --dir apps/atmo-rsvp typecheck +pnpm --dir apps/atmo-rsvp deploy +``` + +The deployed D1 database is already provisioned. Production bulk provisioning does not use Wrangler's remote development proxy. The planned repeatable workflow builds and verifies a fresh native SQLite generation, imports canonical tables into a fresh D1 database, rebuilds derived projections, verifies readiness, and then activates it. + +Consumer projects connect after installing Contrail: + +```bash +pnpm contrail connect https://api.atmo.rsvp +``` + +That verifies the canonical service and Lexicon digests, writes a provider lock, installs the provider-owned Lexicons, and runs Atcute TypeScript generation. Reconnecting an existing project requires `--update`. diff --git a/apps/atmo-rsvp/lex.config.js b/apps/atmo-rsvp/lex.config.js new file mode 100644 index 0000000..8a1d6cf --- /dev/null +++ b/apps/atmo-rsvp/lex.config.js @@ -0,0 +1,31 @@ +import { defineLexiconConfig } from "@atcute/lex-cli"; + +export default defineLexiconConfig({ + generate: { + files: [ + "lexicons/custom/**/*.json", + "lexicons/pulled/**/*.json", + "lexicons/generated/**/*.json", + ], + outdir: "src/lexicon-types/", + }, + pull: { + outdir: "lexicons/pulled/", + clean: true, + sources: [ + { + type: "atproto", + mode: "nsids", + nsids: [ + "com.atproto.repo.strongRef", + "community.lexicon.calendar.event", + "community.lexicon.calendar.rsvp", + "community.lexicon.location.address", + "community.lexicon.location.fsq", + "community.lexicon.location.geo", + "community.lexicon.location.hthree" + ], + }, + ], + }, +}); diff --git a/apps/atmo-rsvp/lexicons/generated/index.ts b/apps/atmo-rsvp/lexicons/generated/index.ts new file mode 100644 index 0000000..7d5efaa --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/index.ts @@ -0,0 +1,17 @@ +// Auto-generated by @atmo-dev/contrail. Do not edit. +// Regenerate with `contrail lexicons generate`. + +import _0 from "../pulled/com/atproto/repo/strongRef.json"; +import _1 from "../pulled/community/lexicon/calendar/event.json"; +import _2 from "../pulled/community/lexicon/calendar/rsvp.json"; +import _3 from "../pulled/community/lexicon/location/address.json"; +import _4 from "../pulled/community/lexicon/location/fsq.json"; +import _5 from "../pulled/community/lexicon/location/geo.json"; +import _6 from "../pulled/community/lexicon/location/hthree.json"; +import _7 from "./rsvp/atmo/event/getRecord.json"; +import _8 from "./rsvp/atmo/event/listRecords.json"; +import _9 from "./rsvp/atmo/getCursor.json"; +import _10 from "./rsvp/atmo/rsvp/getRecord.json"; +import _11 from "./rsvp/atmo/rsvp/listRecords.json"; + +export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11]; diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json new file mode 100644 index 0000000..31fa80b --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json @@ -0,0 +1,163 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.event.getRecord", + "defs": { + "main": { + "type": "query", + "description": "Get a community.lexicon.calendar.event record by AT URI", + "parameters": { + "type": "params", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT URI of the record" + }, + "hydrateRsvps": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "description": "Number of rsvps records to embed" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.event#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "rsvpsCount": { + "type": "integer", + "description": "Total rsvps count" + }, + "rsvpsGoingCount": { + "type": "integer", + "description": "rsvps count where status = going" + }, + "rsvpsInterestedCount": { + "type": "integer", + "description": "rsvps count where status = interested" + }, + "rsvpsNotgoingCount": { + "type": "integer", + "description": "rsvps count where status = notgoing" + }, + "rsvps": { + "type": "ref", + "ref": "#hydrateRsvps" + } + } + } + } + }, + "hydrateRsvpsRecord": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.rsvp#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + }, + "hydrateRsvps": { + "type": "object", + "properties": { + "going": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "interested": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "notgoing": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "other": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json new file mode 100644 index 0000000..1e49974 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json @@ -0,0 +1,263 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.event.listRecords", + "defs": { + "main": { + "type": "query", + "description": "Query community.lexicon.calendar.event records", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 50 + }, + "cursor": { + "type": "string" + }, + "actor": { + "type": "string", + "format": "at-identifier", + "description": "Filter by an indexed DID or cached handle" + }, + "search": { + "type": "string", + "description": "Full-text search across: name, description" + }, + "createdAtMin": { + "type": "string", + "description": "Minimum value for createdAt" + }, + "createdAtMax": { + "type": "string", + "description": "Maximum value for createdAt" + }, + "endsAtMin": { + "type": "string", + "description": "Minimum value for endsAt" + }, + "endsAtMax": { + "type": "string", + "description": "Maximum value for endsAt" + }, + "mode": { + "type": "string", + "description": "Filter by mode" + }, + "startsAtMin": { + "type": "string", + "description": "Minimum value for startsAt" + }, + "startsAtMax": { + "type": "string", + "description": "Maximum value for startsAt" + }, + "status": { + "type": "string", + "description": "Filter by status" + }, + "rsvpsCountMin": { + "type": "integer", + "description": "Minimum total rsvps count" + }, + "rsvpsGoingCountMin": { + "type": "integer", + "description": "Minimum rsvps count where status = going" + }, + "rsvpsInterestedCountMin": { + "type": "integer", + "description": "Minimum rsvps count where status = interested" + }, + "rsvpsNotgoingCountMin": { + "type": "integer", + "description": "Minimum rsvps count where status = notgoing" + }, + "hydrateRsvps": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "description": "Number of rsvps records to embed" + }, + "sort": { + "type": "string", + "knownValues": [ + "createdAt", + "endsAt", + "mode", + "rsvpsCount", + "rsvpsGoingCount", + "rsvpsInterestedCount", + "rsvpsNotgoingCount", + "startsAt", + "status" + ], + "description": "Field to sort by (default: time_us)" + }, + "order": { + "type": "string", + "knownValues": [ + "asc", + "desc" + ], + "description": "Sort direction" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "type": "ref", + "ref": "#record" + } + }, + "cursor": { + "type": "string" + } + } + } + } + }, + "record": { + "type": "object", + "required": [ + "uri", + "cid", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.event#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "rsvpsCount": { + "type": "integer", + "description": "Total rsvps count" + }, + "rsvpsGoingCount": { + "type": "integer", + "description": "rsvps count where status = going" + }, + "rsvpsInterestedCount": { + "type": "integer", + "description": "rsvps count where status = interested" + }, + "rsvpsNotgoingCount": { + "type": "integer", + "description": "rsvps count where status = notgoing" + }, + "rsvps": { + "type": "ref", + "ref": "#hydrateRsvps" + } + } + }, + "hydrateRsvpsRecord": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.rsvp#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + }, + "hydrateRsvps": { + "type": "object", + "properties": { + "going": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "interested": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "notgoing": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "other": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getCursor.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getCursor.json new file mode 100644 index 0000000..03925c6 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getCursor.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.getCursor", + "defs": { + "main": { + "type": "query", + "description": "Get the committed primary ordered-source position", + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "position": { + "type": "ref", + "ref": "#sourcePosition" + }, + "updatedAt": { + "type": "integer" + }, + "updatedAtDate": { + "type": "string", + "format": "datetime" + } + } + } + } + }, + "sourcePosition": { + "type": "object", + "required": [ + "source", + "epoch", + "cursor" + ], + "properties": { + "source": { + "type": "string" + }, + "epoch": { + "type": "string" + }, + "cursor": { + "type": "string" + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json new file mode 100644 index 0000000..1a9ab26 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json @@ -0,0 +1,112 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.rsvp.getRecord", + "defs": { + "main": { + "type": "query", + "description": "Get a community.lexicon.calendar.rsvp record by AT URI", + "parameters": { + "type": "params", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "AT URI of the record" + }, + "hydrateEvent": { + "type": "boolean", + "description": "Embed the referenced event record" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.rsvp#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "event": { + "type": "ref", + "ref": "#refEventRecord" + } + } + } + } + }, + "refEventRecord": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.event#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json new file mode 100644 index 0000000..d0046b0 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json @@ -0,0 +1,170 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.rsvp.listRecords", + "defs": { + "main": { + "type": "query", + "description": "Query community.lexicon.calendar.rsvp records", + "parameters": { + "type": "params", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 50 + }, + "cursor": { + "type": "string" + }, + "actor": { + "type": "string", + "format": "at-identifier", + "description": "Filter by an indexed DID or cached handle" + }, + "createdAtMin": { + "type": "string", + "description": "Minimum value for createdAt" + }, + "createdAtMax": { + "type": "string", + "description": "Maximum value for createdAt" + }, + "status": { + "type": "string", + "description": "Filter by status" + }, + "subjectUri": { + "type": "string", + "description": "Filter by subject.uri" + }, + "hydrateEvent": { + "type": "boolean", + "description": "Embed the referenced event record" + }, + "sort": { + "type": "string", + "knownValues": [ + "createdAt", + "status", + "subjectUri" + ], + "description": "Field to sort by (default: time_us)" + }, + "order": { + "type": "string", + "knownValues": [ + "asc", + "desc" + ], + "description": "Sort direction" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "type": "ref", + "ref": "#record" + } + }, + "cursor": { + "type": "string" + } + } + } + } + }, + "record": { + "type": "object", + "required": [ + "uri", + "cid", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.rsvp#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "event": { + "type": "ref", + "ref": "#refEventRecord" + } + } + }, + "refEventRecord": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.event#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/pulled/README.md b/apps/atmo-rsvp/lexicons/pulled/README.md new file mode 100644 index 0000000..b77ef3e --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/README.md @@ -0,0 +1,5 @@ +# lexicon sources + +this directory contains lexicon documents pulled from the following sources: + +- atproto (nsids: com.atproto.repo.strongRef, community.lexicon.calendar.event, community.lexicon.calendar.rsvp, community.lexicon.location.address, community.lexicon.location.fsq, community.lexicon.location.geo, community.lexicon.location.hthree) diff --git a/apps/atmo-rsvp/lexicons/pulled/com/atproto/repo/strongRef.json b/apps/atmo-rsvp/lexicons/pulled/com/atproto/repo/strongRef.json new file mode 100644 index 0000000..fffb47b --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/com/atproto/repo/strongRef.json @@ -0,0 +1,25 @@ +{ + "id": "com.atproto.repo.strongRef", + "defs": { + "main": { + "type": "object", + "required": [ + "cid", + "uri" + ], + "properties": { + "cid": { + "type": "string", + "format": "cid" + }, + "uri": { + "type": "string", + "format": "at-uri" + } + } + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1, + "description": "A URI with a content-hash fingerprint." +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/event.json b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/event.json new file mode 100644 index 0000000..8da7e7a --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/event.json @@ -0,0 +1,147 @@ +{ + "id": "community.lexicon.calendar.event", + "defs": { + "uri": { + "type": "object", + "required": [ + "uri" + ], + "properties": { + "uri": { + "type": "string", + "format": "uri" + }, + "name": { + "type": "string", + "description": "The display name of the URI." + } + }, + "description": "A URI associated with the event." + }, + "main": { + "key": "tid", + "type": "record", + "record": { + "type": "object", + "required": [ + "createdAt", + "name" + ], + "properties": { + "mode": { + "ref": "community.lexicon.calendar.event#mode", + "type": "ref", + "description": "The attendance mode of the event." + }, + "name": { + "type": "string", + "description": "The name of the event." + }, + "uris": { + "type": "array", + "items": { + "ref": "community.lexicon.calendar.event#uri", + "type": "ref" + }, + "description": "URIs associated with the event." + }, + "endsAt": { + "type": "string", + "format": "datetime", + "description": "Client-declared timestamp when the event ends." + }, + "status": { + "ref": "community.lexicon.calendar.event#status", + "type": "ref", + "description": "The status of the event." + }, + "startsAt": { + "type": "string", + "format": "datetime", + "description": "Client-declared timestamp when the event starts." + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "Client-declared timestamp when the event was created." + }, + "locations": { + "type": "array", + "items": { + "refs": [ + "community.lexicon.calendar.event#uri", + "community.lexicon.location.address", + "community.lexicon.location.fsq", + "community.lexicon.location.geo", + "community.lexicon.location.hthree" + ], + "type": "union" + }, + "description": "The locations where the event takes place." + }, + "description": { + "type": "string", + "description": "The description of the event." + } + } + }, + "description": "A calendar event." + }, + "mode": { + "type": "string", + "default": "community.lexicon.calendar.event#inperson", + "description": "The mode of the event.", + "knownValues": [ + "community.lexicon.calendar.event#hybrid", + "community.lexicon.calendar.event#inperson", + "community.lexicon.calendar.event#virtual" + ] + }, + "hybrid": { + "type": "token", + "description": "A hybrid event that takes place both online and offline." + }, + "status": { + "type": "string", + "default": "community.lexicon.calendar.event#scheduled", + "description": "The status of the event.", + "knownValues": [ + "community.lexicon.calendar.event#cancelled", + "community.lexicon.calendar.event#planned", + "community.lexicon.calendar.event#postponed", + "community.lexicon.calendar.event#rescheduled", + "community.lexicon.calendar.event#scheduled" + ] + }, + "planned": { + "type": "token", + "description": "The event has been created, but not finalized." + }, + "virtual": { + "type": "token", + "description": "A virtual event that takes place online." + }, + "inperson": { + "type": "token", + "description": "An in-person event that takes place offline." + }, + "cancelled": { + "type": "token", + "description": "The event has been cancelled." + }, + "postponed": { + "type": "token", + "description": "The event has been postponed and a new start date has not been set." + }, + "scheduled": { + "type": "token", + "description": "The event has been created and scheduled." + }, + "rescheduled": { + "type": "token", + "description": "The event has been rescheduled." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/rsvp.json b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/rsvp.json new file mode 100644 index 0000000..a1586e4 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/calendar/rsvp.json @@ -0,0 +1,46 @@ +{ + "id": "community.lexicon.calendar.rsvp", + "defs": { + "main": { + "key": "tid", + "type": "record", + "record": { + "type": "object", + "required": [ + "status", + "subject" + ], + "properties": { + "status": { + "type": "string", + "default": "community.lexicon.calendar.rsvp#going", + "knownValues": [ + "community.lexicon.calendar.rsvp#going", + "community.lexicon.calendar.rsvp#interested", + "community.lexicon.calendar.rsvp#notgoing" + ] + }, + "subject": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + } + } + }, + "description": "An RSVP for an event." + }, + "going": { + "type": "token", + "description": "Going to the event" + }, + "notgoing": { + "type": "token", + "description": "Not going to the event" + }, + "interested": { + "type": "token", + "description": "Interested in the event" + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/address.json b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/address.json new file mode 100644 index 0000000..a77c90b --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/address.json @@ -0,0 +1,42 @@ +{ + "id": "community.lexicon.location.address", + "defs": { + "main": { + "type": "object", + "required": [ + "country" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the location." + }, + "region": { + "type": "string", + "description": "The administrative region of the country. For example, a state in the USA." + }, + "street": { + "type": "string", + "description": "The street address." + }, + "country": { + "type": "string", + "maxLength": 10, + "minLength": 2, + "description": "The ISO 3166 country code. Preferably the 2-letter code." + }, + "locality": { + "type": "string", + "description": "The locality of the region. For example, a city in the USA." + }, + "postalCode": { + "type": "string", + "description": "The postal code of the location." + } + }, + "description": "A physical location in the form of a street address." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/fsq.json b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/fsq.json new file mode 100644 index 0000000..649a5f2 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/fsq.json @@ -0,0 +1,30 @@ +{ + "id": "community.lexicon.location.fsq", + "defs": { + "main": { + "type": "object", + "required": [ + "fsq_place_id" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the location." + }, + "latitude": { + "type": "string" + }, + "longitude": { + "type": "string" + }, + "fsq_place_id": { + "type": "string", + "description": "The unique identifier of a Foursquare POI." + } + }, + "description": "A physical location contained in the Foursquare Open Source Places dataset." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/geo.json b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/geo.json new file mode 100644 index 0000000..be1cd2a --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/geo.json @@ -0,0 +1,30 @@ +{ + "id": "community.lexicon.location.geo", + "defs": { + "main": { + "type": "object", + "required": [ + "latitude", + "longitude" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the location." + }, + "altitude": { + "type": "string" + }, + "latitude": { + "type": "string" + }, + "longitude": { + "type": "string" + } + }, + "description": "A physical location in the form of a WGS84 coordinate." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/hthree.json b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/hthree.json new file mode 100644 index 0000000..9a5c0af --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/community/lexicon/location/hthree.json @@ -0,0 +1,24 @@ +{ + "id": "community.lexicon.location.hthree", + "defs": { + "main": { + "type": "object", + "required": [ + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the location." + }, + "value": { + "type": "string", + "description": "The h3 encoded location." + } + }, + "description": "A physical location in the form of a H3 encoded location." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/package.json b/apps/atmo-rsvp/package.json new file mode 100644 index 0000000..47809f8 --- /dev/null +++ b/apps/atmo-rsvp/package.json @@ -0,0 +1,27 @@ +{ + "name": "atmo-rsvp-public-contrail", + "version": "0.0.1", + "private": true, + "type": "module", + "scripts": { + "dev": "node ../../packages/contrail/dist/cli.js dev", + "deploy": "wrangler deploy", + "backfill:dev": "node ../../packages/contrail/dist/cli.js backfill --only records", + "lexicons": "node ../../packages/contrail/dist/cli.js lexicons generate --public", + "lexicons:all": "node ../../packages/contrail/dist/cli.js lexicons all --public", + "lexicons:check": "node ../../packages/contrail/dist/cli.js lexicons check --public", + "typecheck": "node ../../packages/contrail/dist/cli.js lexicons check --public && tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@atmo-dev/contrail": "workspace:*" + }, + "devDependencies": { + "@atcute/lex-cli": "^3.2.1", + "@atcute/lexicons": "^2.0.3", + "@cloudflare/workers-types": "^5.20260804.1", + "typescript": "^6.0.3", + "vitest": "^4.1.10", + "wrangler": "^4.118.0" + } +} diff --git a/apps/atmo-rsvp/src/contrail.config.ts b/apps/atmo-rsvp/src/contrail.config.ts new file mode 100644 index 0000000..1a6b7b7 --- /dev/null +++ b/apps/atmo-rsvp/src/contrail.config.ts @@ -0,0 +1,58 @@ +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { lexicons } from "../lexicons/generated"; + +export const config: ContrailConfig = { + namespace: "rsvp.atmo", + profiles: [], + constellation: false, + jetstreams: ["wss://jetstream1.us-east.bsky.network"], + orderedSource: { + source: "jetstream", + epoch: "api-atmo-rsvp-primary-2026-08", + }, + validation: { + lexicons: lexicons as unknown as NonNullable< + ContrailConfig["validation"] + >["lexicons"], + strict: true, + verifyCid: true, + }, + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { + mode: {}, + status: {}, + startsAt: { type: "range" }, + endsAt: { type: "range" }, + createdAt: { type: "range" }, + }, + searchable: ["name", "description"], + relations: { + rsvps: { + collection: "rsvp", + groupBy: "status", + groups: { + going: "community.lexicon.calendar.rsvp#going", + interested: "community.lexicon.calendar.rsvp#interested", + notgoing: "community.lexicon.calendar.rsvp#notgoing", + }, + }, + }, + }, + rsvp: { + collection: "community.lexicon.calendar.rsvp", + queryable: { + status: {}, + "subject.uri": {}, + createdAt: { type: "range" }, + }, + references: { + event: { + collection: "event", + field: "subject.uri", + }, + }, + }, + }, +}; diff --git a/apps/atmo-rsvp/src/worker.ts b/apps/atmo-rsvp/src/worker.ts new file mode 100644 index 0000000..9ea3b90 --- /dev/null +++ b/apps/atmo-rsvp/src/worker.ts @@ -0,0 +1,10 @@ +import { createWorker } from "@atmo-dev/contrail/worker"; +import { lexicons } from "../lexicons/generated"; +import { config } from "./contrail.config"; + +export default createWorker(config, { + lexicons, + publicService: { + endpoint: "https://api.atmo.rsvp", + }, +}); diff --git a/apps/atmo-rsvp/tests/contract.test.ts b/apps/atmo-rsvp/tests/contract.test.ts new file mode 100644 index 0000000..c38c80c --- /dev/null +++ b/apps/atmo-rsvp/tests/contract.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { + contractFromManifest, + describePublicService, + digestPublicContract, +} from "@atmo-dev/contrail"; +import { createWorker } from "@atmo-dev/contrail/worker"; +import { lexicons } from "../lexicons/generated"; +import { config } from "../src/contrail.config"; + +const EXPECTED_METHODS = [ + "rsvp.atmo.event.getRecord", + "rsvp.atmo.event.listRecords", + "rsvp.atmo.getCursor", + "rsvp.atmo.rsvp.getRecord", + "rsvp.atmo.rsvp.listRecords", +]; + +describe("api.atmo.rsvp public contract", () => { + it("advertises the exact generated calendar API", async () => { + const service = await describePublicService( + config, + { endpoint: "https://api.atmo.rsvp" }, + lexicons, + ); + + expect(service.manifest.namespace).toBe("rsvp.atmo"); + expect(service.manifest.methods).toEqual(EXPECTED_METHODS); + expect(service.manifest.methods).not.toContain("rsvp.atmo.getOverview"); + expect(service.manifest.methods).not.toContain("rsvp.atmo.notifyOfUpdate"); + expect(service.lexicons.map((document) => document.id)).not.toContain( + "com.atproto.label.defs", + ); + expect( + await digestPublicContract(contractFromManifest(service.manifest)), + ).toBe(service.manifest.contract.digest); + expect(service.manifest.contract.digest).not.toBe( + service.manifest.lexicons.digest, + ); + }); + + it("passes synchronous Worker startup validation", () => { + expect(() => + createWorker(config, { + lexicons, + publicService: { endpoint: "https://api.atmo.rsvp" }, + }), + ).not.toThrow(); + }); +}); diff --git a/apps/atmo-rsvp/tsconfig.json b/apps/atmo-rsvp/tsconfig.json new file mode 100644 index 0000000..dcfa099 --- /dev/null +++ b/apps/atmo-rsvp/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "types": ["@cloudflare/workers-types"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "isolatedModules": true + }, + "include": ["."] +} diff --git a/apps/atmo-rsvp/wrangler.jsonc b/apps/atmo-rsvp/wrangler.jsonc new file mode 100644 index 0000000..0f3a477 --- /dev/null +++ b/apps/atmo-rsvp/wrangler.jsonc @@ -0,0 +1,25 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "atmo-rsvp-contrail", + "main": "src/worker.ts", + "compatibility_date": "2025-12-25", + "observability": { + "enabled": true + }, + "routes": [ + { + "pattern": "api.atmo.rsvp", + "custom_domain": true + } + ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "atmo-rsvp-contrail", + "database_id": "8a483108-4614-41c4-b9db-86b7a2ea27ca" + } + ], + "triggers": { + "crons": ["*/1 * * * *"] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8b90d6..053154e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,31 @@ importers: specifier: ^6.0.3 version: 6.0.3 + apps/atmo-rsvp: + dependencies: + '@atmo-dev/contrail': + specifier: workspace:* + version: link:../../packages/contrail + devDependencies: + '@atcute/lex-cli': + specifier: ^3.2.1 + version: 3.2.1(@atcute/cbor@2.3.6(@atcute/cid@2.4.2))(@atcute/cid@2.4.2)(prettier@3.9.6)(typescript@6.0.3) + '@atcute/lexicons': + specifier: ^2.0.3 + version: 2.0.3 + '@cloudflare/workers-types': + specifier: ^5.20260804.1 + version: 5.20260804.1 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.10 + version: 4.1.10(@types/node@26.1.2)(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)) + wrangler: + specifier: ^4.118.0 + version: 4.118.0(@cloudflare/workers-types@5.20260804.1) + apps/benchmark: dependencies: '@atmo-dev/contrail': @@ -5681,6 +5706,14 @@ snapshots: optionalDependencies: vite: 8.2.0(@types/node@26.1.2)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.23.6) + '@vitest/mocker@4.1.10(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6))': + dependencies: + '@vitest/spy': 4.1.10 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6) + '@vitest/pretty-format@4.1.10': dependencies: tinyrainbow: 3.1.1 @@ -7206,6 +7239,33 @@ snapshots: transitivePeerDependencies: - msw + vitest@4.1.10(@types/node@26.1.2)(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)): + dependencies: + '@vitest/expect': 4.1.10 + '@vitest/mocker': 4.1.10(vite@8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6)) + '@vitest/pretty-format': 4.1.10 + '@vitest/runner': 4.1.10 + '@vitest/snapshot': 4.1.10 + '@vitest/spy': 4.1.10 + '@vitest/utils': 4.1.10 + es-module-lexer: 2.3.1 + expect-type: 1.4.0 + magic-string: 0.30.21 + obug: 2.1.4 + pathe: 2.0.3 + picomatch: 4.0.5 + std-env: 4.2.0 + tinybench: 2.9.0 + tinyexec: 1.3.0 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.1 + vite: 8.2.0(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.6) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 26.1.2 + transitivePeerDependencies: + - msw + w3c-keyname@2.2.8: {} web-haptics@0.0.6(svelte@5.56.8(@typescript-eslint/types@8.66.0)): diff --git a/todo/other-stuff.md b/todo/other-stuff.md index aee9482..8c976c8 100644 --- a/todo/other-stuff.md +++ b/todo/other-stuff.md @@ -1 +1,2 @@ -- allow running custom functions before ingestion (e.g. for filtering out unallowed writes) \ No newline at end of file +- allow running custom functions before ingestion (e.g. for filtering out unallowed writes) +- replace production `backfill --remote` usage with a repeatable fresh-generation pipeline: build and verify native SQLite, export canonical tables, import into fresh D1, rebuild derived projections, verify readiness, then activate; keep `contrail backfill` for local development -- 2.51.2 From f7fd28ad2141520938cb63c8f5a4d26742aa571d Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:48:30 +0200 Subject: [PATCH 09/15] Mark generated Atmo Lexicon config --- apps/atmo-rsvp/lex.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/atmo-rsvp/lex.config.js b/apps/atmo-rsvp/lex.config.js index 8a1d6cf..e710bda 100644 --- a/apps/atmo-rsvp/lex.config.js +++ b/apps/atmo-rsvp/lex.config.js @@ -1,3 +1,4 @@ +// Generated by `contrail lexicons generate`. Re-run the command to update; do not edit. import { defineLexiconConfig } from "@atcute/lex-cli"; export default defineLexiconConfig({ -- 2.51.2 From 9baa877ab7f9b7eb010492c02df2a820f9553bf6 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:30:50 +0200 Subject: [PATCH 10/15] Add AT Protocol service auth --- .changeset/atproto-service-auth.md | 5 + packages/contrail/package.json | 2 + packages/contrail/src/cli/commands/connect.ts | 3 + packages/contrail/src/core/router/feed.ts | 20 ++- packages/contrail/src/core/router/index.ts | 39 +++- packages/contrail/src/core/router/notify.ts | 23 ++- packages/contrail/src/core/service-auth.ts | 68 +++++++ packages/contrail/src/core/types.ts | 51 +++++- packages/contrail/src/lexicons/generate.ts | 21 ++- packages/contrail/src/public-service.ts | 93 +++++++++- packages/contrail/tests/connect.test.ts | 63 +++++++ packages/contrail/tests/service-auth.test.ts | 168 ++++++++++++++++++ pnpm-lock.yaml | 33 ++++ 13 files changed, 578 insertions(+), 11 deletions(-) create mode 100644 .changeset/atproto-service-auth.md create mode 100644 packages/contrail/src/core/service-auth.ts create mode 100644 packages/contrail/tests/service-auth.test.ts diff --git a/.changeset/atproto-service-auth.md b/.changeset/atproto-service-auth.md new file mode 100644 index 0000000..d677108 --- /dev/null +++ b/.changeset/atproto-service-auth.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": minor +--- + +Add discoverable AT Protocol service authentication for personalized feeds and authoritative update notifications. diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 5ab9994..f754305 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -79,12 +79,14 @@ "@atcute/lexicon-doc": "3.0.2", "@atcute/lexicons": "^2.0.3", "@atcute/tid": "1.1.4", + "@atcute/xrpc-server": "^2.0.2", "cac": "^7.0.0", "hono": "^4.13.0", "jiti": "^2.7.0", "valibot": "1.4.2" }, "devDependencies": { + "@atcute/crypto": "2.4.4", "@cloudflare/workers-types": "^5.20260804.1", "@types/node": "^26.1.2", "@types/pg": "^8.20.4", diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index b0d18b2..e973751 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -24,6 +24,7 @@ import { isPublicServiceManifest, normalizePublicServiceEndpoint, validateManifestContract, + type PublicServiceAuthContract, type PublicServiceManifest, } from "../../public-service.js"; import { generateLexiconTypesWithAtcute } from "../atcute.js"; @@ -47,6 +48,7 @@ export interface ProviderLock { contractDigest: string; lexiconDigest: string; methods: string[]; + serviceAuth: PublicServiceAuthContract | null; lexiconRoot: string; } @@ -257,6 +259,7 @@ export async function connectPublicService(options: { contractDigest: manifest.contract.digest, lexiconDigest: manifest.lexicons.digest, methods: [...manifest.methods].sort(), + serviceAuth: manifest.serviceAuth ?? null, lexiconRoot: relative(projectRoot, providerRoot), }; const lockDirectory = dirname(lockPath); diff --git a/packages/contrail/src/core/router/feed.ts b/packages/contrail/src/core/router/feed.ts index 8ef149e..6dc0974 100644 --- a/packages/contrail/src/core/router/feed.ts +++ b/packages/contrail/src/core/router/feed.ts @@ -1,3 +1,4 @@ +import type { Nsid } from "@atcute/lexicons/syntax"; import type { Context, Hono } from "hono"; import type { ContrailConfig, @@ -16,6 +17,7 @@ import { import { resolveActor } from "../identity"; import { backfillUser } from "../backfill"; import { runPipeline } from "./collection"; +import type { ServiceAuthGate } from "../service-auth"; const BACKFILL_TIMEOUT_MS = 30_000; const BACKFILL_REQUEST_TIMEOUT_MS = 10_000; @@ -223,7 +225,8 @@ async function maybeBackfillFeed( export function registerFeedRoutes( app: Hono, db: Database, - config: ContrailConfig + config: ContrailConfig, + serviceAuth?: ServiceAuthGate | null ): void { if (!config.feeds) return; @@ -243,8 +246,23 @@ export function registerFeedRoutes( return c.json({ error: "Unknown feed" }, 404); } + const method = `${ns}.getFeed` as Nsid; + const authorization = serviceAuth?.protects("getFeed") + ? await serviceAuth.authorize(c.req.raw, method) + : null; + if (authorization?.response) return authorization.response; + const did = await resolveActor(db, actor, config); if (!did) return c.json({ error: "Could not resolve actor" }, 400); + if ( + authorization?.principal?.issuer !== undefined && + authorization.principal.issuer !== did + ) { + return c.json( + { error: "feed actor must match the service-auth issuer" }, + 403 + ); + } await maybeBackfillFeed(c, db, config, did, feedName, feedConfig); diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index a928954..d7ff777 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -11,6 +11,7 @@ import { registerCollectionRoutes } from "./collection"; import { registerFeedRoutes } from "./feed"; import { registerNotifyRoute } from "./notify"; import { resolveProfiles } from "./profiles"; +import { createServiceAuthGate } from "../service-auth"; import { describePublicService, normalizeLexiconDocuments, @@ -32,7 +33,18 @@ export function createApp( options: CreateAppOptions = {}, ): Hono { const app = new Hono(); - app.use("*", cors()); + app.use( + "*", + cors({ + allowHeaders: [ + "Authorization", + "Content-Type", + "Atproto-Accept-Labelers", + ], + exposeHeaders: ["Atproto-Content-Labelers", "WWW-Authenticate"], + }), + ); + const serviceAuth = createServiceAuthGate(config); app.get("/", (c) => c.json({ status: "ok" })); app.get("/status", async (c) => { @@ -72,6 +84,27 @@ export function createApp( c.header("etag", `\"${manifest.contract.digest}\"`); return c.json(manifest); }); + if ( + serviceAuth && + serviceAuth.audience === + `did:web:${new URL(options.publicService.endpoint).hostname}` + ) { + app.get("/.well-known/did.json", (c) => { + c.header("content-type", "application/did+ld+json; charset=UTF-8"); + c.header("cache-control", "public, max-age=300"); + return c.json({ + "@context": ["https://www.w3.org/ns/did/v1"], + id: serviceAuth.audience, + service: [ + { + id: `${serviceAuth.audience}#contrail`, + type: "ContrailService", + serviceEndpoint: options.publicService!.endpoint, + }, + ], + }); + }); + } app.get("/lexicons", async (c) => { const service = await description; c.header("content-type", "application/json; charset=UTF-8"); @@ -140,8 +173,8 @@ export function createApp( registerCursorRoute(app, db, config); registerCollectionRoutes(app, db, config); - registerFeedRoutes(app, db, config); - registerNotifyRoute(app, db, config); + registerFeedRoutes(app, db, config, serviceAuth); + registerNotifyRoute(app, db, config, serviceAuth); return app; } diff --git a/packages/contrail/src/core/router/notify.ts b/packages/contrail/src/core/router/notify.ts index 9ecf5e5..1f11282 100644 --- a/packages/contrail/src/core/router/notify.ts +++ b/packages/contrail/src/core/router/notify.ts @@ -1,11 +1,12 @@ import type { Hono } from "hono"; import type { Database, ContrailConfig, IngestEvent } from "../types"; +import type { ServiceAuthGate } from "../service-auth"; import { shortNameForNsid, getFeedMutatingNsids } from "../types"; import { lookupExistingRecords } from "../db/records"; import { createIngestEvent, ingestRecords, recordTimeUs } from "../ingest"; import { runGatedFeedPrune } from "../jetstream"; import { getPDS } from "../client"; -import type { Did } from "@atcute/lexicons"; +import type { Did, Nsid } from "@atcute/lexicons"; import { parseCanonicalResourceUri } from "@atcute/lexicons/syntax"; /** Parse a canonical (DID-authority) record AT-URI into its components, or null @@ -286,7 +287,8 @@ export async function processNotifyUris( export function registerNotifyRoute( app: Hono, db: Database, - config: ContrailConfig + config: ContrailConfig, + serviceAuth?: ServiceAuthGate | null ) { // Endpoint is off by default. Set config.notify to true or a secret string to enable. if (!config.notify) return; @@ -295,6 +297,12 @@ export function registerNotifyRoute( const secret = typeof config.notify === "string" ? config.notify : null; app.post(`/xrpc/${ns}.notifyOfUpdate`, async (c) => { + const method = `${ns}.notifyOfUpdate` as Nsid; + const authorization = serviceAuth?.protects("notifyOfUpdate") + ? await serviceAuth.authorize(c.req.raw, method) + : null; + if (authorization?.response) return authorization.response; + if (secret) { const auth = c.req.header("Authorization"); if (auth !== `Bearer ${secret}`) { @@ -316,6 +324,17 @@ export function registerNotifyRoute( if (uris.length > MAX_NOTIFY_URIS) { return c.json({ error: `max ${MAX_NOTIFY_URIS} URIs per request` }, 400); } + if (authorization?.principal) { + for (const uri of uris) { + const parsed = parseAtUri(uri); + if (parsed && parsed.did !== authorization.principal.issuer) { + return c.json( + { error: "notified records must belong to the service-auth issuer" }, + 403 + ); + } + } + } const result = await processNotifyUris(db, config, uris); return c.json(result); diff --git a/packages/contrail/src/core/service-auth.ts b/packages/contrail/src/core/service-auth.ts new file mode 100644 index 0000000..0eec5bc --- /dev/null +++ b/packages/contrail/src/core/service-auth.ts @@ -0,0 +1,68 @@ +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, + type DidDocumentResolver, +} from "@atcute/identity-resolver"; +import type { Did, Nsid } from "@atcute/lexicons/syntax"; +import { ServiceJwtVerifier, type VerifiedJwt } from "@atcute/xrpc-server/auth"; +import { XRPCError } from "@atcute/xrpc-server"; +import type { AtprotoServiceAuthMethod, ContrailConfig } from "./types.js"; + +const AUTH_TIMEOUT_MS = 5_000; + +export interface ServiceAuthResult { + principal?: VerifiedJwt; + response?: Response; +} + +export interface ServiceAuthGate { + readonly audience: Did; + protects(method: AtprotoServiceAuthMethod): boolean; + authorize(request: Request, method: Nsid): Promise; +} + +function defaultResolver(): DidDocumentResolver { + return new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, + }); +} + +/** Create the shared verifier used by protected built-in routes. Tokens remain + * method-bound even when a client obtained permission through one wildcard + * OAuth scope (`rpc?lxm=*&aud=`). */ +export function createServiceAuthGate( + config: ContrailConfig, +): ServiceAuthGate | null { + if (!config.serviceAuth) return null; + const serviceAuth = config.serviceAuth; + const protectedMethods = new Set(serviceAuth.methods); + const audience = serviceAuth.audience as Did; + const verifier = new ServiceJwtVerifier({ + acceptAudiences: [audience], + resolver: serviceAuth.resolver ?? defaultResolver(), + maxAge: serviceAuth.maxTokenAgeSeconds, + }); + + return { + audience, + protects(method) { + return protectedMethods.has(method); + }, + async authorize(request, method) { + try { + const principal = await verifier.verifyRequest(request, { + lxm: method, + signal: AbortSignal.timeout(AUTH_TIMEOUT_MS), + }); + return { principal }; + } catch (error) { + if (error instanceof XRPCError) return { response: error.toResponse() }; + throw error; + } + }, + }; +} diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index e71cdb5..610c683 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -1,4 +1,5 @@ import type { LexiconDoc } from "@atcute/lexicon-doc"; +import { isDid } from "@atcute/lexicons/syntax"; import type { SqlDialect } from "./dialect"; // Database interface — D1 implements this natively @@ -245,6 +246,19 @@ export interface OrderedSourceConfig { epoch: string; } +export type AtprotoServiceAuthMethod = "getFeed" | "notifyOfUpdate"; + +export interface AtprotoServiceAuthConfig { + /** Plain service DID used as the exact JWT audience. */ + audience: string; + /** Built-in methods that require a method-bound AT Protocol service token. */ + methods: AtprotoServiceAuthMethod[]; + /** Maximum accepted token lifetime and age. Default: 300 seconds. */ + maxTokenAgeSeconds?: number; + /** Optional DID resolver for private networks or controlled resolution. */ + resolver?: import("@atcute/identity-resolver").DidDocumentResolver; +} + export interface ContrailConfig { namespace: string; /** Collections to index, keyed by short name. Short names become endpoint URL segments @@ -270,8 +284,11 @@ export interface ContrailConfig { feeds?: Record; logger?: Logger; /** Expose the notifyOfUpdate HTTP endpoint. Off by default. - * Set to `true` for open access, or a string to require `Authorization: Bearer `. */ + * Set to `true` for open access, or a string to require `Authorization: Bearer `. + * Prefer `serviceAuth.methods: ["notifyOfUpdate"]` for portable user auth. */ notify?: boolean | string; + /** Verify method-bound AT Protocol service JWTs for selected built-in routes. */ + serviceAuth?: AtprotoServiceAuthConfig; /** Labels module configuration. When set, contrail subscribes to the * configured labelers, indexes their labels into a single `labels` table, * and hydrates `record.labels` onto `listRecords` / `getRecord` / profile @@ -399,6 +416,38 @@ export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig { ) { throw new TypeError("orderedSource requires non-empty source and epoch values"); } + if (config.serviceAuth) { + if (!isDid(config.serviceAuth.audience)) { + throw new TypeError("serviceAuth.audience must be a plain DID"); + } + if ( + !Array.isArray(config.serviceAuth.methods) || + new Set(config.serviceAuth.methods).size !== config.serviceAuth.methods.length + ) { + throw new TypeError("serviceAuth.methods must contain unique methods"); + } + if ( + config.serviceAuth.methods.includes("getFeed") && + (!config.feeds || Object.keys(config.feeds).length === 0) + ) { + throw new TypeError("serviceAuth cannot protect getFeed without configured feeds"); + } + if ( + config.serviceAuth.methods.includes("notifyOfUpdate") && + config.notify !== true + ) { + throw new TypeError( + "serviceAuth notifyOfUpdate requires notify: true and replaces shared-secret auth", + ); + } + if ( + config.serviceAuth.maxTokenAgeSeconds !== undefined && + (!Number.isSafeInteger(config.serviceAuth.maxTokenAgeSeconds) || + config.serviceAuth.maxTokenAgeSeconds <= 0) + ) { + throw new TypeError("serviceAuth.maxTokenAgeSeconds must be a positive integer"); + } + } const profiles = (config.profiles ?? DEFAULT_PROFILES).map( normalizeProfileConfig ); diff --git a/packages/contrail/src/lexicons/generate.ts b/packages/contrail/src/lexicons/generate.ts index 56574c8..4107c9f 100644 --- a/packages/contrail/src/lexicons/generate.ts +++ b/packages/contrail/src/lexicons/generate.ts @@ -807,13 +807,19 @@ export function generateLexicons( } const feed = feedLexicon(config, sourceDirs); if (feed) emit(`${config.namespace}.getFeed`, feed); - if (surface === "full" && config.notify) { + if ( + config.notify && + (surface === "full" || + config.serviceAuth?.methods.includes("notifyOfUpdate") === true) + ) { emit(`${config.namespace}.notifyOfUpdate`, { lexicon: 1, id: `${config.namespace}.notifyOfUpdate`, defs: { main: { type: "procedure", + description: + "Fetch changed records from their authoritative PDS for immediate indexing", input: { encoding: "application/json", schema: { @@ -830,7 +836,18 @@ export function generateLexicons( }, output: { encoding: "application/json", - schema: { type: "unknown" }, + schema: { + type: "object", + required: ["indexed", "deleted"], + properties: { + indexed: { type: "integer", minimum: 0 }, + deleted: { type: "integer", minimum: 0 }, + errors: { + type: "array", + items: { type: "string" }, + }, + }, + }, }, }, }, diff --git a/packages/contrail/src/public-service.ts b/packages/contrail/src/public-service.ts index efc4085..2beef81 100644 --- a/packages/contrail/src/public-service.ts +++ b/packages/contrail/src/public-service.ts @@ -1,4 +1,4 @@ -import { isNsid } from "@atcute/lexicons/syntax"; +import { isDid, isNsid } from "@atcute/lexicons/syntax"; import type { ContrailConfig } from "./core/types.js"; import { getCollectionMethods, @@ -21,12 +21,24 @@ export interface PublicServiceCollection { references: string[]; } +export interface PublicServiceProtectedMethod { + id: string; + type: "query" | "procedure"; +} + +export interface PublicServiceAuthContract { + type: "atproto-service-auth"; + audience: string; + methods: PublicServiceProtectedMethod[]; +} + export interface PublicContract { format: "contrail.contract"; version: 1; namespace: string; collections: PublicServiceCollection[]; methods: string[]; + serviceAuth?: PublicServiceAuthContract | null; lexiconDigest: string; } @@ -40,6 +52,7 @@ export interface PublicServiceManifest { status: { url: string }; collections: PublicServiceCollection[]; methods: string[]; + serviceAuth?: PublicServiceAuthContract | null; } export interface LexiconDocument { @@ -151,22 +164,46 @@ function publicTopLevelMethods(config: ContrailConfig): string[] { return methods; } +function publicServiceAuth( + config: ContrailConfig, +): PublicServiceAuthContract | null { + if (!config.serviceAuth || config.serviceAuth.methods.length === 0) + return null; + const methods = config.serviceAuth.methods + .map((method): PublicServiceProtectedMethod => + method === "getFeed" + ? { id: `${config.namespace}.getFeed`, type: "query" } + : { id: `${config.namespace}.notifyOfUpdate`, type: "procedure" }, + ) + .sort((left, right) => left.id.localeCompare(right.id)); + return { + type: "atproto-service-auth", + audience: config.serviceAuth.audience, + methods, + }; +} + export function createPublicContract( config: ContrailConfig, lexiconDigest: string, ): PublicContract { const resolved = resolveConfig(config); const collections = publicCollections(resolved); + const serviceAuth = publicServiceAuth(resolved); + const protectedMethods = new Set( + serviceAuth?.methods.map((method) => method.id) ?? [], + ); const methods = [ ...publicTopLevelMethods(resolved), ...collections.flatMap((collection) => collection.methods), - ]; + ].filter((method) => !protectedMethods.has(method)); return { format: "contrail.contract", version: 1, namespace: resolved.namespace, collections, methods: [...new Set(methods)].sort(), + serviceAuth, lexiconDigest, }; } @@ -195,6 +232,15 @@ export function validateContractLexicons( ); } } + for (const method of contract.serviceAuth?.methods ?? []) { + const document = byId.get(method.id) as + { defs?: { main?: { type?: unknown } } } | undefined; + if (document?.defs?.main?.type !== method.type) { + throw new Error( + `protected method requires a matching ${method.type} Lexicon: ${method.id}`, + ); + } + } return lexicons; } @@ -261,6 +307,7 @@ export async function describePublicService( status: { url: `${endpoint}/status` }, collections: contract.collections, methods: contract.methods, + serviceAuth: contract.serviceAuth, }; return { endpoint, lexicons, manifest, canonicalLexicons }; } @@ -278,6 +325,7 @@ export function contractFromManifest( namespace: manifest.namespace, collections: manifest.collections, methods: manifest.methods, + serviceAuth: manifest.serviceAuth, lexiconDigest: manifest.lexicons.digest, }; } @@ -286,6 +334,9 @@ export function validateManifestContract( manifest: PublicServiceManifest, values: readonly object[], ): LexiconDocument[] { + if (!isPublicServiceAuthContract(manifest.serviceAuth)) { + throw new Error("service manifest contains invalid service auth"); + } if (!uniqueStrings(manifest.methods)) { throw new Error("service manifest contains duplicate methods"); } @@ -297,6 +348,21 @@ export function validateManifestContract( if (manifest.methods.some((method) => !method.startsWith(prefix))) { throw new Error("service manifest method is outside its namespace"); } + const protectedMethods = manifest.serviceAuth?.methods ?? []; + const protectedIds = protectedMethods.map((method) => method.id); + if (!uniqueStrings(protectedIds)) { + throw new Error("service manifest contains duplicate protected methods"); + } + if (protectedIds.some((method) => !method.startsWith(prefix))) { + throw new Error( + "service manifest protected method is outside its namespace", + ); + } + if (protectedIds.some((method) => manifest.methods.includes(method))) { + throw new Error( + "service manifest method cannot be both anonymous and protected", + ); + } const advertised = new Set(manifest.methods); for (const collection of manifest.collections) { if (!uniqueStrings(collection.methods)) { @@ -315,6 +381,28 @@ export function validateManifestContract( return validateContractLexicons(contractFromManifest(manifest), values); } +function isPublicServiceAuthContract( + value: unknown, +): value is PublicServiceAuthContract | null | undefined { + if (value === null || value === undefined) return true; + if (!value || typeof value !== "object") return false; + const auth = value as Partial; + return ( + auth.type === "atproto-service-auth" && + typeof auth.audience === "string" && + isDid(auth.audience) && + Array.isArray(auth.methods) && + auth.methods.every( + (method) => + !!method && + typeof method === "object" && + typeof method.id === "string" && + isNsid(method.id) && + (method.type === "query" || method.type === "procedure"), + ) + ); +} + export function isPublicServiceManifest( value: unknown, ): value is PublicServiceManifest { @@ -335,6 +423,7 @@ export function isPublicServiceManifest( typeof manifest.status?.url !== "string" || !Array.isArray(manifest.collections) || !Array.isArray(manifest.methods) || + !isPublicServiceAuthContract(manifest.serviceAuth) || !manifest.methods.every( (method) => typeof method === "string" && isNsid(method), ) diff --git a/packages/contrail/tests/connect.test.ts b/packages/contrail/tests/connect.test.ts index 58a4714..3e8e765 100644 --- a/packages/contrail/tests/connect.test.ts +++ b/packages/contrail/tests/connect.test.ts @@ -32,6 +32,12 @@ const sourceLexicon = { id: "community.lexicon.calendar.event", defs: { main: { type: "record" } }, }; +const notifyMethod = "atmo.rsvp.notifyOfUpdate"; +const notifyLexicon = { + lexicon: 1, + id: notifyMethod, + defs: { main: { type: "procedure" } }, +}; function providerLock(): ProviderLock { return { @@ -42,6 +48,7 @@ function providerLock(): ProviderLock { contractDigest: `sha256:${"a".repeat(64)}`, lexiconDigest: `sha256:${"b".repeat(64)}`, methods: [method], + serviceAuth: null, lexiconRoot: "lexicons/pulled/api.atmo.rsvp", }; } @@ -68,6 +75,7 @@ async function serviceFixture(values = [methodLexicon, sourceLexicon]) { }, ], methods: [method], + serviceAuth: null, }; manifest.contract.digest = await digestPublicContract( contractFromManifest(manifest), @@ -185,6 +193,34 @@ describe("contrail connect", () => { expect(fetcher).not.toHaveBeenCalled(); }); + it("locks discoverable service-auth methods separately", async () => { + const root = await temporaryRoot(); + const fixture = await serviceFixture([ + methodLexicon, + sourceLexicon, + notifyLexicon, + ]); + fixture.manifest.serviceAuth = { + type: "atproto-service-auth", + audience: "did:web:api.atmo.rsvp", + methods: [{ id: notifyMethod, type: "procedure" }], + }; + fixture.manifest.contract.digest = await digestPublicContract( + contractFromManifest(fixture.manifest), + ); + + const result = await connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: fixture.fetcher, + }); + + expect(result.lock.methods).toEqual([method]); + expect(result.lock.serviceAuth).toEqual(fixture.manifest.serviceAuth); + }); + it("preserves the previous provider and lock when an update fails validation", async () => { const root = await temporaryRoot(); const fixture = await serviceFixture(); @@ -262,6 +298,33 @@ describe("contrail connect", () => { ).rejects.toThrow("matching query Lexicon"); }); + it("rejects protected methods without matching procedure or query Lexicons", async () => { + const root = await temporaryRoot(); + const fixture = await serviceFixture([ + methodLexicon, + sourceLexicon, + { ...notifyLexicon, defs: { main: { type: "query" } } }, + ]); + fixture.manifest.serviceAuth = { + type: "atproto-service-auth", + audience: "did:web:api.atmo.rsvp", + methods: [{ id: notifyMethod, type: "procedure" }], + }; + fixture.manifest.contract.digest = await digestPublicContract( + contractFromManifest(fixture.manifest), + ); + + await expect( + connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: fixture.fetcher, + }), + ).rejects.toThrow("matching procedure Lexicon"); + }); + it("rejects inconsistent method namespaces and collection capabilities", async () => { const root = await temporaryRoot(); const outside = await serviceFixture(); diff --git a/packages/contrail/tests/service-auth.test.ts b/packages/contrail/tests/service-auth.test.ts new file mode 100644 index 0000000..8550f30 --- /dev/null +++ b/packages/contrail/tests/service-auth.test.ts @@ -0,0 +1,168 @@ +import { Secp256k1PrivateKeyExportable } from "@atcute/crypto"; +import type { Did, Nsid } from "@atcute/lexicons/syntax"; +import { createServiceJwt } from "@atcute/xrpc-server/auth"; +import { beforeAll, describe, expect, it } from "vitest"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { createApp } from "../src/core/router"; +import { + initSchema, + resolveConfig, + type ContrailConfig, + type Database, +} from "../src/index"; + +const issuer = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" as Did; +const other = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" as Did; +const audience = "did:web:api.example.com" as Did; +let keypair: Secp256k1PrivateKeyExportable; + +beforeAll(async () => { + keypair = await Secp256k1PrivateKeyExportable.createKeypair(); +}); + +function config(): ContrailConfig { + return { + namespace: "com.example", + profiles: [], + notify: true, + serviceAuth: { + audience, + methods: ["getFeed", "notifyOfUpdate"], + resolver: { + async resolve(did) { + return { + "@context": [], + id: did, + verificationMethod: [ + { + id: `${did}#atproto`, + type: "Multikey", + controller: did, + publicKeyMultibase: await keypair.exportPublicKey("multikey"), + }, + ], + }; + }, + }, + }, + collections: { + event: { collection: "community.example.event" }, + follow: { + collection: "app.bsky.graph.follow", + discover: false, + subjectField: "subject", + methods: [], + }, + }, + feeds: { network: { targets: ["event"] } }, + }; +} + +async function setup(): Promise<{ + db: Database; + app: ReturnType; +}> { + const resolved = resolveConfig(config()); + const db = createSqliteDatabase(":memory:"); + await initSchema(db, resolved); + for (const did of [issuer, other]) { + await db + .prepare( + "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, NULL, ?, ?)", + ) + .bind(did, "https://pds.example.com", Date.now()) + .run(); + } + await db + .prepare( + "INSERT INTO feed_backfills (actor, feed, completed) VALUES (?, 'network', 1)", + ) + .bind(issuer) + .run(); + return { db, app: createApp(db, resolved) }; +} + +async function token(lxm: string, options: { aud?: Did; iss?: Did } = {}) { + return createServiceJwt({ + keypair, + issuer: options.iss ?? issuer, + audience: options.aud ?? audience, + lxm: lxm as Nsid, + }); +} + +function authorized(url: string, jwt: string, init: RequestInit = {}) { + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${jwt}`); + return new Request(url, { ...init, headers }); +} + +describe("AT Protocol service auth", () => { + it("requires exact audience and method-bound tokens", async () => { + const { app } = await setup(); + const url = `https://api.example.com/xrpc/com.example.getFeed?feed=network&actor=${issuer}`; + + const missing = await app.fetch(new Request(url)); + expect(missing.status).toBe(401); + expect(missing.headers.get("www-authenticate")).toContain("Bearer"); + + const wrongMethod = await app.fetch( + authorized(url, await token("com.example.notifyOfUpdate")), + ); + expect(wrongMethod.status).toBe(401); + expect(wrongMethod.headers.get("www-authenticate")).toContain( + "BadJwtLexiconMethod", + ); + + const wrongAudience = await app.fetch( + authorized( + url, + await token("com.example.getFeed", { + aud: "did:web:other.example.com" as Did, + }), + ), + ); + expect(wrongAudience.status).toBe(401); + }); + + it("binds a personalized feed to the token issuer", async () => { + const { app } = await setup(); + const jwt = await token("com.example.getFeed"); + + const allowed = await app.fetch( + authorized( + `https://api.example.com/xrpc/com.example.getFeed?feed=network&actor=${issuer}`, + jwt, + ), + ); + expect(allowed.status).toBe(200); + + const forbidden = await app.fetch( + authorized( + `https://api.example.com/xrpc/com.example.getFeed?feed=network&actor=${other}`, + jwt, + ), + ); + expect(forbidden.status).toBe(403); + }); + + it("only lets an issuer notify its own record URIs", async () => { + const { app } = await setup(); + const jwt = await token("com.example.notifyOfUpdate"); + const response = await app.fetch( + authorized( + "https://api.example.com/xrpc/com.example.notifyOfUpdate", + jwt, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + uri: `at://${other}/community.example.event/1`, + }), + }, + ), + ); + + expect(response.status).toBe(403); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 053154e..82c7c18 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -253,6 +253,9 @@ importers: '@atcute/tid': specifier: 1.1.4 version: 1.1.4 + '@atcute/xrpc-server': + specifier: ^2.0.2 + version: 2.0.2(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.3)(typescript@6.0.3) cac: specifier: ^7.0.0 version: 7.0.0 @@ -266,6 +269,9 @@ importers: specifier: 1.4.2 version: 1.4.2(typescript@6.0.3) devDependencies: + '@atcute/crypto': + specifier: 2.4.4 + version: 2.4.4 '@cloudflare/workers-types': specifier: ^5.20260804.1 version: 5.20260804.1 @@ -428,6 +434,11 @@ packages: '@atcute/varint@2.0.2': resolution: {integrity: sha512-/+hS1juMgnmf6eL6lICUkTw7wcGTo3I+Q0L1PI521mUz77rGSC6nXAUNKtvm2wYJpuWdEGq+GILGoYkOArn0TQ==} + '@atcute/xrpc-server@2.0.2': + resolution: {integrity: sha512-tjOAqJfrBOwit6ccKg5s/eosvzoHmRszafS008/VP0yYdZkiBcSgYpy9S32e4aVoKeVurudBZjGD937bMcLg1Q==} + peerDependencies: + '@atcute/lexicons': ^2.0.0 + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} @@ -3062,6 +3073,11 @@ packages: engines: {node: ^18 || >=20} hasBin: true + nanoid@6.0.1: + resolution: {integrity: sha512-3wVS3i51pE2pi1k5FFL/95BGfVS0kSsvDVuGXHOtxox/TywUmtgq+3qiTOTbs9J7KfHaXPiN171k/A6dBnaXFw==} + engines: {node: ^22 || ^24 || >=26} + hasBin: true + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -4181,6 +4197,21 @@ snapshots: '@atcute/varint@2.0.2': {} + '@atcute/xrpc-server@2.0.2(@atcute/cid@2.4.2)(@atcute/lexicons@2.0.3)(typescript@6.0.3)': + dependencies: + '@atcute/cbor': 2.3.6(@atcute/cid@2.4.2) + '@atcute/crypto': 2.4.4 + '@atcute/identity': 2.0.2(@atcute/lexicons@2.0.3)(typescript@6.0.3) + '@atcute/identity-resolver': 2.0.1(@atcute/identity@2.0.2(@atcute/lexicons@2.0.3)(typescript@6.0.3))(@atcute/lexicons@2.0.3)(typescript@6.0.3) + '@atcute/lexicons': 2.0.3 + '@atcute/multibase': 1.2.5 + '@atcute/uint8array': 1.1.5 + nanoid: 6.0.1 + valibot: 1.4.2(typescript@6.0.3) + transitivePeerDependencies: + - '@atcute/cid' + - typescript + '@babel/runtime@7.29.7': {} '@changesets/apply-release-plan@7.1.1': @@ -6468,6 +6499,8 @@ snapshots: nanoid@5.1.16: {} + nanoid@6.0.1: {} + natural-compare@1.4.0: {} number-flow@0.6.2: -- 2.51.2 From bfafd93b211d843ac1ef9c301d61435658fb5d38 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:31:07 +0200 Subject: [PATCH 11/15] Add authenticated Atmo profiles and feeds --- apps/atmo-rsvp/lex.config.js | 3 + apps/atmo-rsvp/lexicons/generated/index.ts | 32 +- .../generated/rsvp/atmo/event/getRecord.json | 45 ++ .../rsvp/atmo/event/listRecords.json | 45 ++ .../lexicons/generated/rsvp/atmo/getFeed.json | 419 ++++++++++++++++++ .../generated/rsvp/atmo/getProfile.json | 73 +++ .../generated/rsvp/atmo/notifyOfUpdate.json | 56 +++ .../generated/rsvp/atmo/rsvp/getRecord.json | 45 ++ .../generated/rsvp/atmo/rsvp/listRecords.json | 45 ++ apps/atmo-rsvp/lexicons/pulled/README.md | 2 +- .../pulled/app/bsky/actor/profile.json | 75 ++++ .../pulled/app/bsky/graph/follow.json | 33 ++ .../pulled/com/atproto/label/defs.json | 190 ++++++++ apps/atmo-rsvp/src/contrail.config.ts | 34 +- apps/atmo-rsvp/tests/contract.test.ts | 74 +++- apps/atmo-rsvp/wrangler.jsonc | 4 +- 16 files changed, 1147 insertions(+), 28 deletions(-) create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getFeed.json create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getProfile.json create mode 100644 apps/atmo-rsvp/lexicons/generated/rsvp/atmo/notifyOfUpdate.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/app/bsky/actor/profile.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/app/bsky/graph/follow.json create mode 100644 apps/atmo-rsvp/lexicons/pulled/com/atproto/label/defs.json diff --git a/apps/atmo-rsvp/lex.config.js b/apps/atmo-rsvp/lex.config.js index e710bda..2447175 100644 --- a/apps/atmo-rsvp/lex.config.js +++ b/apps/atmo-rsvp/lex.config.js @@ -18,6 +18,9 @@ export default defineLexiconConfig({ type: "atproto", mode: "nsids", nsids: [ + "app.bsky.actor.profile", + "app.bsky.graph.follow", + "com.atproto.label.defs", "com.atproto.repo.strongRef", "community.lexicon.calendar.event", "community.lexicon.calendar.rsvp", diff --git a/apps/atmo-rsvp/lexicons/generated/index.ts b/apps/atmo-rsvp/lexicons/generated/index.ts index 7d5efaa..f1d79b9 100644 --- a/apps/atmo-rsvp/lexicons/generated/index.ts +++ b/apps/atmo-rsvp/lexicons/generated/index.ts @@ -1,17 +1,23 @@ // Auto-generated by @atmo-dev/contrail. Do not edit. // Regenerate with `contrail lexicons generate`. -import _0 from "../pulled/com/atproto/repo/strongRef.json"; -import _1 from "../pulled/community/lexicon/calendar/event.json"; -import _2 from "../pulled/community/lexicon/calendar/rsvp.json"; -import _3 from "../pulled/community/lexicon/location/address.json"; -import _4 from "../pulled/community/lexicon/location/fsq.json"; -import _5 from "../pulled/community/lexicon/location/geo.json"; -import _6 from "../pulled/community/lexicon/location/hthree.json"; -import _7 from "./rsvp/atmo/event/getRecord.json"; -import _8 from "./rsvp/atmo/event/listRecords.json"; -import _9 from "./rsvp/atmo/getCursor.json"; -import _10 from "./rsvp/atmo/rsvp/getRecord.json"; -import _11 from "./rsvp/atmo/rsvp/listRecords.json"; +import _0 from "../pulled/app/bsky/actor/profile.json"; +import _1 from "../pulled/app/bsky/graph/follow.json"; +import _2 from "../pulled/com/atproto/label/defs.json"; +import _3 from "../pulled/com/atproto/repo/strongRef.json"; +import _4 from "../pulled/community/lexicon/calendar/event.json"; +import _5 from "../pulled/community/lexicon/calendar/rsvp.json"; +import _6 from "../pulled/community/lexicon/location/address.json"; +import _7 from "../pulled/community/lexicon/location/fsq.json"; +import _8 from "../pulled/community/lexicon/location/geo.json"; +import _9 from "../pulled/community/lexicon/location/hthree.json"; +import _10 from "./rsvp/atmo/event/getRecord.json"; +import _11 from "./rsvp/atmo/event/listRecords.json"; +import _12 from "./rsvp/atmo/getCursor.json"; +import _13 from "./rsvp/atmo/getFeed.json"; +import _14 from "./rsvp/atmo/getProfile.json"; +import _15 from "./rsvp/atmo/notifyOfUpdate.json"; +import _16 from "./rsvp/atmo/rsvp/getRecord.json"; +import _17 from "./rsvp/atmo/rsvp/listRecords.json"; -export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11]; +export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17]; diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json index 31fa80b..0b9e019 100644 --- a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/getRecord.json @@ -16,6 +16,10 @@ "format": "at-uri", "description": "AT URI of the record" }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + }, "hydrateRsvps": { "type": "integer", "minimum": 1, @@ -82,6 +86,13 @@ "rsvps": { "type": "ref", "ref": "#hydrateRsvps" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } } } } @@ -158,6 +169,40 @@ } } } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } } } } diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json index 1e49974..43e8c64 100644 --- a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/event/listRecords.json @@ -22,6 +22,10 @@ "format": "at-identifier", "description": "Filter by an indexed DID or cached handle" }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + }, "search": { "type": "string", "description": "Full-text search across: name, description" @@ -122,6 +126,13 @@ }, "cursor": { "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } } } } @@ -258,6 +269,40 @@ } } } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } } } } diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getFeed.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getFeed.json new file mode 100644 index 0000000..e971631 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getFeed.json @@ -0,0 +1,419 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.getFeed", + "defs": { + "main": { + "type": "query", + "description": "Get a configured personalized feed", + "parameters": { + "type": "params", + "required": [ + "feed", + "actor" + ], + "properties": { + "feed": { + "type": "string", + "knownValues": [ + "network" + ] + }, + "actor": { + "type": "string", + "format": "at-identifier", + "description": "DID or handle whose feed should be queried" + }, + "collection": { + "type": "string", + "knownValues": [ + "community.lexicon.calendar.event", + "community.lexicon.calendar.rsvp" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "default": 50 + }, + "cursor": { + "type": "string" + }, + "profiles": { + "type": "boolean" + }, + "search": { + "type": "string", + "description": "Full-text search across: name, description" + }, + "createdAtMin": { + "type": "string", + "description": "Minimum value for createdAt" + }, + "createdAtMax": { + "type": "string", + "description": "Maximum value for createdAt" + }, + "endsAtMin": { + "type": "string", + "description": "Minimum value for endsAt" + }, + "endsAtMax": { + "type": "string", + "description": "Maximum value for endsAt" + }, + "mode": { + "type": "string", + "description": "Filter by mode" + }, + "startsAtMin": { + "type": "string", + "description": "Minimum value for startsAt" + }, + "startsAtMax": { + "type": "string", + "description": "Maximum value for startsAt" + }, + "status": { + "type": "string", + "description": "Filter by status" + }, + "rsvpsCountMin": { + "type": "integer", + "description": "Minimum total rsvps count" + }, + "rsvpsGoingCountMin": { + "type": "integer", + "description": "Minimum rsvps count where status = going" + }, + "rsvpsInterestedCountMin": { + "type": "integer", + "description": "Minimum rsvps count where status = interested" + }, + "rsvpsNotgoingCountMin": { + "type": "integer", + "description": "Minimum rsvps count where status = notgoing" + }, + "hydrateRsvps": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "description": "Number of rsvps records to embed" + }, + "sort": { + "type": "string", + "knownValues": [ + "createdAt", + "endsAt", + "mode", + "rsvpsCount", + "rsvpsGoingCount", + "rsvpsInterestedCount", + "rsvpsNotgoingCount", + "startsAt", + "status", + "subjectUri" + ], + "description": "Field to sort by (default: time_us)" + }, + "order": { + "type": "string", + "knownValues": [ + "asc", + "desc" + ], + "description": "Sort direction" + }, + "subjectUri": { + "type": "string", + "description": "Filter by subject.uri" + }, + "hydrateEvent": { + "type": "boolean", + "description": "Embed the referenced event record" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "records" + ], + "properties": { + "records": { + "type": "array", + "items": { + "type": "union", + "refs": [ + "#feedRecordEvent", + "#feedRecordRsvp" + ] + } + }, + "cursor": { + "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "feedRecordEvent": { + "type": "object", + "required": [ + "uri", + "cid", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.event#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "rsvpsCount": { + "type": "integer", + "description": "Total rsvps count" + }, + "rsvpsGoingCount": { + "type": "integer", + "description": "rsvps count where status = going" + }, + "rsvpsInterestedCount": { + "type": "integer", + "description": "rsvps count where status = interested" + }, + "rsvpsNotgoingCount": { + "type": "integer", + "description": "rsvps count where status = notgoing" + }, + "rsvps": { + "type": "ref", + "ref": "#hydrateRsvps" + } + } + }, + "feedRecordRsvp": { + "type": "object", + "required": [ + "uri", + "cid", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.rsvp#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + }, + "event": { + "type": "ref", + "ref": "#refEventRecord" + } + } + }, + "hydrateRsvpsRecord": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.rsvp#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + }, + "hydrateRsvps": { + "type": "object", + "properties": { + "going": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "interested": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "notgoing": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + }, + "other": { + "type": "array", + "items": { + "type": "ref", + "ref": "#hydrateRsvpsRecord" + } + } + } + }, + "refEventRecord": { + "type": "object", + "required": [ + "uri", + "value", + "did", + "collection", + "rkey", + "time_us" + ], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "community.lexicon.calendar.event#main" + }, + "did": { + "type": "string", + "format": "did" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + }, + "time_us": { + "type": "integer" + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getProfile.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getProfile.json new file mode 100644 index 0000000..54e562c --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/getProfile.json @@ -0,0 +1,73 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.getProfile", + "defs": { + "main": { + "type": "query", + "parameters": { + "type": "params", + "required": [ + "actor" + ], + "properties": { + "actor": { + "type": "string", + "format": "at-identifier" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "profiles" + ], + "properties": { + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } + } + } + } + } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/notifyOfUpdate.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/notifyOfUpdate.json new file mode 100644 index 0000000..5d7e545 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/notifyOfUpdate.json @@ -0,0 +1,56 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.notifyOfUpdate", + "defs": { + "main": { + "type": "procedure", + "description": "Fetch changed records from their authoritative PDS for immediate indexing", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "format": "at-uri" + }, + "uris": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + }, + "maxLength": 25 + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "indexed", + "deleted" + ], + "properties": { + "indexed": { + "type": "integer", + "minimum": 0 + }, + "deleted": { + "type": "integer", + "minimum": 0 + }, + "errors": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } +} diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json index 1a9ab26..99016e4 100644 --- a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/getRecord.json @@ -16,6 +16,10 @@ "format": "at-uri", "description": "AT URI of the record" }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + }, "hydrateEvent": { "type": "boolean", "description": "Embed the referenced event record" @@ -64,6 +68,13 @@ "event": { "type": "ref", "ref": "#refEventRecord" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } } } } @@ -107,6 +118,40 @@ "type": "integer" } } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } } } } diff --git a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json index d0046b0..6b49e21 100644 --- a/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json +++ b/apps/atmo-rsvp/lexicons/generated/rsvp/atmo/rsvp/listRecords.json @@ -22,6 +22,10 @@ "format": "at-identifier", "description": "Filter by an indexed DID or cached handle" }, + "profiles": { + "type": "boolean", + "description": "Include indexed profile and identity information" + }, "createdAtMin": { "type": "string", "description": "Minimum value for createdAt" @@ -78,6 +82,13 @@ }, "cursor": { "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": "ref", + "ref": "#profileEntry" + } } } } @@ -165,6 +176,40 @@ "type": "integer" } } + }, + "profileEntry": { + "type": "object", + "required": [ + "did" + ], + "properties": { + "did": { + "type": "string", + "format": "did" + }, + "handle": { + "type": "string" + }, + "uri": { + "type": "string", + "format": "at-uri" + }, + "cid": { + "type": "string", + "format": "cid" + }, + "value": { + "type": "ref", + "ref": "app.bsky.actor.profile#main" + }, + "collection": { + "type": "string", + "format": "nsid" + }, + "rkey": { + "type": "string" + } + } } } } diff --git a/apps/atmo-rsvp/lexicons/pulled/README.md b/apps/atmo-rsvp/lexicons/pulled/README.md index b77ef3e..cf93039 100644 --- a/apps/atmo-rsvp/lexicons/pulled/README.md +++ b/apps/atmo-rsvp/lexicons/pulled/README.md @@ -2,4 +2,4 @@ this directory contains lexicon documents pulled from the following sources: -- atproto (nsids: com.atproto.repo.strongRef, community.lexicon.calendar.event, community.lexicon.calendar.rsvp, community.lexicon.location.address, community.lexicon.location.fsq, community.lexicon.location.geo, community.lexicon.location.hthree) +- atproto (nsids: app.bsky.actor.profile, app.bsky.graph.follow, com.atproto.label.defs, com.atproto.repo.strongRef, community.lexicon.calendar.event, community.lexicon.calendar.rsvp, community.lexicon.location.address, community.lexicon.location.fsq, community.lexicon.location.geo, community.lexicon.location.hthree) diff --git a/apps/atmo-rsvp/lexicons/pulled/app/bsky/actor/profile.json b/apps/atmo-rsvp/lexicons/pulled/app/bsky/actor/profile.json new file mode 100644 index 0000000..35e72fe --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/app/bsky/actor/profile.json @@ -0,0 +1,75 @@ +{ + "id": "app.bsky.actor.profile", + "defs": { + "main": { + "key": "literal:self", + "type": "record", + "record": { + "type": "object", + "properties": { + "avatar": { + "type": "blob", + "accept": [ + "image/jpeg", + "image/png" + ], + "maxSize": 1000000, + "description": "Small image to be displayed next to posts from account. AKA, 'profile picture'" + }, + "banner": { + "type": "blob", + "accept": [ + "image/jpeg", + "image/png" + ], + "maxSize": 1000000, + "description": "Larger horizontal image to display behind profile view." + }, + "labels": { + "refs": [ + "com.atproto.label.defs#selfLabels" + ], + "type": "union", + "description": "Self-label values, specific to the Bluesky application, on the overall account." + }, + "website": { + "type": "string", + "format": "uri" + }, + "pronouns": { + "type": "string", + "maxLength": 200, + "description": "Free-form pronouns text.", + "maxGraphemes": 20 + }, + "createdAt": { + "type": "string", + "format": "datetime" + }, + "pinnedPost": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + }, + "description": { + "type": "string", + "maxLength": 2560, + "description": "Free-form profile description text.", + "maxGraphemes": 256 + }, + "displayName": { + "type": "string", + "maxLength": 640, + "maxGraphemes": 64 + }, + "joinedViaStarterPack": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + } + } + }, + "description": "A declaration of a Bluesky account profile." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/app/bsky/graph/follow.json b/apps/atmo-rsvp/lexicons/pulled/app/bsky/graph/follow.json new file mode 100644 index 0000000..26ada28 --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/app/bsky/graph/follow.json @@ -0,0 +1,33 @@ +{ + "id": "app.bsky.graph.follow", + "defs": { + "main": { + "key": "tid", + "type": "record", + "record": { + "type": "object", + "required": [ + "createdAt", + "subject" + ], + "properties": { + "via": { + "ref": "com.atproto.repo.strongRef", + "type": "ref" + }, + "subject": { + "type": "string", + "format": "did" + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + }, + "description": "Record declaring a social 'follow' relationship of another account. Duplicate follows will be ignored by the AppView." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/lexicons/pulled/com/atproto/label/defs.json b/apps/atmo-rsvp/lexicons/pulled/com/atproto/label/defs.json new file mode 100644 index 0000000..b7017ec --- /dev/null +++ b/apps/atmo-rsvp/lexicons/pulled/com/atproto/label/defs.json @@ -0,0 +1,190 @@ +{ + "id": "com.atproto.label.defs", + "defs": { + "label": { + "type": "object", + "required": [ + "cts", + "src", + "uri", + "val" + ], + "properties": { + "cid": { + "type": "string", + "format": "cid", + "description": "Optionally, CID specifying the specific version of 'uri' resource this label applies to." + }, + "cts": { + "type": "string", + "format": "datetime", + "description": "Timestamp when this label was created." + }, + "exp": { + "type": "string", + "format": "datetime", + "description": "Timestamp at which this label expires (no longer applies)." + }, + "neg": { + "type": "boolean", + "description": "If true, this is a negation label, overwriting a previous label." + }, + "sig": { + "type": "bytes", + "description": "Signature of dag-cbor encoded label." + }, + "src": { + "type": "string", + "format": "did", + "description": "DID of the actor who created this label." + }, + "uri": { + "type": "string", + "format": "uri", + "description": "AT URI of the record, repository (account), or other resource that this label applies to." + }, + "val": { + "type": "string", + "maxLength": 128, + "description": "The short string name of the value or type of this label." + }, + "ver": { + "type": "integer", + "description": "The AT Protocol version of the label object." + } + }, + "description": "Metadata tag on an atproto resource (eg, repo or record)." + }, + "selfLabel": { + "type": "object", + "required": [ + "val" + ], + "properties": { + "val": { + "type": "string", + "maxLength": 128, + "description": "The short string name of the value or type of this label." + } + }, + "description": "Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel." + }, + "labelValue": { + "type": "string", + "knownValues": [ + "!hide", + "!no-unauthenticated", + "!warn", + "bot", + "graphic-media", + "nudity", + "porn", + "sexual" + ] + }, + "selfLabels": { + "type": "object", + "required": [ + "values" + ], + "properties": { + "values": { + "type": "array", + "items": { + "ref": "#selfLabel", + "type": "ref" + }, + "maxLength": 10 + } + }, + "description": "Metadata tags on an atproto record, published by the author within the record." + }, + "labelValueDefinition": { + "type": "object", + "required": [ + "blurs", + "identifier", + "locales", + "severity" + ], + "properties": { + "blurs": { + "type": "string", + "description": "What should this label hide in the UI, if applied? 'content' hides all of the target; 'media' hides the images/video/audio; 'none' hides nothing.", + "knownValues": [ + "content", + "media", + "none" + ] + }, + "locales": { + "type": "array", + "items": { + "ref": "#labelValueDefinitionStrings", + "type": "ref" + } + }, + "severity": { + "type": "string", + "description": "How should a client visually convey this label? 'inform' means neutral and informational; 'alert' means negative and warning; 'none' means show nothing.", + "knownValues": [ + "alert", + "inform", + "none" + ] + }, + "adultOnly": { + "type": "boolean", + "description": "Does the user need to have adult content enabled in order to configure this label?" + }, + "identifier": { + "type": "string", + "maxLength": 100, + "description": "The value of the label being defined. Must only include lowercase ascii and the '-' character ([a-z-]+).", + "maxGraphemes": 100 + }, + "defaultSetting": { + "type": "string", + "default": "warn", + "description": "The default setting for this label.", + "knownValues": [ + "hide", + "ignore", + "warn" + ] + } + }, + "description": "Declares a label value and its expected interpretations and behaviors." + }, + "labelValueDefinitionStrings": { + "type": "object", + "required": [ + "description", + "lang", + "name" + ], + "properties": { + "lang": { + "type": "string", + "format": "language", + "description": "The code of the language these strings are written in." + }, + "name": { + "type": "string", + "maxLength": 640, + "description": "A short human-readable name for the label.", + "maxGraphemes": 64 + }, + "description": { + "type": "string", + "maxLength": 100000, + "description": "A longer description of what the label means and why it might be applied.", + "maxGraphemes": 10000 + } + }, + "description": "Strings which describe the label in the UI, localized into a specific language." + } + }, + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 +} \ No newline at end of file diff --git a/apps/atmo-rsvp/src/contrail.config.ts b/apps/atmo-rsvp/src/contrail.config.ts index 1a6b7b7..78faab4 100644 --- a/apps/atmo-rsvp/src/contrail.config.ts +++ b/apps/atmo-rsvp/src/contrail.config.ts @@ -1,22 +1,19 @@ import type { ContrailConfig } from "@atmo-dev/contrail"; -import { lexicons } from "../lexicons/generated"; export const config: ContrailConfig = { namespace: "rsvp.atmo", - profiles: [], - constellation: false, + profiles: ["app.bsky.actor.profile"], jetstreams: ["wss://jetstream1.us-east.bsky.network"], orderedSource: { source: "jetstream", epoch: "api-atmo-rsvp-primary-2026-08", }, - validation: { - lexicons: lexicons as unknown as NonNullable< - ContrailConfig["validation"] - >["lexicons"], - strict: true, - verifyCid: true, + notify: true, + serviceAuth: { + audience: "did:web:api.atmo.rsvp", + methods: ["getFeed", "notifyOfUpdate"], }, + maintenance: { optimize: true }, collections: { event: { collection: "community.lexicon.calendar.event", @@ -54,5 +51,24 @@ export const config: ContrailConfig = { }, }, }, + profile: { + collection: "app.bsky.actor.profile", + discover: false, + methods: [], + }, + follow: { + collection: "app.bsky.graph.follow", + discover: false, + subjectField: "subject", + methods: [], + }, + }, + feeds: { + network: { + targets: [ + { collection: "event", maxItems: 100 }, + { collection: "rsvp", maxItems: 250 }, + ], + }, }, }; diff --git a/apps/atmo-rsvp/tests/contract.test.ts b/apps/atmo-rsvp/tests/contract.test.ts index c38c80c..f8c305d 100644 --- a/apps/atmo-rsvp/tests/contract.test.ts +++ b/apps/atmo-rsvp/tests/contract.test.ts @@ -4,6 +4,7 @@ import { describePublicService, digestPublicContract, } from "@atmo-dev/contrail"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { createWorker } from "@atmo-dev/contrail/worker"; import { lexicons } from "../lexicons/generated"; import { config } from "../src/contrail.config"; @@ -12,11 +13,31 @@ const EXPECTED_METHODS = [ "rsvp.atmo.event.getRecord", "rsvp.atmo.event.listRecords", "rsvp.atmo.getCursor", + "rsvp.atmo.getProfile", "rsvp.atmo.rsvp.getRecord", "rsvp.atmo.rsvp.listRecords", ]; +const EXPECTED_PROTECTED_METHODS = [ + { id: "rsvp.atmo.getFeed", type: "query" }, + { id: "rsvp.atmo.notifyOfUpdate", type: "procedure" }, +]; + describe("api.atmo.rsvp public contract", () => { + it("keeps profile and follow projections internal and validation disabled", () => { + expect(config.validation).toBeUndefined(); + expect(config.collections.profile?.methods).toEqual([]); + expect(config.collections.follow).toMatchObject({ + discover: false, + subjectField: "subject", + methods: [], + }); + expect(config.feeds?.network.targets).toEqual([ + { collection: "event", maxItems: 100 }, + { collection: "rsvp", maxItems: 250 }, + ]); + }); + it("advertises the exact generated calendar API", async () => { const service = await describePublicService( config, @@ -27,9 +48,18 @@ describe("api.atmo.rsvp public contract", () => { expect(service.manifest.namespace).toBe("rsvp.atmo"); expect(service.manifest.methods).toEqual(EXPECTED_METHODS); expect(service.manifest.methods).not.toContain("rsvp.atmo.getOverview"); - expect(service.manifest.methods).not.toContain("rsvp.atmo.notifyOfUpdate"); - expect(service.lexicons.map((document) => document.id)).not.toContain( - "com.atproto.label.defs", + expect(service.manifest.serviceAuth).toEqual({ + type: "atproto-service-auth", + audience: "did:web:api.atmo.rsvp", + methods: EXPECTED_PROTECTED_METHODS, + }); + expect(service.lexicons.map((document) => document.id)).toEqual( + expect.arrayContaining([ + "app.bsky.actor.profile", + "app.bsky.graph.follow", + "rsvp.atmo.getFeed", + "rsvp.atmo.notifyOfUpdate", + ]), ); expect( await digestPublicContract(contractFromManifest(service.manifest)), @@ -47,4 +77,42 @@ describe("api.atmo.rsvp public contract", () => { }), ).not.toThrow(); }); + + it("publishes its service DID and permits browser auth headers", async () => { + const worker = createWorker(config, { + lexicons, + publicService: { endpoint: "https://api.atmo.rsvp" }, + }); + const env = { DB: createSqliteDatabase(":memory:") }; + const did = await worker.fetch( + new Request("https://api.atmo.rsvp/.well-known/did.json"), + env, + ); + expect(did.status).toBe(200); + expect(await did.json()).toMatchObject({ + id: "did:web:api.atmo.rsvp", + service: [ + { + id: "did:web:api.atmo.rsvp#contrail", + serviceEndpoint: "https://api.atmo.rsvp", + }, + ], + }); + + const preflight = await worker.fetch( + new Request("https://api.atmo.rsvp/xrpc/rsvp.atmo.getFeed", { + method: "OPTIONS", + headers: { + origin: "https://client.example", + "access-control-request-method": "GET", + "access-control-request-headers": "authorization", + }, + }), + env, + ); + expect(preflight.status).toBe(204); + expect(preflight.headers.get("access-control-allow-headers")).toContain( + "Authorization", + ); + }); }); diff --git a/apps/atmo-rsvp/wrangler.jsonc b/apps/atmo-rsvp/wrangler.jsonc index 0f3a477..9412581 100644 --- a/apps/atmo-rsvp/wrangler.jsonc +++ b/apps/atmo-rsvp/wrangler.jsonc @@ -15,8 +15,8 @@ "d1_databases": [ { "binding": "DB", - "database_name": "atmo-rsvp-contrail", - "database_id": "8a483108-4614-41c4-b9db-86b7a2ea27ca" + "database_name": "atmo-rsvp-contrail-g20260807", + "database_id": "e89f629a-17a8-4fd3-b3c0-a4c6c5ff3bf1" } ], "triggers": { -- 2.51.2 From 92b0b019ba7fc4b70e689f2481c9e46f12aee492 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:46:54 +0200 Subject: [PATCH 12/15] Simplify public service clients --- packages/contrail/package.json | 6 +- packages/contrail/src/cli/commands/connect.ts | 43 ++++ packages/contrail/src/public-client.ts | 207 ++++++++++++++++++ packages/contrail/tests/built-client.mjs | 11 + packages/contrail/tests/connect.test.ts | 28 +++ packages/contrail/tests/public-client.test.ts | 127 +++++++++++ .../contrail/tests/public-service-e2e.test.ts | 21 +- packages/contrail/tsup.config.ts | 1 + 8 files changed, 431 insertions(+), 13 deletions(-) create mode 100644 packages/contrail/src/public-client.ts create mode 100644 packages/contrail/tests/built-client.mjs create mode 100644 packages/contrail/tests/public-client.test.ts diff --git a/packages/contrail/package.json b/packages/contrail/package.json index f754305..23e19c9 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -19,6 +19,10 @@ "types": "./dist/server.d.ts", "import": "./dist/server.js" }, + "./client": { + "types": "./dist/public-client.d.ts", + "import": "./dist/public-client.js" + }, "./sqlite": { "types": "./dist/adapters/sqlite.d.ts", "import": "./dist/adapters/sqlite.js" @@ -65,7 +69,7 @@ "clean": "rm -rf dist", "typecheck": "tsc --noEmit", "test": "vitest run", - "test:built": "node tests/built-sqlite.mjs && node tests/built-lexicons.mjs", + "test:built": "node tests/built-sqlite.mjs && node tests/built-lexicons.mjs && node tests/built-client.mjs", "test:watch": "vitest" }, "dependencies": { diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index e973751..7a6e0da 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -32,6 +32,15 @@ import { generateLexiconTypesWithAtcute } from "../atcute.js"; const MAX_DISCOVERY_BYTES = 10 * 1024 * 1024; const REQUEST_TIMEOUT_MS = 15_000; +const LEXICON_CONFIG_NAMES = [ + "lex.config.js", + "lex.config.mjs", + "lex.config.cjs", + "lex.config.ts", + "lex.config.mts", + "lex.config.cts", +]; + interface ConnectOptions { root: string; out: string; @@ -149,6 +158,33 @@ async function exists(path: string): Promise { } } +/** Create a dependency-free Atcute config for ordinary consumer projects. + * Existing JavaScript or TypeScript configs remain entirely consumer-owned. */ +export async function ensureConsumerLexiconConfig(options: { + root: string; + out: string; +}): Promise<{ path: string; created: boolean }> { + const root = resolve(options.root); + for (const name of LEXICON_CONFIG_NAMES) { + const path = join(root, name); + if (await exists(path)) return { path, created: false }; + } + + const lexiconRoot = resolveInsideRoot(root, options.out); + const patternRoot = relative(root, lexiconRoot).replaceAll("\\", "/"); + const path = join(root, "lex.config.js"); + const source = `// Generated by \`contrail connect\`. Customize as needed.\nexport default {\n generate: {\n files: [${JSON.stringify(`${patternRoot}/**/*.json`)}],\n outdir: "src/lexicons/",\n },\n};\n`; + try { + await writeFile(path, source, { flag: "wx" }); + return { path, created: true }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + return { path, created: false }; + } + throw error; + } +} + export async function connectPublicService(options: { endpoint: string; root: string; @@ -335,6 +371,13 @@ export function registerConnect(cli: CAC): void { `connected ${result.lock.endpoint}: ${result.written} Lexicons, contract ${result.lock.contractDigest}`, ); if (options.generate !== false) { + const config = await ensureConsumerLexiconConfig({ + root: options.root, + out: options.out, + }); + if (config.created) { + console.log(`created ${relative(resolve(options.root), config.path)}`); + } generateLexiconTypesWithAtcute(resolve(options.root)); } }); diff --git a/packages/contrail/src/public-client.ts b/packages/contrail/src/public-client.ts new file mode 100644 index 0000000..8979cc3 --- /dev/null +++ b/packages/contrail/src/public-client.ts @@ -0,0 +1,207 @@ +import type {} from "@atcute/atproto"; +import { + Client, + simpleFetchHandler, + type FetchHandler, +} from "@atcute/client"; +import type { Did, Nsid } from "@atcute/lexicons/syntax"; +import { + isPublicServiceManifest, + normalizePublicServiceEndpoint, + type PublicServiceAuthContract, +} from "./public-service.js"; + +const DISCOVERY_TIMEOUT_MS = 15_000; +const TOKEN_EXPIRY_SKEW_MS = 5_000; + +interface CachedToken { + value: string; + expiresAt: number; +} + +export interface PublicServiceClientOptions { + /** Canonical public Contrail HTTPS origin. */ + endpoint: string; + /** Existing authenticated PDS client used to mint service tokens. Omit when + * the consumer only needs anonymous methods. */ + authenticatedPds?: Client; + /** Optional contract pin from `contrail.lock.json`. */ + contractDigest?: string; + /** Browser, test, or instrumented fetch implementation. */ + fetch?: typeof globalThis.fetch; +} + +function xrpcMethod(pathname: string): Nsid | null { + const path = pathname.startsWith("http") + ? new URL(pathname).pathname + : new URL(pathname, "https://contrail.invalid").pathname; + const prefix = "/xrpc/"; + if (!path.startsWith(prefix)) return null; + try { + const method = decodeURIComponent(path.slice(prefix.length)); + return method.includes("/") ? null : (method as Nsid); + } catch { + return null; + } +} + +function tokenExpiration(token: string): number { + try { + const part = token.split(".")[1]; + if (!part) return Date.now() + 30_000; + const base64 = part.replace(/-/g, "+").replace(/_/g, "/"); + const padded = base64.padEnd(Math.ceil(base64.length / 4) * 4, "="); + const bytes = Uint8Array.from(atob(padded), (character) => + character.charCodeAt(0), + ); + const payload = JSON.parse(new TextDecoder().decode(bytes)) as { + exp?: unknown; + }; + return typeof payload.exp === "number" && Number.isSafeInteger(payload.exp) + ? payload.exp * 1_000 + : Date.now() + 30_000; + } catch { + return Date.now() + 30_000; + } +} + +function withBearer(init: RequestInit, token: string): RequestInit { + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${token}`); + return { ...init, headers }; +} + +/** Fetch handler that keeps anonymous reads cheap while automatically minting, + * caching, and attaching method-bound AT Protocol service tokens after a + * protected route challenges the first request. */ +export function publicServiceFetchHandler( + options: PublicServiceClientOptions, +): FetchHandler { + const endpoint = normalizePublicServiceEndpoint(options.endpoint); + const fetcher = options.fetch ?? fetch; + const base = simpleFetchHandler({ service: endpoint, fetch: fetcher }); + const tokens = new Map(); + const pendingTokens = new Map>(); + let serviceAuthPromise: Promise | null = null; + + const discoverServiceAuth = () => { + if (serviceAuthPromise) return serviceAuthPromise; + serviceAuthPromise = (async () => { + const response = await fetcher(`${endpoint}/.well-known/contrail`, { + signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`Contrail discovery failed: ${response.status}`); + } + if (response.url && new URL(response.url).origin !== endpoint) { + throw new Error("Contrail discovery redirected to a different origin"); + } + const value: unknown = await response.json(); + if (!isPublicServiceManifest(value)) { + throw new Error("response is not a supported Contrail service manifest"); + } + if (normalizePublicServiceEndpoint(value.endpoint) !== endpoint) { + throw new Error("Contrail manifest endpoint mismatch"); + } + if ( + options.contractDigest && + value.contract.digest !== options.contractDigest + ) { + throw new Error( + `Contrail contract digest mismatch: expected ${options.contractDigest}, received ${value.contract.digest}`, + ); + } + return value.serviceAuth ?? null; + })(); + return serviceAuthPromise; + }; + + const protectedMethod = async (method: string) => { + const auth = await discoverServiceAuth(); + return auth?.methods.some((candidate) => candidate.id === method) + ? auth + : null; + }; + + const tokenFor = async ( + method: Nsid, + auth: PublicServiceAuthContract, + force = false, + ): Promise => { + if (!options.authenticatedPds) { + throw new Error( + `Contrail method ${method} requires an authenticated PDS client`, + ); + } + const cached = tokens.get(method); + if (!force && cached && cached.expiresAt > Date.now() + TOKEN_EXPIRY_SKEW_MS) { + return cached.value; + } + if (!force) { + const pending = pendingTokens.get(method); + if (pending) return pending; + } + + const pending = (async () => { + const response = await options.authenticatedPds!.get( + "com.atproto.server.getServiceAuth", + { + params: { + aud: auth.audience as Did, + lxm: method, + }, + }, + ); + if (!response.ok) { + throw new Error( + `Could not mint service token for ${method}: ${response.status}`, + ); + } + const token = response.data.token; + tokens.set(method, { value: token, expiresAt: tokenExpiration(token) }); + return token; + })(); + pendingTokens.set(method, pending); + try { + return await pending; + } finally { + if (pendingTokens.get(method) === pending) pendingTokens.delete(method); + } + }; + + return async (pathname, init) => { + const method = xrpcMethod(pathname); + if (!method || !options.authenticatedPds) return base(pathname, init); + + // Once discovery has been loaded, avoid the initial challenge on subsequent + // protected calls. Anonymous calls never wait for discovery. + if (serviceAuthPromise) { + const auth = await protectedMethod(method); + if (auth) { + const token = await tokenFor(method, auth); + const response = await base(pathname, withBearer(init, token)); + if (response.status !== 401) return response; + await response.body?.cancel(); + tokens.delete(method); + const refreshed = await tokenFor(method, auth, true); + return base(pathname, withBearer(init, refreshed)); + } + } + + const response = await base(pathname, init); + if (response.status !== 401) return response; + const auth = await protectedMethod(method); + if (!auth) return response; + await response.body?.cancel(); + const token = await tokenFor(method, auth); + return base(pathname, withBearer(init, token)); + }; +} + +/** Create a typed Atcute client for anonymous and service-auth Contrail methods. + * Generated Lexicon imports still supply the method-specific TypeScript API. */ +export function createPublicServiceClient( + options: PublicServiceClientOptions, +): Client { + return new Client({ handler: publicServiceFetchHandler(options) }); +} diff --git a/packages/contrail/tests/built-client.mjs b/packages/contrail/tests/built-client.mjs new file mode 100644 index 0000000..1bc59d4 --- /dev/null +++ b/packages/contrail/tests/built-client.mjs @@ -0,0 +1,11 @@ +import assert from "node:assert/strict"; +import { createPublicServiceClient } from "../dist/public-client.js"; + +const client = createPublicServiceClient({ + endpoint: "https://api.example.com", + fetch: async () => Response.json({ records: [] }), +}); +const response = await client.get("com.example.listRecords"); +assert.equal(response.ok, true); +assert.deepEqual(response.data, { records: [] }); +console.log("built public client passed"); diff --git a/packages/contrail/tests/connect.test.ts b/packages/contrail/tests/connect.test.ts index 3e8e765..ee73699 100644 --- a/packages/contrail/tests/connect.test.ts +++ b/packages/contrail/tests/connect.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { connectPublicService, + ensureConsumerLexiconConfig, type ProviderLock, } from "../src/cli/commands/connect"; import { @@ -98,6 +99,33 @@ async function temporaryRoot() { } describe("contrail connect", () => { + it("creates a default Atcute config without replacing consumer config", async () => { + const root = await temporaryRoot(); + const generated = await ensureConsumerLexiconConfig({ + root, + out: "lexicons/providers", + }); + expect(generated.created).toBe(true); + expect(await readFile(generated.path, "utf8")).toContain( + 'files: ["lexicons/providers/**/*.json"]', + ); + expect(await readFile(generated.path, "utf8")).toContain( + 'outdir: "src/lexicons/"', + ); + + await writeFile(join(root, "lex.config.ts"), "export default { mine: true }"); + await rm(generated.path); + const existing = await ensureConsumerLexiconConfig({ + root, + out: "lexicons/other", + }); + expect(existing.created).toBe(false); + expect(existing.path).toBe(join(root, "lex.config.ts")); + expect(await readFile(existing.path, "utf8")).toBe( + "export default { mine: true }", + ); + }); + it("verifies and atomically locks a discovered service", async () => { const root = await temporaryRoot(); const { fetcher, manifest, values } = await serviceFixture(); diff --git a/packages/contrail/tests/public-client.test.ts b/packages/contrail/tests/public-client.test.ts new file mode 100644 index 0000000..97187c5 --- /dev/null +++ b/packages/contrail/tests/public-client.test.ts @@ -0,0 +1,127 @@ +import type {} from "@atcute/atproto"; +import { Client } from "@atcute/client"; +import { describe, expect, it, vi } from "vitest"; +import { + createPublicServiceClient, + publicServiceFetchHandler, +} from "../src/public-client"; +import type { PublicServiceManifest } from "../src/public-service"; + +const endpoint = "https://api.example.com"; +const method = "com.example.getFeed"; +const digest = `sha256:${"a".repeat(64)}`; + +function token() { + const encode = (value: unknown) => + btoa(JSON.stringify(value)) + .replaceAll("+", "-") + .replaceAll("/", "_") + .replaceAll("=", ""); + return `${encode({ alg: "none" })}.${encode({ exp: Math.floor(Date.now() / 1000) + 60 })}.signature`; +} + +function manifest(): PublicServiceManifest { + return { + format: "contrail.service", + version: 1, + endpoint, + namespace: "com.example", + contract: { digest }, + lexicons: { url: `${endpoint}/lexicons/${digest}`, digest }, + status: { url: `${endpoint}/status` }, + collections: [], + methods: ["com.example.getCursor"], + serviceAuth: { + type: "atproto-service-auth", + audience: "did:web:api.example.com", + methods: [{ id: method, type: "query" }], + }, + }; +} + +function authenticatedPds(jwt: string) { + const handler = vi.fn(async (pathname: string) => { + const url = new URL(pathname, "https://pds.example.com"); + expect(url.pathname).toBe("/xrpc/com.atproto.server.getServiceAuth"); + expect(url.searchParams.get("aud")).toBe("did:web:api.example.com"); + expect(url.searchParams.get("lxm")).toBe(method); + return Response.json({ token: jwt }); + }); + return { client: new Client({ handler }), handler }; +} + +describe("public service client", () => { + it("keeps anonymous requests anonymous", async () => { + const fetcher = vi.fn(async () => Response.json({ records: [] })); + const handler = publicServiceFetchHandler({ endpoint, fetch: fetcher }); + + const response = await handler("/xrpc/com.example.listRecords", { + method: "get", + }); + + expect(response.status).toBe(200); + expect(fetcher).toHaveBeenCalledTimes(1); + }); + + it("discovers, mints, caches, and attaches method-bound tokens", async () => { + const jwt = token(); + const pds = authenticatedPds(jwt); + const requests: Array<{ url: string; authorization: string | null }> = []; + const fetcher = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const authorization = new Headers(init?.headers).get("authorization"); + requests.push({ url, authorization }); + if (url.endsWith("/.well-known/contrail")) { + return Response.json(manifest()); + } + return authorization === `Bearer ${jwt}` + ? Response.json({ records: [] }) + : Response.json( + { error: "AuthenticationRequired" }, + { status: 401, headers: { "www-authenticate": "Bearer" } }, + ); + }); + const client = createPublicServiceClient({ + endpoint, + authenticatedPds: pds.client, + contractDigest: digest, + fetch: fetcher, + }); + + const first = await (client as any).get(method, { + params: { actor: "did:plc:test", feed: "network" }, + }); + const second = await (client as any).get(method, { + params: { actor: "did:plc:test", feed: "network" }, + }); + + expect(first.ok).toBe(true); + expect(second.ok).toBe(true); + expect(pds.handler).toHaveBeenCalledTimes(1); + expect( + requests.filter((request) => + request.url.includes(`/xrpc/${method}`), + ).map((request) => request.authorization), + ).toEqual([null, `Bearer ${jwt}`, `Bearer ${jwt}`]); + }); + + it("refuses runtime discovery that differs from an optional lock pin", async () => { + const pds = authenticatedPds(token()); + const fetcher = vi.fn(async (input: RequestInfo | URL) => + String(input).endsWith("/.well-known/contrail") + ? Response.json(manifest()) + : new Response(null, { status: 401 }), + ); + const handler = publicServiceFetchHandler({ + endpoint, + authenticatedPds: pds.client, + contractDigest: `sha256:${"b".repeat(64)}`, + fetch: fetcher, + }); + + await expect( + handler(`/xrpc/${method}`, { method: "get" }), + ).rejects.toThrow("contract digest mismatch"); + expect(pds.handler).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/contrail/tests/public-service-e2e.test.ts b/packages/contrail/tests/public-service-e2e.test.ts index 84200f2..881be2d 100644 --- a/packages/contrail/tests/public-service-e2e.test.ts +++ b/packages/contrail/tests/public-service-e2e.test.ts @@ -2,7 +2,10 @@ import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { connectPublicService } from "../src/cli/commands/connect"; +import { + connectPublicService, + ensureConsumerLexiconConfig, +} from "../src/cli/commands/connect"; import { generateLexiconTypesWithAtcute } from "../src/cli/atcute"; import { createSqliteDatabase } from "../src/adapters/sqlite"; import { createApp } from "../src/core/router"; @@ -101,17 +104,6 @@ describe("public service consumer integration", () => { const fetcher: typeof fetch = (input, init) => app.fetch(new Request(input, init)); - writeFileSync( - join(consumerRoot, "lex.config.js"), - `import { defineLexiconConfig } from "@atcute/lex-cli"; -export default defineLexiconConfig({ - generate: { - files: ["lexicons/pulled/**/*.json"], - outdir: "src/lexicons/", - }, -}); -`, - ); await connectPublicService({ endpoint: "https://api.example.com", root: consumerRoot, @@ -119,6 +111,11 @@ export default defineLexiconConfig({ lock: "contrail.lock.json", fetcher, }); + const generatedConfig = await ensureConsumerLexiconConfig({ + root: consumerRoot, + out: "lexicons/pulled", + }); + expect(generatedConfig.created).toBe(true); generateLexiconTypesWithAtcute(consumerRoot); mkdirSync(join(consumerRoot, "src"), { recursive: true }); diff --git a/packages/contrail/tsup.config.ts b/packages/contrail/tsup.config.ts index 813e522..f6632ba 100644 --- a/packages/contrail/tsup.config.ts +++ b/packages/contrail/tsup.config.ts @@ -4,6 +4,7 @@ export default defineConfig({ entry: [ "src/index.ts", "src/server.ts", + "src/public-client.ts", "src/adapters/sqlite.ts", "src/adapters/postgres.ts", "src/workers/backfill.ts", -- 2.51.2 From a02a3f4ce3675b08254b72e4e8aa5ab532722d3b Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:14:43 +0200 Subject: [PATCH 13/15] Generate connected service clients --- .changeset/atproto-service-auth.md | 2 +- packages/contrail/src/cli/commands/connect.ts | 181 ++++++++++- packages/contrail/src/public-client.ts | 291 +++++++++++++++++- packages/contrail/tests/built-client.mjs | 7 + packages/contrail/tests/connect.test.ts | 109 ++++++- packages/contrail/tests/public-client.test.ts | 232 +++++++++++++- .../contrail/tests/public-service-e2e.test.ts | 21 +- 7 files changed, 799 insertions(+), 44 deletions(-) diff --git a/.changeset/atproto-service-auth.md b/.changeset/atproto-service-auth.md index d677108..51ea2de 100644 --- a/.changeset/atproto-service-auth.md +++ b/.changeset/atproto-service-auth.md @@ -2,4 +2,4 @@ "@atmo-dev/contrail": minor --- -Add discoverable AT Protocol service authentication for personalized feeds and authoritative update notifications. +Add discoverable AT Protocol service authentication, unified authenticated clients for PDS and provider methods, and automatic authoritative update notifications after tracked record writes. diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index 7a6e0da..1397534 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -45,7 +45,10 @@ interface ConnectOptions { root: string; out: string; lock: string; + client: string; + clientTypes: string; generate?: boolean; + skipClient?: boolean; update?: boolean; } @@ -57,6 +60,7 @@ export interface ProviderLock { contractDigest: string; lexiconDigest: string; methods: string[]; + collections: string[]; serviceAuth: PublicServiceAuthContract | null; lexiconRoot: string; } @@ -158,28 +162,154 @@ async function exists(path: string): Promise { } } -/** Create a dependency-free Atcute config for ordinary consumer projects. - * Existing JavaScript or TypeScript configs remain entirely consumer-owned. */ +function formatStringArray( + values: readonly string[], + indentation: number, +): string { + if (values.length === 0) return "[]"; + const items = values + .map((value) => `${" ".repeat(indentation + 2)}${JSON.stringify(value)},`) + .join("\n"); + return `[\n${items}\n${" ".repeat(indentation)}]`; +} + +const GENERATED_LEXICON_CONFIG_HEADER = + "// Generated by `contrail connect`. Re-run the command to update; do not edit.\n"; + +/** Create a dependency-free Atcute config containing the connected service + * metadata. Unmarked JavaScript or TypeScript configs remain consumer-owned. */ export async function ensureConsumerLexiconConfig(options: { root: string; out: string; -}): Promise<{ path: string; created: boolean }> { + types?: string; + lock: ProviderLock; +}): Promise<{ path: string; created: boolean; updated: boolean }> { const root = resolve(options.root); + let path = join(root, "lex.config.js"); for (const name of LEXICON_CONFIG_NAMES) { - const path = join(root, name); - if (await exists(path)) return { path, created: false }; + const candidate = join(root, name); + if (await exists(candidate)) { + path = candidate; + break; + } } const lexiconRoot = resolveInsideRoot(root, options.out); const patternRoot = relative(root, lexiconRoot).replaceAll("\\", "/"); - const path = join(root, "lex.config.js"); - const source = `// Generated by \`contrail connect\`. Customize as needed.\nexport default {\n generate: {\n files: [${JSON.stringify(`${patternRoot}/**/*.json`)}],\n outdir: "src/lexicons/",\n },\n};\n`; + const typesIndex = resolveInsideRoot( + root, + options.types ?? "src/contrail/types/index.ts", + ); + const typesRoot = relative(root, dirname(typesIndex)).replaceAll("\\", "/"); + const serviceDid = options.lock.serviceAuth?.audience ?? null; + const scope = serviceDid ? `rpc?lxm=*&aud=${serviceDid}` : null; + const source = `${GENERATED_LEXICON_CONFIG_HEADER}export default {\n contrail: {\n endpoint: ${JSON.stringify(options.lock.endpoint)},\n serviceDid: ${JSON.stringify(serviceDid)},\n scope: ${JSON.stringify(scope)},\n collections: ${formatStringArray(options.lock.collections, 4)},\n },\n generate: {\n files: [${JSON.stringify(`${patternRoot}/**/*.json`)}],\n outdir: ${JSON.stringify(`${typesRoot}/`)},\n },\n};\n`; + + if (await exists(path)) { + const current = await readFile(path, "utf8"); + if ( + !current.startsWith(GENERATED_LEXICON_CONFIG_HEADER) || + current === source + ) { + return { path, created: false, updated: false }; + } + const stagedDirectory = await mkdtemp(join(dirname(path), ".contrail-lex-")); + const staged = join(stagedDirectory, basename(path)); + try { + await writeFile(staged, source); + await rename(staged, path); + } finally { + await rm(stagedDirectory, { recursive: true, force: true }); + } + return { path, created: false, updated: true }; + } + try { await writeFile(path, source, { flag: "wx" }); - return { path, created: true }; + return { path, created: true, updated: false }; } catch (error) { if ((error as NodeJS.ErrnoException).code === "EEXIST") { - return { path, created: false }; + return { path, created: false, updated: false }; + } + throw error; + } +} + +const GENERATED_CLIENT_HEADER = + "// Generated by `contrail connect`. Re-run the command to update; do not edit.\n"; + +/** Create the small provider-specific module used by application and OAuth + * setup code. Unmarked TypeScript or JavaScript modules remain consumer-owned. */ +export async function ensureConsumerClientModule(options: { + root: string; + file?: string; + types?: string; + lock: ProviderLock; +}): Promise<{ path: string; created: boolean; updated: boolean }> { + const root = resolve(options.root); + const requested = options.file ?? "src/contrail/index.ts"; + const defaultNames = ["src/contrail/index.ts", "src/contrail/index.js"]; + let selected = requested; + if (defaultNames.includes(requested)) { + for (const name of defaultNames) { + if (await exists(join(root, name))) { + selected = name; + break; + } + } + } + if (!/\.(?:ts|js)$/.test(selected)) { + throw new Error("Contrail client module must end in .ts or .js"); + } + + const path = resolveInsideRoot(root, selected); + const isTypeScript = selected.endsWith(".ts"); + let generatedImport = ""; + if (isTypeScript) { + const types = resolveInsideRoot( + root, + options.types ?? "src/contrail/types/index.ts", + ); + let specifier = relative(dirname(path), types).replaceAll("\\", "/"); + specifier = specifier.replace(/\.(?:ts|js)$/, ".js"); + if (!specifier.startsWith(".")) specifier = `./${specifier}`; + generatedImport = `import type {} from ${JSON.stringify(specifier)};\n`; + } + const serviceDid = options.lock.serviceAuth?.audience; + const scope = serviceDid ? `rpc?lxm=*&aud=${serviceDid}` : null; + const protectedMethods = + options.lock.serviceAuth?.methods.map(({ id }) => id) ?? []; + const serviceMethods = [ + ...new Set([...options.lock.methods, ...protectedMethods]), + ].sort(); + const notifyMethod = protectedMethods.find( + (method) => method === `${options.lock.namespace}.notifyOfUpdate`, + ); + const source = `${GENERATED_CLIENT_HEADER}import { createPublicServiceClient } from "@atmo-dev/contrail/client";\n${generatedImport}\nexport const contrail = createPublicServiceClient({\n endpoint: ${JSON.stringify(options.lock.endpoint)},${serviceDid ? `\n serviceDid: ${JSON.stringify(serviceDid)},\n scope: ${JSON.stringify(scope)},` : ""}\n serviceMethods: ${formatStringArray(serviceMethods, 2)},\n collections: ${formatStringArray(options.lock.collections, 2)},${notifyMethod ? `\n notifyMethod: ${JSON.stringify(notifyMethod)},` : ""}\n});\n`; + await mkdir(dirname(path), { recursive: true }); + + if (await exists(path)) { + const current = await readFile(path, "utf8"); + if (!current.startsWith(GENERATED_CLIENT_HEADER) || current === source) { + return { path, created: false, updated: false }; + } + const stagedDirectory = await mkdtemp(join(dirname(path), ".contrail-client-")); + const staged = join(stagedDirectory, basename(path)); + try { + await writeFile(staged, source); + await rename(staged, path); + } finally { + await rm(stagedDirectory, { recursive: true, force: true }); + } + return { path, created: false, updated: true }; + } + + try { + await writeFile(path, source, { flag: "wx" }); + return { path, created: true, updated: false }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + return { path, created: false, updated: false }; } throw error; } @@ -295,6 +425,9 @@ export async function connectPublicService(options: { contractDigest: manifest.contract.digest, lexiconDigest: manifest.lexicons.digest, methods: [...manifest.methods].sort(), + collections: [ + ...new Set(manifest.collections.map(({ nsid }) => nsid)), + ].sort(), serviceAuth: manifest.serviceAuth ?? null, lexiconRoot: relative(projectRoot, providerRoot), }; @@ -346,19 +479,26 @@ export function registerConnect(cli: CAC): void { cli .command( "connect ", - "Discover a public Contrail, lock its API, pull Lexicons, and generate types", + "Discover a public Contrail, lock its API, and generate a typed client", ) .option("--root ", "Consumer project root", { default: process.cwd(), }) .option("--out ", "Provider-owned Lexicon storage relative to root", { - default: "lexicons/pulled", + default: "src/contrail/lexicons", }) .option("--lock ", "Provider lock file relative to root", { default: "contrail.lock.json", }) + .option("--client ", "Generated client module (.ts or .js)", { + default: "src/contrail/index.ts", + }) + .option("--client-types ", "Generated Lexicon index imported by a TypeScript client", { + default: "src/contrail/types/index.ts", + }) + .option("--skip-client", "Do not create a provider client module") .option("--update", "Replace an existing provider lock and owned Lexicons") - .option("--no-generate", "Pull and lock without running Atcute lex-cli") + .option("--no-generate", "Pull and lock without generating types or a client module") .action(async (endpoint: string, options: ConnectOptions) => { const result = await connectPublicService({ endpoint, @@ -374,11 +514,28 @@ export function registerConnect(cli: CAC): void { const config = await ensureConsumerLexiconConfig({ root: options.root, out: options.out, + types: options.clientTypes, + lock: result.lock, }); if (config.created) { console.log(`created ${relative(resolve(options.root), config.path)}`); + } else if (config.updated) { + console.log(`updated ${relative(resolve(options.root), config.path)}`); } generateLexiconTypesWithAtcute(resolve(options.root)); + if (!options.skipClient) { + const client = await ensureConsumerClientModule({ + root: options.root, + file: options.client, + types: options.clientTypes, + lock: result.lock, + }); + if (client.created) { + console.log(`created ${relative(resolve(options.root), client.path)}`); + } else if (client.updated) { + console.log(`updated ${relative(resolve(options.root), client.path)}`); + } + } } }); } diff --git a/packages/contrail/src/public-client.ts b/packages/contrail/src/public-client.ts index 8979cc3..af41414 100644 --- a/packages/contrail/src/public-client.ts +++ b/packages/contrail/src/public-client.ts @@ -22,15 +22,61 @@ interface CachedToken { export interface PublicServiceClientOptions { /** Canonical public Contrail HTTPS origin. */ endpoint: string; - /** Existing authenticated PDS client used to mint service tokens. Omit when - * the consumer only needs anonymous methods. */ - authenticatedPds?: Client; + /** Existing authenticated AT Protocol client used to mint service tokens. + * Omit when the consumer only needs anonymous methods. */ + authenticatedClient?: Client; /** Optional contract pin from `contrail.lock.json`. */ contractDigest?: string; + /** Optional receiving-service DID from `lex.config.js`. Supplying it also + * makes the required OAuth permission available as `client.scope`. */ + serviceDid?: Did; + /** Optional precomputed OAuth permission. Must match `serviceDid`. */ + scope?: `rpc?lxm=*&aud=${string}`; + /** Exact XRPC methods served by this provider. Supplying the verified list + * lets authenticated clients route all other methods to the user's PDS. */ + serviceMethods?: readonly Nsid[]; + /** Record collections whose successful PDS writes should notify Contrail. */ + collections?: readonly Nsid[]; + /** Protected notification procedure advertised by the provider. */ + notifyMethod?: Nsid; /** Browser, test, or instrumented fetch implementation. */ fetch?: typeof globalThis.fetch; } +export interface PublicServiceNotificationErrorContext { + method: Nsid; + uris: readonly string[]; +} + +export interface PublicServiceAuthenticatedOptions { + onNotificationError?: ( + error: unknown, + context: PublicServiceNotificationErrorContext, + ) => void; +} + +export type PublicServiceClient = Client & { + /** Canonical public Contrail origin. */ + readonly endpoint: string; + /** OAuth permission required by protected methods, or null when unconfigured. */ + readonly scope: `rpc?lxm=*&aud=${string}` | null; + /** Record collections whose successful writes trigger notification. */ + readonly collections: readonly Nsid[]; + /** Combine this provider with an authenticated PDS client. Provider methods + * route to Contrail; other methods route to the PDS; successful tracked + * record writes notify Contrail before returning their original response. */ + authenticated( + authenticatedClient: Client, + options?: PublicServiceAuthenticatedOptions, + ): PublicServiceClient; +}; + +export function publicServiceOAuthScope( + audience: Did, +): `rpc?lxm=*&aud=${string}` { + return `rpc?lxm=*&aud=${audience}`; +} + function xrpcMethod(pathname: string): Nsid | null { const path = pathname.startsWith("http") ? new URL(pathname).pathname @@ -111,7 +157,16 @@ export function publicServiceFetchHandler( `Contrail contract digest mismatch: expected ${options.contractDigest}, received ${value.contract.digest}`, ); } - return value.serviceAuth ?? null; + const serviceAuth = value.serviceAuth ?? null; + if ( + options.serviceDid && + serviceAuth?.audience !== options.serviceDid + ) { + throw new Error( + `Contrail service DID mismatch: expected ${options.serviceDid}, received ${serviceAuth?.audience ?? "none"}`, + ); + } + return serviceAuth; })(); return serviceAuthPromise; }; @@ -128,9 +183,9 @@ export function publicServiceFetchHandler( auth: PublicServiceAuthContract, force = false, ): Promise => { - if (!options.authenticatedPds) { + if (!options.authenticatedClient) { throw new Error( - `Contrail method ${method} requires an authenticated PDS client`, + `Contrail method ${method} requires an authenticated AT Protocol client`, ); } const cached = tokens.get(method); @@ -143,7 +198,7 @@ export function publicServiceFetchHandler( } const pending = (async () => { - const response = await options.authenticatedPds!.get( + const response = await options.authenticatedClient!.get( "com.atproto.server.getServiceAuth", { params: { @@ -171,7 +226,7 @@ export function publicServiceFetchHandler( return async (pathname, init) => { const method = xrpcMethod(pathname); - if (!method || !options.authenticatedPds) return base(pathname, init); + if (!method || !options.authenticatedClient) return base(pathname, init); // Once discovery has been loaded, avoid the initial challenge on subsequent // protected calls. Anonymous calls never wait for discovery. @@ -198,10 +253,226 @@ export function publicServiceFetchHandler( }; } +interface UntypedRequestOptions { + input?: unknown; + [key: string]: unknown; +} + +interface UntypedClientResponse { + ok: boolean; + status: number; + headers: Headers; + data: unknown; +} + +type UntypedMethod = ( + name: string, + options?: UntypedRequestOptions, +) => Promise; +type UntypedCall = ( + schema: unknown, + options?: UntypedRequestOptions, +) => Promise; + +const NOTIFIED_WRITE_METHODS = new Set([ + "com.atproto.repo.createRecord", + "com.atproto.repo.putRecord", + "com.atproto.repo.deleteRecord", +]); + +function objectValue(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function writtenRecordUri( + method: string, + request: UntypedRequestOptions | undefined, + response: UntypedClientResponse, + collections: ReadonlySet, +): string | null { + if (!response.ok || !NOTIFIED_WRITE_METHODS.has(method)) return null; + const input = objectValue(request?.input); + const collection = input?.collection; + if (typeof collection !== "string" || !collections.has(collection)) { + return null; + } + + if ( + method === "com.atproto.repo.createRecord" || + method === "com.atproto.repo.putRecord" + ) { + const uri = objectValue(response.data)?.uri; + return typeof uri === "string" ? uri : null; + } + + const repo = input?.repo; + const rkey = input?.rkey; + return typeof repo === "string" && + repo.startsWith("did:") && + typeof rkey === "string" + ? `at://${repo}/${collection}/${rkey}` + : null; +} + +function schemaNsid(schema: unknown): string | null { + const namespace = objectValue(schema); + const value = objectValue(namespace?.mainSchema) ?? namespace; + return typeof value?.nsid === "string" ? value.nsid : null; +} + +function createClient( + options: PublicServiceClientOptions, + authenticatedOptions: PublicServiceAuthenticatedOptions = {}, +): PublicServiceClient { + const endpoint = normalizePublicServiceEndpoint(options.endpoint); + const authenticatedClients = new WeakMap(); + const client = new Client({ + handler: publicServiceFetchHandler({ ...options, endpoint }), + }) as PublicServiceClient; + const expectedScope = options.serviceDid + ? publicServiceOAuthScope(options.serviceDid) + : null; + if (options.scope && options.scope !== expectedScope) { + throw new Error( + `Contrail OAuth scope mismatch: expected ${expectedScope ?? "none"}, received ${options.scope}`, + ); + } + const scope = options.scope ?? expectedScope; + const collections = Object.freeze([...(options.collections ?? [])]); + + Object.defineProperties(client, { + endpoint: { value: endpoint, enumerable: true }, + scope: { value: scope, enumerable: true }, + collections: { value: collections, enumerable: true }, + authenticated: { + enumerable: true, + value( + authenticatedClient: Client, + childOptions: PublicServiceAuthenticatedOptions = {}, + ) { + if ( + options.authenticatedClient === authenticatedClient && + !childOptions.onNotificationError + ) { + return client; + } + if (!childOptions.onNotificationError) { + const existing = authenticatedClients.get(authenticatedClient); + if (existing) return existing; + } + const created = createClient( + { ...options, endpoint, authenticatedClient }, + childOptions, + ); + if (!childOptions.onNotificationError) { + authenticatedClients.set(authenticatedClient, created); + } + return created; + }, + }, + }); + + if (options.authenticatedClient && options.serviceMethods) { + const serviceMethods = new Set(options.serviceMethods); + const trackedCollections = new Set(collections); + const serviceGet = client.get.bind(client) as unknown as UntypedMethod; + const servicePost = client.post.bind(client) as unknown as UntypedMethod; + const serviceCall = client.call.bind(client) as unknown as UntypedCall; + const pdsGet = options.authenticatedClient.get.bind( + options.authenticatedClient, + ) as unknown as UntypedMethod; + const pdsPost = options.authenticatedClient.post.bind( + options.authenticatedClient, + ) as unknown as UntypedMethod; + const pdsCall = options.authenticatedClient.call.bind( + options.authenticatedClient, + ) as unknown as UntypedCall; + + const reportNotificationError = ( + error: unknown, + method: string, + uris: readonly string[], + ) => { + try { + authenticatedOptions.onNotificationError?.(error, { + method: method as Nsid, + uris, + }); + } catch { + // A reporting callback must never turn a committed PDS write into a + // failed write response. + } + }; + + const notifyWrite = async (method: string, uri: string) => { + if (!options.notifyMethod) return; + try { + const notified = await servicePost(options.notifyMethod, { + input: { uris: [uri] }, + }); + if (!notified.ok) { + throw new Error( + `Contrail notification failed with status ${notified.status}`, + ); + } + const errors = objectValue(notified.data)?.errors; + if (Array.isArray(errors) && errors.length > 0) { + throw new Error( + `Contrail notification reported errors: ${errors.join("; ")}`, + ); + } + } catch (error) { + reportNotificationError(error, method, [uri]); + } + }; + + Object.defineProperties(client, { + get: { + value: ((name: string, request?: UntypedRequestOptions) => + serviceMethods.has(name) + ? serviceGet(name, request) + : pdsGet(name, request)) as Client["get"], + }, + post: { + value: (async (name: string, request?: UntypedRequestOptions) => { + if (serviceMethods.has(name)) return servicePost(name, request); + const response = await pdsPost(name, request); + const uri = writtenRecordUri( + name, + request, + response, + trackedCollections, + ); + if (uri) await notifyWrite(name, uri); + return response; + }) as Client["post"], + }, + call: { + value: ((schema: unknown, request?: UntypedRequestOptions) => { + const method = schemaNsid(schema); + return method && serviceMethods.has(method) + ? serviceCall(schema, request) + : pdsCall(schema, request); + }) as Client["call"], + }, + }); + } + + return client; +} + /** Create a typed Atcute client for anonymous and service-auth Contrail methods. * Generated Lexicon imports still supply the method-specific TypeScript API. */ +export function createPublicServiceClient( + options: PublicServiceClientOptions & { serviceDid: Did }, +): PublicServiceClient & { readonly scope: `rpc?lxm=*&aud=${string}` }; +export function createPublicServiceClient( + options: PublicServiceClientOptions, +): PublicServiceClient; export function createPublicServiceClient( options: PublicServiceClientOptions, -): Client { - return new Client({ handler: publicServiceFetchHandler(options) }); +): PublicServiceClient { + return createClient(options); } diff --git a/packages/contrail/tests/built-client.mjs b/packages/contrail/tests/built-client.mjs index 1bc59d4..479d61d 100644 --- a/packages/contrail/tests/built-client.mjs +++ b/packages/contrail/tests/built-client.mjs @@ -3,8 +3,15 @@ import { createPublicServiceClient } from "../dist/public-client.js"; const client = createPublicServiceClient({ endpoint: "https://api.example.com", + serviceDid: "did:web:api.example.com", + scope: "rpc?lxm=*&aud=did:web:api.example.com", + serviceMethods: ["com.example.listRecords"], + collections: ["community.example.event"], fetch: async () => Response.json({ records: [] }), }); +assert.equal(client.endpoint, "https://api.example.com"); +assert.equal(client.scope, "rpc?lxm=*&aud=did:web:api.example.com"); +assert.deepEqual(client.collections, ["community.example.event"]); const response = await client.get("com.example.listRecords"); assert.equal(response.ok, true); assert.deepEqual(response.data, { records: [] }); diff --git a/packages/contrail/tests/connect.test.ts b/packages/contrail/tests/connect.test.ts index ee73699..a3a9781 100644 --- a/packages/contrail/tests/connect.test.ts +++ b/packages/contrail/tests/connect.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { connectPublicService, + ensureConsumerClientModule, ensureConsumerLexiconConfig, type ProviderLock, } from "../src/cli/commands/connect"; @@ -49,8 +50,13 @@ function providerLock(): ProviderLock { contractDigest: `sha256:${"a".repeat(64)}`, lexiconDigest: `sha256:${"b".repeat(64)}`, methods: [method], - serviceAuth: null, - lexiconRoot: "lexicons/pulled/api.atmo.rsvp", + collections: ["community.lexicon.calendar.event"], + serviceAuth: { + type: "atproto-service-auth", + audience: "did:web:api.atmo.rsvp", + methods: [{ id: notifyMethod, type: "procedure" }], + }, + lexiconRoot: "src/contrail/lexicons/api.atmo.rsvp", }; } @@ -103,21 +109,44 @@ describe("contrail connect", () => { const root = await temporaryRoot(); const generated = await ensureConsumerLexiconConfig({ root, - out: "lexicons/providers", + out: "src/contrail/lexicons", + lock: providerLock(), }); expect(generated.created).toBe(true); expect(await readFile(generated.path, "utf8")).toContain( - 'files: ["lexicons/providers/**/*.json"]', + 'files: ["src/contrail/lexicons/**/*.json"]', + ); + expect(await readFile(generated.path, "utf8")).toContain( + 'outdir: "src/contrail/types/"', + ); + expect(await readFile(generated.path, "utf8")).toContain( + 'endpoint: "https://api.atmo.rsvp"', + ); + expect(await readFile(generated.path, "utf8")).toContain( + 'serviceDid: "did:web:api.atmo.rsvp"', ); expect(await readFile(generated.path, "utf8")).toContain( - 'outdir: "src/lexicons/"', + 'scope: "rpc?lxm=*&aud=did:web:api.atmo.rsvp"', + ); + expect(await readFile(generated.path, "utf8")).toContain( + '"community.lexicon.calendar.event",', + ); + const updated = await ensureConsumerLexiconConfig({ + root, + out: "src/contrail/lexicons", + lock: { ...providerLock(), endpoint: "https://next.example.com" }, + }); + expect(updated.updated).toBe(true); + expect(await readFile(updated.path, "utf8")).toContain( + 'endpoint: "https://next.example.com"', ); await writeFile(join(root, "lex.config.ts"), "export default { mine: true }"); await rm(generated.path); const existing = await ensureConsumerLexiconConfig({ root, - out: "lexicons/other", + out: "src/other", + lock: providerLock(), }); expect(existing.created).toBe(false); expect(existing.path).toBe(join(root, "lex.config.ts")); @@ -126,6 +155,74 @@ describe("contrail connect", () => { ); }); + it("generates TypeScript or JavaScript client modules without replacing one", async () => { + const root = await temporaryRoot(); + const fixture = await serviceFixture([ + methodLexicon, + sourceLexicon, + notifyLexicon, + ]); + fixture.manifest.serviceAuth = { + type: "atproto-service-auth", + audience: "did:web:api.atmo.rsvp", + methods: [{ id: notifyMethod, type: "procedure" }], + }; + fixture.manifest.contract.digest = await digestPublicContract( + contractFromManifest(fixture.manifest), + ); + const { lock } = await connectPublicService({ + endpoint, + root, + out: "lexicons/pulled", + lock: "contrail.lock.json", + fetcher: fixture.fetcher, + }); + + const generated = await ensureConsumerClientModule({ root, lock }); + expect(generated.created).toBe(true); + const source = await readFile(generated.path, "utf8"); + expect(source).toContain( + 'import type {} from "./types/index.js"', + ); + expect(source).toContain(`endpoint: ${JSON.stringify(endpoint)}`); + expect(source).toContain( + 'serviceDid: "did:web:api.atmo.rsvp"', + ); + expect(source).toContain( + 'scope: "rpc?lxm=*&aud=did:web:api.atmo.rsvp"', + ); + expect(source).toContain('"community.lexicon.calendar.event",'); + expect(source).toContain( + `notifyMethod: ${JSON.stringify(notifyMethod)}`, + ); + + const updated = await ensureConsumerClientModule({ + root, + lock: { ...lock, endpoint: "https://next.example.com" }, + }); + expect(updated.updated).toBe(true); + expect(await readFile(updated.path, "utf8")).toContain( + 'endpoint: "https://next.example.com"', + ); + + await writeFile(generated.path, "export const mine = true;\n"); + const existing = await ensureConsumerClientModule({ root, lock }); + expect(existing.created).toBe(false); + expect(await readFile(existing.path, "utf8")).toBe( + "export const mine = true;\n", + ); + + const javascript = await ensureConsumerClientModule({ + root, + file: "client/contrail.js", + lock, + }); + expect(javascript.created).toBe(true); + expect(await readFile(javascript.path, "utf8")).not.toContain( + "lexicons/index", + ); + }); + it("verifies and atomically locks a discovered service", async () => { const root = await temporaryRoot(); const { fetcher, manifest, values } = await serviceFixture(); diff --git a/packages/contrail/tests/public-client.test.ts b/packages/contrail/tests/public-client.test.ts index 97187c5..7cae200 100644 --- a/packages/contrail/tests/public-client.test.ts +++ b/packages/contrail/tests/public-client.test.ts @@ -9,6 +9,8 @@ import type { PublicServiceManifest } from "../src/public-service"; const endpoint = "https://api.example.com"; const method = "com.example.getFeed"; +const notifyMethod = "com.example.notifyOfUpdate"; +const collection = "community.example.event"; const digest = `sha256:${"a".repeat(64)}`; function token() { @@ -39,7 +41,7 @@ function manifest(): PublicServiceManifest { }; } -function authenticatedPds(jwt: string) { +function authenticatedClient(jwt: string) { const handler = vi.fn(async (pathname: string) => { const url = new URL(pathname, "https://pds.example.com"); expect(url.pathname).toBe("/xrpc/com.atproto.server.getServiceAuth"); @@ -51,6 +53,16 @@ function authenticatedPds(jwt: string) { } describe("public service client", () => { + it("rejects a configured OAuth scope for a different service DID", () => { + expect(() => + createPublicServiceClient({ + endpoint, + serviceDid: "did:web:api.example.com", + scope: "rpc?lxm=*&aud=did:web:other.example.com", + }), + ).toThrow("OAuth scope mismatch"); + }); + it("keeps anonymous requests anonymous", async () => { const fetcher = vi.fn(async () => Response.json({ records: [] })); const handler = publicServiceFetchHandler({ endpoint, fetch: fetcher }); @@ -65,7 +77,7 @@ describe("public service client", () => { it("discovers, mints, caches, and attaches method-bound tokens", async () => { const jwt = token(); - const pds = authenticatedPds(jwt); + const pds = authenticatedClient(jwt); const requests: Array<{ url: string; authorization: string | null }> = []; const fetcher = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { const url = String(input); @@ -81,12 +93,20 @@ describe("public service client", () => { { status: 401, headers: { "www-authenticate": "Bearer" } }, ); }); - const client = createPublicServiceClient({ + const publicClient = createPublicServiceClient({ endpoint, - authenticatedPds: pds.client, contractDigest: digest, + serviceDid: "did:web:api.example.com", + scope: "rpc?lxm=*&aud=did:web:api.example.com", + serviceMethods: [method], fetch: fetcher, }); + expect(publicClient.endpoint).toBe(endpoint); + expect(publicClient.scope).toBe( + "rpc?lxm=*&aud=did:web:api.example.com", + ); + const client = publicClient.authenticated(pds.client); + expect(publicClient.authenticated(pds.client)).toBe(client); const first = await (client as any).get(method, { params: { actor: "did:plc:test", feed: "network" }, @@ -105,8 +125,208 @@ describe("public service client", () => { ).toEqual([null, `Bearer ${jwt}`, `Bearer ${jwt}`]); }); + it("routes PDS writes and service methods through one authenticated client", async () => { + const jwt = token(); + const discovered = manifest(); + discovered.serviceAuth = { + type: "atproto-service-auth", + audience: "did:web:api.example.com", + methods: [ + { id: method, type: "query" }, + { id: notifyMethod, type: "procedure" }, + ], + }; + let pdsWriteShouldFail = false; + const pdsHandler = vi.fn(async (pathname: string) => { + const url = new URL(pathname, "https://pds.example.com"); + if (url.pathname === "/xrpc/com.atproto.server.getServiceAuth") { + return Response.json({ token: jwt }); + } + if (url.pathname === "/xrpc/com.atproto.repo.getRecord") { + return Response.json({ + uri: `at://did:plc:test/${collection}/3test`, + cid: "bafyreicid", + value: { $type: collection, name: "Test event" }, + }); + } + if (url.pathname === "/xrpc/com.atproto.repo.deleteRecord") { + return Response.json({}); + } + if ( + url.pathname === "/xrpc/com.atproto.repo.createRecord" || + url.pathname === "/xrpc/com.atproto.repo.putRecord" + ) { + if (pdsWriteShouldFail) { + return Response.json({ error: "InvalidRecord" }, { status: 400 }); + } + return Response.json({ + uri: `at://did:plc:test/${collection}/3test`, + cid: "bafyreicid", + }); + } + return Response.json({ error: "MethodNotFound" }, { status: 404 }); + }); + const pds = new Client({ handler: pdsHandler }); + const serviceRequests: Array<{ url: string; body: string | null }> = []; + let notificationShouldFail = false; + const fetcher = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/.well-known/contrail")) { + return Response.json(discovered); + } + serviceRequests.push({ + url, + body: typeof init?.body === "string" ? init.body : null, + }); + const authorization = new Headers(init?.headers).get("authorization"); + if (authorization !== `Bearer ${jwt}`) { + return Response.json( + { error: "AuthenticationRequired" }, + { status: 401 }, + ); + } + return url.endsWith(`/xrpc/${notifyMethod}`) + ? Response.json({ + indexed: notificationShouldFail ? 0 : 1, + deleted: 0, + ...(notificationShouldFail ? { errors: ["retry later"] } : {}), + }) + : Response.json({ records: [] }); + }); + const notificationError = vi.fn(); + const client = createPublicServiceClient({ + endpoint, + serviceDid: "did:web:api.example.com", + serviceMethods: [method, notifyMethod], + collections: [collection], + notifyMethod, + fetch: fetcher, + }).authenticated(pds, { onNotificationError: notificationError }); + + const write = await client.post("com.atproto.repo.createRecord", { + input: { + repo: "did:plc:test", + collection, + record: { $type: collection, name: "Test event" }, + }, + }); + const feed = await (client as any).get(method, { + params: { actor: "did:plc:test", feed: "network" }, + }); + const record = await client.get("com.atproto.repo.getRecord", { + params: { + repo: "did:plc:test", + collection, + rkey: "3test", + }, + }); + + expect(write.ok).toBe(true); + expect(feed.ok).toBe(true); + expect(record.ok).toBe(true); + expect(notificationError).not.toHaveBeenCalled(); + + notificationShouldFail = true; + const updated = await client.post("com.atproto.repo.putRecord", { + input: { + repo: "did:plc:test", + collection, + rkey: "3test", + record: { $type: collection, name: "Updated event" }, + }, + }); + expect(updated.ok).toBe(true); + expect(notificationError).toHaveBeenCalledOnce(); + + notificationShouldFail = false; + const notificationsBeforeDelete = serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length; + const deleted = await client.post("com.atproto.repo.deleteRecord", { + input: { + repo: "did:plc:test", + collection, + rkey: "3test", + }, + }); + expect(deleted.ok).toBe(true); + expect( + serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length, + ).toBe(notificationsBeforeDelete + 1); + + pdsWriteShouldFail = true; + const notificationsBeforeFailedWrite = serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length; + const failed = await client.post("com.atproto.repo.createRecord", { + input: { + repo: "did:plc:test", + collection, + record: { $type: collection, name: "Invalid" }, + }, + }); + expect(failed.ok).toBe(false); + expect( + serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length, + ).toBe(notificationsBeforeFailedWrite); + pdsWriteShouldFail = false; + + const notificationsBeforeUntrackedWrite = serviceRequests.filter( + ({ url }) => url.endsWith(`/xrpc/${notifyMethod}`), + ).length; + const untracked = await client.post("com.atproto.repo.createRecord", { + input: { + repo: "did:plc:test", + collection: "app.bsky.feed.post", + record: { $type: "app.bsky.feed.post", text: "Not tracked" }, + }, + }); + expect(untracked.ok).toBe(true); + expect( + serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length, + ).toBe(notificationsBeforeUntrackedWrite); + expect( + serviceRequests.some( + ({ url, body }) => + url.endsWith(`/xrpc/${notifyMethod}`) && + body?.includes(`at://did:plc:test/${collection}/3test`), + ), + ).toBe(true); + expect( + pdsHandler.mock.calls.some(([pathname]) => + String(pathname).includes("com.atproto.repo.createRecord"), + ), + ).toBe(true); + }); + + it("refuses a discovered service-auth audience that differs from its lock", async () => { + const pds = authenticatedClient(token()); + const fetcher = vi.fn(async (input: RequestInfo | URL) => + String(input).endsWith("/.well-known/contrail") + ? Response.json(manifest()) + : new Response(null, { status: 401 }), + ); + const client = createPublicServiceClient({ + endpoint, + authenticatedClient: pds.client, + serviceDid: "did:web:other.example.com", + fetch: fetcher, + }); + + await expect( + (client as any).get(method), + ).rejects.toThrow("service DID mismatch"); + expect(pds.handler).not.toHaveBeenCalled(); + }); + it("refuses runtime discovery that differs from an optional lock pin", async () => { - const pds = authenticatedPds(token()); + const pds = authenticatedClient(token()); const fetcher = vi.fn(async (input: RequestInfo | URL) => String(input).endsWith("/.well-known/contrail") ? Response.json(manifest()) @@ -114,7 +334,7 @@ describe("public service client", () => { ); const handler = publicServiceFetchHandler({ endpoint, - authenticatedPds: pds.client, + authenticatedClient: pds.client, contractDigest: `sha256:${"b".repeat(64)}`, fetch: fetcher, }); diff --git a/packages/contrail/tests/public-service-e2e.test.ts b/packages/contrail/tests/public-service-e2e.test.ts index 881be2d..6d3c14c 100644 --- a/packages/contrail/tests/public-service-e2e.test.ts +++ b/packages/contrail/tests/public-service-e2e.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { connectPublicService, + ensureConsumerClientModule, ensureConsumerLexiconConfig, } from "../src/cli/commands/connect"; import { generateLexiconTypesWithAtcute } from "../src/cli/atcute"; @@ -104,29 +105,31 @@ describe("public service consumer integration", () => { const fetcher: typeof fetch = (input, init) => app.fetch(new Request(input, init)); - await connectPublicService({ + const connection = await connectPublicService({ endpoint: "https://api.example.com", root: consumerRoot, - out: "lexicons/pulled", + out: "src/contrail/lexicons", lock: "contrail.lock.json", fetcher, }); const generatedConfig = await ensureConsumerLexiconConfig({ root: consumerRoot, - out: "lexicons/pulled", + out: "src/contrail/lexicons", + lock: connection.lock, }); expect(generatedConfig.created).toBe(true); generateLexiconTypesWithAtcute(consumerRoot); + const generatedClient = await ensureConsumerClientModule({ + root: consumerRoot, + lock: connection.lock, + }); + expect(generatedClient.created).toBe(true); mkdirSync(join(consumerRoot, "src"), { recursive: true }); writeFileSync( join(consumerRoot, "src", "consumer.ts"), - `import { Client, simpleFetchHandler } from "@atcute/client"; -import "./lexicons/index.js"; -const client = new Client({ - handler: simpleFetchHandler({ service: "https://api.example.com" }), -}); -const response = await client.get("com.example.event.listRecords", { + `import { contrail } from "./contrail/index.js"; +const response = await contrail.get("com.example.event.listRecords", { params: { name: "Typed event", limit: 1 }, }); if (response.ok) { -- 2.51.2 From f832257eb9a2ac535d2ff798c45ed649c0ce2e67 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:43:21 +0200 Subject: [PATCH 14/15] Document public Contrail services --- README.md | 6 +- apps/atmo-rsvp/README.md | 40 +++- docs/public-services/api-atmo-rsvp.md | 240 ++++++++++++++++++++++ docs/public-services/creating.md | 279 ++++++++++++++++++++++++++ docs/public-services/using.md | 136 +++++++++++++ packages/contrail/README.md | 21 +- 6 files changed, 712 insertions(+), 10 deletions(-) create mode 100644 docs/public-services/api-atmo-rsvp.md create mode 100644 docs/public-services/creating.md create mode 100644 docs/public-services/using.md diff --git a/README.md b/README.md index 145ed35..85c9774 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Contrail remains a read-through cache over public AT Protocol data: anonymous re Consumers connect and generate Atcute types with one command: ```bash -pnpm contrail connect https://api.example.com +pnpx @atmo-dev/contrail connect https://api.example.com ``` Public-service mode requires `orderedSource`; `getCursor` then returns the committed opaque `{ source, epoch, cursor }` position of that primary source. Compare complete positions for equality only; a source or epoch change requires a full client refetch. To avoid racing ingestion, read a position before and after a query and accept the query snapshot only when both positions match. Existing non-public deployments without `orderedSource` retain the legacy `time_us`, `date`, and `seconds_ago` response. @@ -128,6 +128,10 @@ See [Indexing](docs/01-indexing.md) for adapter setup and [Querying](docs/02-que - [Querying](docs/02-querying.md) - [Feeds](docs/04-feeds.md) - [Labels](docs/09-labels.md) +- Public Contrail services: + - [Creating a service](docs/public-services/creating.md) + - [Using a service](docs/public-services/using.md) + - [Example: api.atmo.rsvp](docs/public-services/api-atmo-rsvp.md) - [SvelteKit + Cloudflare](docs/frameworks/sveltekit-cloudflare.md) ## Repository layout diff --git a/apps/atmo-rsvp/README.md b/apps/atmo-rsvp/README.md index 6e14e4e..a908dc4 100644 --- a/apps/atmo-rsvp/README.md +++ b/apps/atmo-rsvp/README.md @@ -1,39 +1,63 @@ # api.atmo.rsvp -Anonymous public Contrail read-through service for calendar events and RSVPs. Records remain owned by their authors' PDSes; reads may acquire and cache missing public data. +Public Contrail read-through service for calendar events, RSVPs, actor profiles, and personalized network feeds. Records remain owned by their authors' PDSes; reads may acquire and cache missing public data. Public discovery: ```text https://api.atmo.rsvp/.well-known/contrail +https://api.atmo.rsvp/.well-known/did.json https://api.atmo.rsvp/lexicons https://api.atmo.rsvp/status ``` -Typed XRPC methods: +Anonymous XRPC queries: ```text rsvp.atmo.getCursor +rsvp.atmo.getProfile rsvp.atmo.event.getRecord rsvp.atmo.event.listRecords rsvp.atmo.rsvp.getRecord rsvp.atmo.rsvp.listRecords ``` +AT Protocol service-auth methods: + +```text +rsvp.atmo.getFeed +rsvp.atmo.notifyOfUpdate +``` + +The service-auth audience is `did:web:api.atmo.rsvp`. A consumer can request one OAuth permission for both protected methods: + +```text +rpc?lxm=*&aud=did:web:api.atmo.rsvp +``` + +Tokens remain method-bound. Call `com.atproto.server.getServiceAuth` with the specific `lxm` being invoked, then send its token as `Authorization: Bearer `. + +`getFeed` requires the requested actor to resolve to the token issuer. `notifyOfUpdate` accepts only AT URIs owned by the token issuer and always refetches their current authoritative state from that issuer's PDS. Notify is an authenticated cache hint, not a write proxy. + +Event, RSVP, and feed reads can hydrate indexed actor profiles. Follows are an internal feed input: the service retains scoped follow records whose subjects are already in its acquisition scope rather than attempting to mirror or expose the network-wide social graph. + `getCursor` returns the committed opaque `{ source, epoch, cursor }` position of the primary Jetstream source. Clients compare complete positions for equality and fully refetch when the source or epoch changes. -The service has no user sessions, service DID, or write proxy. Applications authenticate and write through users' PDSes. +The service has no user sessions and never signs or publishes records. Applications authenticate users and write through their PDSes. ## Development +Build the workspace package before running the app directly: + ```bash +pnpm --filter @atmo-dev/contrail build pnpm --dir apps/atmo-rsvp lexicons:all pnpm --dir apps/atmo-rsvp typecheck pnpm --dir apps/atmo-rsvp dev pnpm --dir apps/atmo-rsvp backfill:dev ``` -The development backfill uses the local Wrangler/Miniflare D1 binding. It remains resumable and uses the same validation, retry, and completion logic as production ingestion. +The development backfill uses the local Wrangler/Miniflare D1 binding. It remains resumable and uses the same retry and completion logic as production ingestion. ## Deployment @@ -43,12 +67,14 @@ pnpm --dir apps/atmo-rsvp typecheck pnpm --dir apps/atmo-rsvp deploy ``` -The deployed D1 database is already provisioned. Production bulk provisioning does not use Wrangler's remote development proxy. The planned repeatable workflow builds and verifies a fresh native SQLite generation, imports canonical tables into a fresh D1 database, rebuilds derived projections, verifies readiness, and then activates it. +Production bulk provisioning does not use Wrangler's remote development proxy. A fresh deployment generation is built with capture-first native SQLite snapshot/replay, verified, imported into a fresh D1 database, checked through a candidate Worker, and then activated by deploying the matching Worker/D1 binding together. Consumer projects connect after installing Contrail: ```bash -pnpm contrail connect https://api.atmo.rsvp +pnpx @atmo-dev/contrail connect https://api.atmo.rsvp ``` -That verifies the canonical service and Lexicon digests, writes a provider lock, installs the provider-owned Lexicons, and runs Atcute TypeScript generation. Reconnecting an existing project requires `--update`. +That verifies the anonymous and service-auth contracts, verifies the canonical contract and Lexicon digests, writes a provider lock, installs provider-owned Lexicons, and runs Atcute TypeScript generation. Reconnecting an existing project requires `--update`. + +See [Example: api.atmo.rsvp](../../docs/public-services/api-atmo-rsvp.md) for the complete method, authentication, acquisition, and deployment walkthrough. diff --git a/docs/public-services/api-atmo-rsvp.md b/docs/public-services/api-atmo-rsvp.md new file mode 100644 index 0000000..f6d2183 --- /dev/null +++ b/docs/public-services/api-atmo-rsvp.md @@ -0,0 +1,240 @@ +# Example: api.atmo.rsvp + +[`https://api.atmo.rsvp`](https://api.atmo.rsvp) is a public Contrail read-through service for AT Protocol calendar events and RSVPs. It demonstrates anonymous collection queries, profile hydration, a personalized network feed, authenticated update notifications, verified remote discovery, and an immutable D1 deployment generation. + +## Discovery + +```text +https://api.atmo.rsvp/.well-known/contrail +https://api.atmo.rsvp/.well-known/did.json +https://api.atmo.rsvp/lexicons +https://api.atmo.rsvp/status +``` + +The XRPC namespace is DNS-authoritative: + +```text +rsvp.atmo.* +``` + +The service DID and service-auth audience are: + +```text +did:web:api.atmo.rsvp +``` + +The service DID identifies the API as a JWT audience. The API does not use it to sign user records and does not act as a PDS. + +## Anonymous methods + +```text +rsvp.atmo.getCursor +rsvp.atmo.getProfile +rsvp.atmo.event.getRecord +rsvp.atmo.event.listRecords +rsvp.atmo.rsvp.getRecord +rsvp.atmo.rsvp.listRecords +``` + +Event queries support equality and date-range filtering, full-text search over names and descriptions, stable keyset pagination, RSVP relation counts, RSVP hydration, and actor profile hydration. + +RSVP queries support status, subject URI, and creation-time filtering. They can hydrate the referenced event and actor profiles. + +`getProfile` resolves an actor and reads their indexed `app.bsky.actor.profile` record. Missing public profile data may be fetched from that actor's PDS as part of the read-through request. + +## Protected methods + +Discovery lists these separately under AT Protocol service auth: + +```text +rsvp.atmo.getFeed +rsvp.atmo.notifyOfUpdate +``` + +The module generated by `contrail connect` exposes the required OAuth permission: + +```ts +import { contrail } from "./contrail/index.js"; + +export const scopes = ["atproto", contrail.scope]; +// contrail.scope is rpc?lxm=*&aud=did:web:api.atmo.rsvp +``` + +After login, derive one client from the user's existing authenticated AT Protocol client: + +```ts +const client = contrail.authenticated(authenticatedClient, { + onNotificationError(error, { uris }) { + console.warn("Contrail notification failed", uris, error); + }, +}); +``` + +Advertised service methods route to Contrail; other methods route to the PDS. Protected methods automatically obtain and cache exact method-bound tokens. + +Missing, expired, wrong-audience, wrong-method, and invalid-signature tokens receive `401` with a `WWW-Authenticate` challenge. + +## Personalized network feed + +The configured feed is: + +```text +feed=network +``` + +It contains recent events and RSVPs authored by actors followed by the signed-in user. Per-actor projection caps are: + +| Collection | Maximum retained items | +|---|---:| +| `community.lexicon.calendar.event` | 100 | +| `community.lexicon.calendar.rsvp` | 250 | + +A typed query looks like: + +```ts +const response = await client.get( + "rsvp.atmo.getFeed", + { + params: { + feed: "network", + actor: signedInDid, + collection: "community.lexicon.calendar.event", + profiles: true, + limit: 20, + }, + }, +); +``` + +The requested actor must resolve to the token issuer. Service auth therefore prevents one authenticated account from creating or refreshing arbitrary personalized feed projections for other actors. + +The underlying `app.bsky.graph.follow` records are internal. The service does not advertise raw follow collection methods. It indexes follows authored by known actors only when the follow subject is already in the service's acquisition scope. Constellation enrichment helps connect newly observed calendar authors to existing in-scope followers. + +The feed endpoint may start a bounded background follow backfill the first time an actor requests their feed. Until that finishes, an initial response may be empty or partial. The operation remains a read-through cache fill rather than a network-wide social graph crawl. + +## Immediate update notification + +Successful event and RSVP writes through the combined client automatically notify Contrail: + +```ts +const response = await client.post("com.atproto.repo.createRecord", { + input: { + repo: signedInDid, + collection: "community.lexicon.calendar.event", + record: event, + }, +}); +``` + +The original PDS response is returned unchanged. Notification failures are nonfatal and reported through `onNotificationError`. The protected `rsvp.atmo.notifyOfUpdate` procedure remains available for explicit batches or records written elsewhere. + +The endpoint enforces all of the following: + +- at most 25 URIs per request; +- every URI is a canonical record AT URI; +- every URI belongs to the service-token issuer; +- the collection is tracked by this deployment; +- the record body and CID are fetched from the issuer's current PDS; and +- only an explicit XRPC `RecordNotFound` response is treated as deletion. + +Transient DNS, identity, network, timeout, PDS, or malformed-response failures preserve existing indexed state and are returned as bounded per-record errors to the authenticated caller. + +## Profiles + +The deployment indexes: + +```text +app.bsky.actor.profile +``` + +but keeps the underlying profile collection methods internal. Profiles are exposed through: + +- `rsvp.atmo.getProfile`; +- `profiles=true` on event and RSVP reads; and +- `profiles=true` on network feed reads. + +This avoids a redundant raw profile-record API while still providing typed display names, handles, avatars, and profile values alongside calendar data. + +## Acquisition scope + +Relay discovery starts from: + +```text +community.lexicon.calendar.event +community.lexicon.calendar.rsvp +``` + +Profiles and follows are dependent collections. They do not independently discover every Bluesky repository. + +The service is intentionally a shared, possibly incomplete cache. Unavailable identities and PDSes stay visibly pending, retrying, or failed instead of being silently marked complete. Unknown dependent subjects are scope exclusions and do not create tombstones. + +## Runtime validation policy + +This deployment publishes verified API and record Lexicons for discovery and TypeScript generation, but does not enable Contrail's optional runtime record/CID validation during ingestion. + +That distinction is deliberate: + +- provider and consumer contracts are still canonical and digest-verified; +- generated Atcute types still describe response values; +- startup still rejects an inconsistent advertised API; but +- ingestion does not reject records based on runtime Lexicon or canonical-CID checks. + +## Ordered source position + +The primary ordered source is one pinned Jetstream endpoint with an operator-owned continuity epoch: + +```json +{ + "source": "jetstream", + "epoch": "api-atmo-rsvp-primary-2026-08" +} +``` + +`rsvp.atmo.getCursor` returns the currently committed opaque position. Consumers compare the complete source, epoch, and cursor for equality only. A source or epoch change requires a full refetch. + +Backfills do not send historical notifications. Jetstream projection advances the serving position atomically with accepted live mutations. + +## Production generation + +The current expanded generation was built in native SQLite before activation. Its initial canonical projection contained approximately: + +| Collection | Records | +|---|---:| +| Events | 14,751 | +| RSVPs | 6,299 | +| Profiles | 1,408 | +| Scoped follows | 65,798 | + +The exact live totals change as Jetstream and read-through acquisition continue. + +The provisioning process: + +1. captured a replay boundary before relay discovery; +2. ran the resumable native-SQLite backfill; +3. retained 43 unavailable accounts as explicit scheduled retries; +4. replayed Jetstream to the present; +5. imported canonical tables into a fresh D1 database; +6. rebuilt FTS and materialized RSVP counts; +7. verified all visible rows against durable record versions; +8. exercised discovery, profiles, search, CORS, and protected-route rejection through a candidate Worker; and +9. activated the new Worker and D1 binding together. + +The previous D1 generation remains separate for rollback. No percentage traffic split is used between databases with independent serving positions. + +## Connect a consumer + +```bash +pnpx @atmo-dev/contrail connect https://api.atmo.rsvp +``` + +The provider lock records: + +- the HTTPS endpoint; +- `rsvp.atmo` namespace; +- anonymous methods; +- protected methods and their audience; +- the contract digest; +- the Lexicon digest; and +- the provider-owned Lexicon directory. + +See [Using a public Contrail service](./using.md) for a framework-neutral consumer walkthrough. diff --git a/docs/public-services/creating.md b/docs/public-services/creating.md new file mode 100644 index 0000000..b2e7264 --- /dev/null +++ b/docs/public-services/creating.md @@ -0,0 +1,279 @@ +# Creating a public Contrail service + +A public Contrail service lets independent applications query one Contrail AppView from a stable HTTPS origin. The provider chooses the indexed collections, projections, query methods, and authentication policy. Consumers discover that contract, verify its Lexicons, generate local TypeScript types, and make ordinary XRPC requests. + +Public service mode does not turn Contrail into a PDS. Records remain in their authors' repositories, and applications still authenticate users and publish writes through those users' PDSes. + +## Define the index + +Start with a normal Contrail configuration: + +```ts +// src/contrail.config.ts +import type { ContrailConfig } from "@atmo-dev/contrail"; + +export const config: ContrailConfig = { + namespace: "events.example", + orderedSource: { + source: "jetstream", + epoch: "primary-2026-08", + }, + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { + mode: {}, + startsAt: { type: "range" }, + }, + searchable: ["name", "description"], + }, + }, +}; +``` + +The namespace becomes the prefix of generated methods such as: + +```text +events.example.getCursor +events.example.event.getRecord +events.example.event.listRecords +``` + +Use one stable `orderedSource.epoch` for one continuity history. Change the epoch when the Jetstream endpoint set, retention assumptions, or cursor meaning changes. Consumers treat a source or epoch change as a full-refetch boundary. + +## Generate the public Lexicons + +Add Atcute's generator configuration: + +```js +// lex.config.js +import { defineLexiconConfig } from "@atcute/lex-cli"; + +export default defineLexiconConfig({ + generate: { + files: [ + "lexicons/custom/**/*.json", + "lexicons/pulled/**/*.json", + "lexicons/generated/**/*.json", + ], + outdir: "src/lexicon-types/", + }, +}); +``` + +Generate the provider contract, pull referenced record Lexicons, and generate TypeScript types: + +```bash +pnpm contrail lexicons all --public +``` + +Check generated drift in CI: + +```bash +pnpm contrail lexicons check --public +``` + +The public surface includes anonymous queries plus explicitly configured service-auth queries and procedures. Private full-surface procedures are not included merely because they exist in application code. + +## Publish discovery from a Worker + +Pass the generated documents and canonical HTTPS origin to `createWorker`: + +```ts +// src/worker.ts +import { createWorker } from "@atmo-dev/contrail/worker"; +import { lexicons } from "../lexicons/generated"; +import { config } from "./contrail.config"; + +export default createWorker(config, { + lexicons, + publicService: { + endpoint: "https://api.example.com", + }, +}); +``` + +This publishes: + +```text +GET /.well-known/contrail +GET /lexicons +GET /lexicons/ +GET /status +``` + +The discovery manifest contains separate contract and Lexicon digests. The immutable Lexicon URL is content-addressed. Startup fails when advertised methods, capabilities, and bundled Lexicons disagree. + +The public `/status` response contains aggregate readiness and freshness information. It omits DIDs, record bodies, source cursors, raw upstream errors, and other private operational details. + +## Anonymous read-through methods + +Collection queries, `getCursor`, profiles, feeds, and authored custom queries are anonymous unless explicitly protected. An anonymous query may still improve the shared cache by: + +- resolving an actor; +- fetching a missing public record from its PDS; +- populating profile or feed projections; or +- running a trusted custom query handler. + +This is read-through acquisition, not a caller-controlled write API. Only configured collections and trusted provider code can affect the projection. + +## Protecting feeds and notifications with service auth + +AT Protocol service auth lets any suitably authorized AT Protocol client call selected methods without distributing a shared application secret. + +```ts +export const config: ContrailConfig = { + namespace: "events.example", + notify: true, + serviceAuth: { + audience: "did:web:api.example.com", + methods: ["getFeed", "notifyOfUpdate"], + }, + collections, + feeds: { + network: { + targets: [ + { collection: "event", maxItems: 100 }, + ], + }, + }, +}; +``` + +The provider verifies: + +- the JWT signature against the issuer DID's `#atproto` key; +- the exact audience DID; +- the token's exact `lxm` method claim; +- expiration and maximum token age; and +- the route-specific ownership rule. + +For protected feeds, the requested actor must resolve to the token issuer. For `notifyOfUpdate`, every submitted AT URI must belong to the token issuer. Notify still fetches the current authoritative record from that issuer's PDS; callers never submit a record body for Contrail to trust. + +The authenticated methods are listed separately from anonymous methods in discovery. Their query or procedure Lexicons remain in the provider bundle, so consumers still get generated types. + +### OAuth permission versus token binding + +A client can request one OAuth permission for all methods at this service: + +```text +rpc?lxm=*&aud=did:web:api.example.com +``` + +The wildcard belongs to the OAuth scope. It avoids asking the user for one scope per method. Each call to `com.atproto.server.getServiceAuth` should still pass the specific method NSID as `lxm`, producing a short-lived method-bound token. + +For example, a feed token uses: + +```text +lxm=events.example.getFeed +``` + +and cannot be reused for: + +```text +lxm=events.example.notifyOfUpdate +``` + +### Service DID + +When the service-auth audience matches the public origin, such as: + +```text +did:web:api.example.com +https://api.example.com +``` + +Contrail publishes the service DID document at: + +```text +https://api.example.com/.well-known/did.json +``` + +The service DID is the stable JWT audience. The AppView does not need a signing key merely to receive and verify user-issued service tokens. + +## Profiles and internal follow projections + +Profiles can be enabled without exposing raw profile collection methods: + +```ts +profiles: ["app.bsky.actor.profile"], +collections: { + event, + profile: { + collection: "app.bsky.actor.profile", + discover: false, + methods: [], + }, +} +``` + +Likewise, a follow collection can remain an internal feed input: + +```ts +follow: { + collection: "app.bsky.graph.follow", + discover: false, + subjectField: "subject", + methods: [], +} +``` + +`discover: false` prevents network-wide relay discovery for dependent collections. `subjectField: "subject"` excludes follows whose subject is outside the known acquisition scope. Those exclusions do not create tombstones. + +Profiles can then appear through `getProfile` and `profiles=true` hydration, while follows power `getFeed` without creating a public social-graph directory. + +## Runtime validation is independent + +Publishing Lexicons does not automatically enable record validation. Validation remains opt-in: + +```ts +validation: { + lexicons, + strict: true, + verifyCid: true, +} +``` + +When configured, it applies to every acquisition source. When omitted, the provider still publishes and verifies its API contract and generates consumer types, but ingested record values and CIDs are not runtime-validated by Contrail. + +## CORS + +Public services allow browser requests and explicitly permit the `Authorization`, `Content-Type`, and `Atproto-Accept-Labelers` headers. Authentication failures expose `WWW-Authenticate` so browser clients can distinguish missing, expired, wrong-audience, and wrong-method tokens. + +Never place a reusable application secret in browser code. AT Protocol service tokens are short-lived and minted for the authenticated user's DID. + +## Provisioning and activation + +Local development can use the normal local backfill command: + +```bash +pnpm contrail backfill +``` + +For a substantial D1 production deployment, do not run a long bulk load through Wrangler's remote development proxy. Prefer a fresh generation: + +1. capture the ordered-source replay boundary; +2. build canonical state in native SQLite; +3. leave unavailable accounts visibly pending, retrying, or failed; +4. catch up through the ordered source; +5. import canonical tables into a fresh D1 database; +6. rebuild FTS, relation counts, and other derived projections; +7. verify record/version consistency, status, discovery, and representative queries; +8. test the candidate through a non-production Worker; and +9. activate the matching Worker and D1 binding together. + +Keep the previous D1 generation available for rollback. Do not split percentage traffic between independent databases with different serving positions. + +## Provider checklist + +Before announcing an origin: + +- run public Lexicon drift checking and TypeScript typechecking; +- verify the manifest and Lexicon digests differ and both recompute correctly; +- verify every advertised method has a matching query or procedure Lexicon; +- verify protected methods reject missing, wrong-audience, and wrong-`lxm` tokens; +- verify feed actors and notify URIs are bound to the token issuer; +- verify browser CORS preflight with `Authorization`; +- verify `/status` contains no sensitive operational detail; +- verify `getCursor` reports the committed ordered-source position; and +- connect and compile an independent consumer project. diff --git a/docs/public-services/using.md b/docs/public-services/using.md new file mode 100644 index 0000000..47d811e --- /dev/null +++ b/docs/public-services/using.md @@ -0,0 +1,136 @@ +# Using a public Contrail service + +A public Contrail service is a typed, read-through API over public AT Protocol records. This guide uses `https://api.atmo.rsvp`; replace it with the provider you want to use. + +## Install and connect + +```bash +pnpm add @atcute/client @atcute/lexicons @atmo-dev/contrail +pnpx @atmo-dev/contrail connect https://api.atmo.rsvp +``` + +`connect` verifies the provider's contract and Lexicons, writes `contrail.lock.json`, and generates: + +```text +lex.config.js +src/contrail/ + index.ts + lexicons/ + types/ +``` + +The generated configuration contains everything needed to identify the service and generate its types: + +```js +export default { + contrail: { + endpoint: "https://api.atmo.rsvp", + serviceDid: "did:web:api.atmo.rsvp", + scope: "rpc?lxm=*&aud=did:web:api.atmo.rsvp", + collections: [ + "app.bsky.actor.profile", + "app.bsky.graph.follow", + "community.lexicon.calendar.event", + "community.lexicon.calendar.rsvp", + ], + }, + generate: { + files: ["src/contrail/lexicons/**/*.json"], + outdir: "src/contrail/types/", + }, +}; +``` + +For JavaScript output, choose a `.js` client path: + +```bash +pnpx @atmo-dev/contrail connect https://api.atmo.rsvp \ + --client src/contrail/index.js +``` + +Commit `contrail.lock.json` and the generated files. + +## Query anonymous methods + +```ts +import { contrail } from "./contrail/index.js"; + +const response = await contrail.get("rsvp.atmo.event.listRecords", { + params: { + limit: 20, + sort: "startsAt", + order: "asc", + profiles: true, + }, +}); + +if (!response.ok) { + throw new Error(`Contrail query failed: ${response.status}`); +} + +for (const event of response.data.records) { + console.log(event.value.name, event.value.startsAt); +} +``` + +The generated Lexicons provide typed method names, parameters, and responses. + +## Use one authenticated client + +Add the provider's generated scope to the application's OAuth scopes: + +```ts +import { contrail } from "./contrail/index.js"; + +export const scopes = ["atproto", contrail.scope]; +``` + +After login, combine Contrail with the existing authenticated AT Protocol client: + +```ts +const client = contrail.authenticated(authenticatedClient, { + onNotificationError(error, { uris }) { + console.warn("Contrail notification failed", uris, error); + }, +}); + +const response = await client.get("rsvp.atmo.getFeed", { + params: { + feed: "network", + actor: signedInDid, + collection: "community.lexicon.calendar.event", + profiles: true, + limit: 20, + }, +}); +``` + +The same client still handles ordinary PDS calls. Successful writes to connected collections automatically notify Contrail: + +```ts +await client.post("com.atproto.repo.createRecord", { + input: { + repo: signedInDid, + collection: "community.lexicon.calendar.event", + record: event, + }, +}); +``` + +Contrail returns the original PDS response. A notification failure is reported through `onNotificationError` but never turns a committed PDS write into a failed write. The login session remains application-owned. + +## Update the connection + +Remote API changes are never accepted silently. Update deliberately: + +```bash +pnpx @atmo-dev/contrail connect https://api.atmo.rsvp --update +``` + +Review changes to `contrail.lock.json`, `lex.config.js`, and `src/contrail/`, then run the application's typecheck and tests. + +## Completeness + +A public Contrail service is a shared, possibly incomplete read-through cache. Reads may fetch missing public records or profiles, but an empty result does not prove that no matching record exists on the network. + +Applications still authenticate users and publish records through their PDS. Contrail service auth only authorizes the protected methods advertised by that service. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 6fd56c2..3507170 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -89,7 +89,7 @@ pnpm contrail lexicons generate pnpm contrail lexicons check ``` -`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only methods advertised by the anonymous read surface. Generated `lex.config.js` files carry an ownership marker; existing user-owned Atcute configuration is never replaced. Pass `--no-atcute-config` to manage that file yourself. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. +`contrail lexicons all` also pulls referenced record Lexicons and runs Atcute TypeScript generation. Use `--public` when generating only methods advertised by the remote service contract, including explicitly protected service-auth methods. Generated `lex.config.js` files carry an ownership marker; existing user-owned Atcute configuration is never replaced. Pass `--no-atcute-config` to manage that file yourself. The generator is also exported from `@atmo-dev/contrail/lexicons` for programmatic use. ## Public read-through service @@ -100,7 +100,22 @@ export default createWorker(config, { }); ``` -Discovery at `/.well-known/contrail` advertises a canonical contract digest and a content-addressed Lexicon bundle. Collection reads, profiles, feeds, and authored custom queries remain anonymous read-through operations: they may acquire public AT Protocol data and improve the cache behind the response. `notifyOfUpdate` remains separately controlled by `config.notify` and is not part of the anonymous contract. +Discovery at `/.well-known/contrail` advertises a canonical contract digest and a content-addressed Lexicon bundle. Anonymous collection reads, profiles, feeds, and authored custom queries may acquire public AT Protocol data and improve the cache behind the response. + +Personalized feeds and `notifyOfUpdate` can instead require method-bound AT Protocol service tokens: + +```ts +const config = { + notify: true, + serviceAuth: { + audience: "did:web:api.example.com", + methods: ["getFeed", "notifyOfUpdate"], + }, + // ... +}; +``` + +The protected contract advertises the audience and each query/procedure separately from anonymous methods. `contrail connect` generates `src/contrail/index.ts`; its exported `contrail.scope` is the provider's verified OAuth permission, such as `rpc?lxm=*&aud=did:web:api.example.com`. `contrail.authenticated(authenticatedClient)` returns one Atcute client: advertised provider methods route to Contrail, ordinary methods route to the PDS, and successful tracked `createRecord`, `putRecord`, and `deleteRecord` calls automatically notify Contrail. Protected methods use cached exact method-bound tokens. Feed actors must resolve to the token issuer, and every notified AT URI must belong to the issuer. When the audience matches the public endpoint's `did:web`, the Worker publishes its service document at `/.well-known/did.json`. Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Existing non-public deployments without one retain the legacy ingestion-time cursor response: @@ -118,6 +133,8 @@ The returned cursor is opaque. Compare the complete `{ source, epoch, cursor }` Connect an independent consumer with `contrail connect `. A repeated connection to the same endpoint and provider-owned output root requires `--update`; provider files and the lock are staged and swapped without deleting consumer-owned Lexicons. Switching providers or output roots requires removing the existing connection deliberately, so stale Lexicons cannot remain under a broad generator glob. +See [Creating a public service](../../docs/public-services/creating.md), [Using a public service](../../docs/public-services/using.md), and [Example: api.atmo.rsvp](../../docs/public-services/api-atmo-rsvp.md) for complete provider and consumer walkthroughs. + ## Runtime record validation Pass the record Lexicons for every configured collection and their transitive references to enable shared strict validation and CID verification: -- 2.51.2 From 17575cf49236e9525d15c531039fe6efd5beda6a Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:57:06 +0200 Subject: [PATCH 15/15] Address public service review feedback --- README.md | 2 + docs/public-services/api-atmo-rsvp.md | 11 ++- docs/public-services/creating.md | 2 + docs/public-services/using.md | 6 +- packages/contrail/README.md | 2 +- packages/contrail/src/cli/commands/connect.ts | 2 +- packages/contrail/src/core/service-auth.ts | 63 ++++++++++++- packages/contrail/src/public-client.ts | 94 +++++++++++++------ packages/contrail/tests/connect.test.ts | 7 +- packages/contrail/tests/public-client.test.ts | 84 ++++++++++++++++- packages/contrail/tests/service-auth.test.ts | 24 +++++ 11 files changed, 256 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 85c9774..98ee630 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ Consumers connect and generate Atcute types with one command: pnpx @atmo-dev/contrail connect https://api.example.com ``` +The generated client pins and verifies the discovered contract digest before its first provider request; transient discovery failures can retry, while endpoint, service-DID, and contract mismatches fail closed. + Public-service mode requires `orderedSource`; `getCursor` then returns the committed opaque `{ source, epoch, cursor }` position of that primary source. Compare complete positions for equality only; a source or epoch change requires a full client refetch. To avoid racing ingestion, read a position before and after a query and accept the query snapshot only when both positions match. Existing non-public deployments without `orderedSource` retain the legacy `time_us`, `date`, and `seconds_ago` response. ## Other databases diff --git a/docs/public-services/api-atmo-rsvp.md b/docs/public-services/api-atmo-rsvp.md index f6d2183..ff3fb57 100644 --- a/docs/public-services/api-atmo-rsvp.md +++ b/docs/public-services/api-atmo-rsvp.md @@ -172,13 +172,18 @@ The service is intentionally a shared, possibly incomplete cache. Unavailable id This deployment publishes verified API and record Lexicons for discovery and TypeScript generation, but does not enable Contrail's optional runtime record/CID validation during ingestion. -That distinction is deliberate: +That distinction is deliberate. A matched benchmark using the current calendar Lexicons rejected 12,291 historical records, reducing indexed events from roughly 14,600 to 4,800 and RSVPs from roughly 6,300 to 3,800. Those records include historical shapes that predate the current published definitions, so enabling latest-schema validation would silently discard most of the useful archive. + +Under the compatibility policy: - provider and consumer contracts are still canonical and digest-verified; -- generated Atcute types still describe response values; -- startup still rejects an inconsistent advertised API; but +- generated Atcute types describe the current expected response values, but are not a runtime guarantee for every historical record; +- startup still rejects an inconsistent advertised API; +- records and CIDs are fetched from authoritative sources rather than accepted from callers; but - ingestion does not reject records based on runtime Lexicon or canonical-CID checks. +A future strict deployment needs version-aware historical schemas or an explicitly looser response value, rather than pretending the compatibility loss does not exist. + ## Ordered source position The primary ordered source is one pinned Jetstream endpoint with an operator-owned continuity epoch: diff --git a/docs/public-services/creating.md b/docs/public-services/creating.md index b2e7264..3631480 100644 --- a/docs/public-services/creating.md +++ b/docs/public-services/creating.md @@ -150,6 +150,8 @@ The provider verifies: For protected feeds, the requested actor must resolve to the token issuer. For `notifyOfUpdate`, every submitted AT URI must belong to the token issuer. Notify still fetches the current authoritative record from that issuer's PDS; callers never submit a record body for Contrail to trust. +The default PLC/`did:web` resolver keeps a bounded five-minute in-process cache and deduplicates concurrent lookups. Signature failure forces an uncached refresh so key rotation does not remain hidden behind a stale entry. Deployments can still provide their own resolver policy. + The authenticated methods are listed separately from anonymous methods in discovery. Their query or procedure Lexicons remain in the provider bundle, so consumers still get generated types. ### OAuth permission versus token binding diff --git a/docs/public-services/using.md b/docs/public-services/using.md index 47d811e..0c41759 100644 --- a/docs/public-services/using.md +++ b/docs/public-services/using.md @@ -117,17 +117,17 @@ await client.post("com.atproto.repo.createRecord", { }); ``` -Contrail returns the original PDS response. A notification failure is reported through `onNotificationError` but never turns a committed PDS write into a failed write. The login session remains application-owned. +Contrail returns the original PDS response. A notification failure is reported through `onNotificationError` but never turns a committed PDS write into a failed write. Handle-form `deleteRecord` inputs are resolved through the PDS to construct the canonical DID record URI before notification. The login session remains application-owned. ## Update the connection -Remote API changes are never accepted silently. Update deliberately: +The generated client pins the lock's contract digest and verifies discovery before its first provider request. Transient discovery failures remain retryable; endpoint, service-DID, and contract mismatches fail closed. Update a changed contract deliberately at the same provider endpoint: ```bash pnpx @atmo-dev/contrail connect https://api.atmo.rsvp --update ``` -Review changes to `contrail.lock.json`, `lex.config.js`, and `src/contrail/`, then run the application's typecheck and tests. +Review changes to `contrail.lock.json`, `lex.config.js`, and `src/contrail/`, then run the application's typecheck and tests. `--update` cannot repoint an existing lock or abandon its provider-owned Lexicon root; remove the existing connection deliberately before switching providers or output roots. ## Completeness diff --git a/packages/contrail/README.md b/packages/contrail/README.md index 3507170..56f2b32 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -115,7 +115,7 @@ const config = { }; ``` -The protected contract advertises the audience and each query/procedure separately from anonymous methods. `contrail connect` generates `src/contrail/index.ts`; its exported `contrail.scope` is the provider's verified OAuth permission, such as `rpc?lxm=*&aud=did:web:api.example.com`. `contrail.authenticated(authenticatedClient)` returns one Atcute client: advertised provider methods route to Contrail, ordinary methods route to the PDS, and successful tracked `createRecord`, `putRecord`, and `deleteRecord` calls automatically notify Contrail. Protected methods use cached exact method-bound tokens. Feed actors must resolve to the token issuer, and every notified AT URI must belong to the issuer. When the audience matches the public endpoint's `did:web`, the Worker publishes its service document at `/.well-known/did.json`. +The protected contract advertises the audience and each query/procedure separately from anonymous methods. `contrail connect` generates `src/contrail/index.ts`; the module pins the discovered contract digest and verifies it once at runtime before sending provider requests. Its exported `contrail.scope` is the provider's verified OAuth permission, such as `rpc?lxm=*&aud=did:web:api.example.com`. `contrail.authenticated(authenticatedClient)` returns one Atcute client: advertised provider methods route to Contrail, ordinary methods route to the PDS, and successful tracked `createRecord`, `putRecord`, and `deleteRecord` calls automatically notify Contrail. Handle-form deletes are resolved to canonical DID URIs. Protected methods use cached exact method-bound tokens, while transient discovery failures remain retryable. Feed actors must resolve to the token issuer, and every notified AT URI must belong to the issuer. When the audience matches the public endpoint's `did:web`, the Worker publishes its service document at `/.well-known/did.json`. Public-service mode requires a primary ordered source so `getCursor` can expose its committed position. Existing non-public deployments without one retain the legacy ingestion-time cursor response: diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index 1397534..5bb1a60 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -285,7 +285,7 @@ export async function ensureConsumerClientModule(options: { const notifyMethod = protectedMethods.find( (method) => method === `${options.lock.namespace}.notifyOfUpdate`, ); - const source = `${GENERATED_CLIENT_HEADER}import { createPublicServiceClient } from "@atmo-dev/contrail/client";\n${generatedImport}\nexport const contrail = createPublicServiceClient({\n endpoint: ${JSON.stringify(options.lock.endpoint)},${serviceDid ? `\n serviceDid: ${JSON.stringify(serviceDid)},\n scope: ${JSON.stringify(scope)},` : ""}\n serviceMethods: ${formatStringArray(serviceMethods, 2)},\n collections: ${formatStringArray(options.lock.collections, 2)},${notifyMethod ? `\n notifyMethod: ${JSON.stringify(notifyMethod)},` : ""}\n});\n`; + const source = `${GENERATED_CLIENT_HEADER}import { createPublicServiceClient } from "@atmo-dev/contrail/client";\n${generatedImport}\nexport const contrail = createPublicServiceClient({\n endpoint: ${JSON.stringify(options.lock.endpoint)},\n contractDigest: ${JSON.stringify(options.lock.contractDigest)},${serviceDid ? `\n serviceDid: ${JSON.stringify(serviceDid)},\n scope: ${JSON.stringify(scope)},` : ""}\n serviceMethods: ${formatStringArray(serviceMethods, 2)},\n collections: ${formatStringArray(options.lock.collections, 2)},${notifyMethod ? `\n notifyMethod: ${JSON.stringify(notifyMethod)},` : ""}\n});\n`; await mkdir(dirname(path), { recursive: true }); if (await exists(path)) { diff --git a/packages/contrail/src/core/service-auth.ts b/packages/contrail/src/core/service-auth.ts index 0eec5bc..82d9b85 100644 --- a/packages/contrail/src/core/service-auth.ts +++ b/packages/contrail/src/core/service-auth.ts @@ -10,6 +10,59 @@ import { XRPCError } from "@atcute/xrpc-server"; import type { AtprotoServiceAuthMethod, ContrailConfig } from "./types.js"; const AUTH_TIMEOUT_MS = 5_000; +const DID_CACHE_TTL_MS = 5 * 60_000; +const DID_CACHE_MAX = 1_000; + +type DidDocument = Awaited>; + +/** Add bounded success caching and in-flight deduplication while preserving the + * resolver's `noCache` escape hatch for signing-key rotation retries. */ +export function createCachedDidDocumentResolver( + resolver: DidDocumentResolver, + options: { ttlMs?: number; maxEntries?: number } = {}, +): DidDocumentResolver { + const ttlMs = options.ttlMs ?? DID_CACHE_TTL_MS; + const maxEntries = options.maxEntries ?? DID_CACHE_MAX; + const cache = new Map(); + const pending = new Map>(); + + const remember = (did: string, value: DidDocument) => { + cache.delete(did); + cache.set(did, { value, expiresAt: Date.now() + ttlMs }); + while (cache.size > maxEntries) { + const oldest = cache.keys().next().value; + if (oldest === undefined) break; + cache.delete(oldest); + } + }; + + return { + async resolve(did, resolveOptions) { + const key = String(did); + if (!resolveOptions?.noCache) { + const cached = cache.get(key); + if (cached && cached.expiresAt > Date.now()) { + cache.delete(key); + cache.set(key, cached); + return cached.value; + } + if (cached) cache.delete(key); + const inflight = pending.get(key); + if (inflight) return inflight; + } + + const request = resolver.resolve(did, resolveOptions); + if (!resolveOptions?.noCache) pending.set(key, request); + try { + const value = await request; + remember(key, value); + return value; + } finally { + if (pending.get(key) === request) pending.delete(key); + } + }, + }; +} export interface ServiceAuthResult { principal?: VerifiedJwt; @@ -22,13 +75,17 @@ export interface ServiceAuthGate { authorize(request: Request, method: Nsid): Promise; } -function defaultResolver(): DidDocumentResolver { - return new CompositeDidDocumentResolver({ +const DEFAULT_RESOLVER = createCachedDidDocumentResolver( + new CompositeDidDocumentResolver({ methods: { plc: new PlcDidDocumentResolver(), web: new WebDidDocumentResolver(), }, - }); + }), +); + +function defaultResolver(): DidDocumentResolver { + return DEFAULT_RESOLVER; } /** Create the shared verifier used by protected built-in routes. Tokens remain diff --git a/packages/contrail/src/public-client.ts b/packages/contrail/src/public-client.ts index af41414..a12605d 100644 --- a/packages/contrail/src/public-client.ts +++ b/packages/contrail/src/public-client.ts @@ -4,7 +4,7 @@ import { simpleFetchHandler, type FetchHandler, } from "@atcute/client"; -import type { Did, Nsid } from "@atcute/lexicons/syntax"; +import { isDid, type Did, type Nsid } from "@atcute/lexicons/syntax"; import { isPublicServiceManifest, normalizePublicServiceEndpoint, @@ -19,6 +19,8 @@ interface CachedToken { expiresAt: number; } +class PublicServiceContractError extends Error {} + export interface PublicServiceClientOptions { /** Canonical public Contrail HTTPS origin. */ endpoint: string; @@ -117,9 +119,10 @@ function withBearer(init: RequestInit, token: string): RequestInit { return { ...init, headers }; } -/** Fetch handler that keeps anonymous reads cheap while automatically minting, - * caching, and attaching method-bound AT Protocol service tokens after a - * protected route challenges the first request. */ +/** Fetch handler that verifies an optional pinned contract once, then keeps + * unpinned anonymous reads cheap while automatically minting, caching, and + * attaching method-bound AT Protocol service tokens after a protected route + * challenges the first request. */ export function publicServiceFetchHandler( options: PublicServiceClientOptions, ): FetchHandler { @@ -132,7 +135,7 @@ export function publicServiceFetchHandler( const discoverServiceAuth = () => { if (serviceAuthPromise) return serviceAuthPromise; - serviceAuthPromise = (async () => { + const pending = (async () => { const response = await fetcher(`${endpoint}/.well-known/contrail`, { signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS), }); @@ -140,20 +143,26 @@ export function publicServiceFetchHandler( throw new Error(`Contrail discovery failed: ${response.status}`); } if (response.url && new URL(response.url).origin !== endpoint) { - throw new Error("Contrail discovery redirected to a different origin"); + throw new PublicServiceContractError( + "Contrail discovery redirected to a different origin", + ); } const value: unknown = await response.json(); if (!isPublicServiceManifest(value)) { - throw new Error("response is not a supported Contrail service manifest"); + throw new PublicServiceContractError( + "response is not a supported Contrail service manifest", + ); } if (normalizePublicServiceEndpoint(value.endpoint) !== endpoint) { - throw new Error("Contrail manifest endpoint mismatch"); + throw new PublicServiceContractError( + "Contrail manifest endpoint mismatch", + ); } if ( options.contractDigest && value.contract.digest !== options.contractDigest ) { - throw new Error( + throw new PublicServiceContractError( `Contrail contract digest mismatch: expected ${options.contractDigest}, received ${value.contract.digest}`, ); } @@ -162,13 +171,23 @@ export function publicServiceFetchHandler( options.serviceDid && serviceAuth?.audience !== options.serviceDid ) { - throw new Error( + throw new PublicServiceContractError( `Contrail service DID mismatch: expected ${options.serviceDid}, received ${serviceAuth?.audience ?? "none"}`, ); } return serviceAuth; })(); - return serviceAuthPromise; + const cached = pending.catch((error: unknown) => { + if ( + !(error instanceof PublicServiceContractError) && + serviceAuthPromise === cached + ) { + serviceAuthPromise = null; + } + throw error; + }); + serviceAuthPromise = cached; + return cached; }; const protectedMethod = async (method: string) => { @@ -226,10 +245,14 @@ export function publicServiceFetchHandler( return async (pathname, init) => { const method = xrpcMethod(pathname); - if (!method || !options.authenticatedClient) return base(pathname, init); + if (!method) return base(pathname, init); + // A generated lock pin is a runtime contract: verify it once even when the + // first operation is anonymous. Unpinned clients remain challenge-driven. + if (options.contractDigest) await discoverServiceAuth(); + if (!options.authenticatedClient) return base(pathname, init); // Once discovery has been loaded, avoid the initial challenge on subsequent - // protected calls. Anonymous calls never wait for discovery. + // protected calls. if (serviceAuthPromise) { const auth = await protectedMethod(method); if (auth) { @@ -286,12 +309,13 @@ function objectValue(value: unknown): Record | null { : null; } -function writtenRecordUri( +async function writtenRecordUri( method: string, request: UntypedRequestOptions | undefined, response: UntypedClientResponse, collections: ReadonlySet, -): string | null { + resolveHandle: (handle: string) => Promise, +): Promise { if (!response.ok || !NOTIFIED_WRITE_METHODS.has(method)) return null; const input = objectValue(request?.input); const collection = input?.collection; @@ -309,11 +333,9 @@ function writtenRecordUri( const repo = input?.repo; const rkey = input?.rkey; - return typeof repo === "string" && - repo.startsWith("did:") && - typeof rkey === "string" - ? `at://${repo}/${collection}/${rkey}` - : null; + if (typeof repo !== "string" || typeof rkey !== "string") return null; + const did = isDid(repo) ? repo : await resolveHandle(repo); + return `at://${did}/${collection}/${rkey}`; } function schemaNsid(schema: unknown): string | null { @@ -406,6 +428,17 @@ function createClient( } }; + const resolveHandle = async (handle: string): Promise => { + const response = await pdsGet("com.atproto.identity.resolveHandle", { + params: { handle }, + }); + const did = objectValue(response.data)?.did; + if (!response.ok || typeof did !== "string" || !isDid(did)) { + throw new Error(`Could not resolve deleted record repo ${handle}`); + } + return did; + }; + const notifyWrite = async (method: string, uri: string) => { if (!options.notifyMethod) return; try { @@ -439,13 +472,20 @@ function createClient( value: (async (name: string, request?: UntypedRequestOptions) => { if (serviceMethods.has(name)) return servicePost(name, request); const response = await pdsPost(name, request); - const uri = writtenRecordUri( - name, - request, - response, - trackedCollections, - ); - if (uri) await notifyWrite(name, uri); + if (options.notifyMethod) { + try { + const uri = await writtenRecordUri( + name, + request, + response, + trackedCollections, + resolveHandle, + ); + if (uri) await notifyWrite(name, uri); + } catch (error) { + reportNotificationError(error, name, []); + } + } return response; }) as Client["post"], }, diff --git a/packages/contrail/tests/connect.test.ts b/packages/contrail/tests/connect.test.ts index a3a9781..57275c9 100644 --- a/packages/contrail/tests/connect.test.ts +++ b/packages/contrail/tests/connect.test.ts @@ -185,6 +185,9 @@ describe("contrail connect", () => { 'import type {} from "./types/index.js"', ); expect(source).toContain(`endpoint: ${JSON.stringify(endpoint)}`); + expect(source).toContain( + `contractDigest: ${JSON.stringify(lock.contractDigest)}`, + ); expect(source).toContain( 'serviceDid: "did:web:api.atmo.rsvp"', ); @@ -296,7 +299,7 @@ describe("contrail connect", () => { connectPublicService({ endpoint, root, - out: "lexicons/pulled", + out: "src/contrail/lexicons", lock: "contrail.lock.json", fetcher, update: true, @@ -309,7 +312,7 @@ describe("contrail connect", () => { connectPublicService({ endpoint, root, - out: "different-lexicons", + out: "src/different-lexicons", lock: "contrail.lock.json", fetcher, update: true, diff --git a/packages/contrail/tests/public-client.test.ts b/packages/contrail/tests/public-client.test.ts index 7cae200..b81f192 100644 --- a/packages/contrail/tests/public-client.test.ts +++ b/packages/contrail/tests/public-client.test.ts @@ -75,6 +75,57 @@ describe("public service client", () => { expect(fetcher).toHaveBeenCalledTimes(1); }); + it("verifies a pinned contract once before anonymous requests", async () => { + const fetcher = vi.fn(async (input: RequestInfo | URL) => + String(input).endsWith("/.well-known/contrail") + ? Response.json(manifest()) + : Response.json({ records: [] }), + ); + const handler = publicServiceFetchHandler({ + endpoint, + contractDigest: digest, + fetch: fetcher, + }); + + expect( + (await handler("/xrpc/com.example.getCursor", { method: "get" })).status, + ).toBe(200); + expect( + (await handler("/xrpc/com.example.getCursor", { method: "get" })).status, + ).toBe(200); + expect( + fetcher.mock.calls.filter(([input]) => + String(input).endsWith("/.well-known/contrail"), + ), + ).toHaveLength(1); + }); + + it("retries transient discovery failures", async () => { + let discoveries = 0; + const fetcher = vi.fn(async (input: RequestInfo | URL) => { + if (String(input).endsWith("/.well-known/contrail")) { + discoveries++; + return discoveries === 1 + ? new Response(null, { status: 503 }) + : Response.json(manifest()); + } + return Response.json({ records: [] }); + }); + const handler = publicServiceFetchHandler({ + endpoint, + contractDigest: digest, + fetch: fetcher, + }); + + await expect( + handler("/xrpc/com.example.getCursor", { method: "get" }), + ).rejects.toThrow("discovery failed: 503"); + expect( + (await handler("/xrpc/com.example.getCursor", { method: "get" })).status, + ).toBe(200); + expect(discoveries).toBe(2); + }); + it("discovers, mints, caches, and attaches method-bound tokens", async () => { const jwt = token(); const pds = authenticatedClient(jwt); @@ -122,7 +173,7 @@ describe("public service client", () => { requests.filter((request) => request.url.includes(`/xrpc/${method}`), ).map((request) => request.authorization), - ).toEqual([null, `Bearer ${jwt}`, `Bearer ${jwt}`]); + ).toEqual([`Bearer ${jwt}`, `Bearer ${jwt}`]); }); it("routes PDS writes and service methods through one authenticated client", async () => { @@ -142,6 +193,9 @@ describe("public service client", () => { if (url.pathname === "/xrpc/com.atproto.server.getServiceAuth") { return Response.json({ token: jwt }); } + if (url.pathname === "/xrpc/com.atproto.identity.resolveHandle") { + return Response.json({ did: "did:plc:test" }); + } if (url.pathname === "/xrpc/com.atproto.repo.getRecord") { return Response.json({ uri: `at://did:plc:test/${collection}/3test`, @@ -256,6 +310,30 @@ describe("public service client", () => { ).length, ).toBe(notificationsBeforeDelete + 1); + const notificationsBeforeHandleDelete = serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length; + const deletedByHandle = await client.post("com.atproto.repo.deleteRecord", { + input: { + repo: "alice.example.com", + collection, + rkey: "3test", + }, + }); + expect(deletedByHandle.ok).toBe(true); + expect( + serviceRequests.filter(({ url }) => + url.endsWith(`/xrpc/${notifyMethod}`), + ).length, + ).toBe(notificationsBeforeHandleDelete + 1); + expect( + serviceRequests.some( + ({ url, body }) => + url.endsWith(`/xrpc/${notifyMethod}`) && + body?.includes(`at://did:plc:test/${collection}/3test`), + ), + ).toBe(true); + pdsWriteShouldFail = true; const notificationsBeforeFailedWrite = serviceRequests.filter(({ url }) => url.endsWith(`/xrpc/${notifyMethod}`), @@ -342,6 +420,10 @@ describe("public service client", () => { await expect( handler(`/xrpc/${method}`, { method: "get" }), ).rejects.toThrow("contract digest mismatch"); + await expect( + handler(`/xrpc/${method}`, { method: "get" }), + ).rejects.toThrow("contract digest mismatch"); + expect(fetcher).toHaveBeenCalledTimes(1); expect(pds.handler).not.toHaveBeenCalled(); }); }); diff --git a/packages/contrail/tests/service-auth.test.ts b/packages/contrail/tests/service-auth.test.ts index 8550f30..cd63f49 100644 --- a/packages/contrail/tests/service-auth.test.ts +++ b/packages/contrail/tests/service-auth.test.ts @@ -4,6 +4,7 @@ import { createServiceJwt } from "@atcute/xrpc-server/auth"; import { beforeAll, describe, expect, it } from "vitest"; import { createSqliteDatabase } from "../src/adapters/sqlite"; import { createApp } from "../src/core/router"; +import { createCachedDidDocumentResolver } from "../src/core/service-auth"; import { initSchema, resolveConfig, @@ -146,6 +147,29 @@ describe("AT Protocol service auth", () => { expect(forbidden.status).toBe(403); }); + it("caches DID documents and honors signing-key refreshes", async () => { + let calls = 0; + const resolver = createCachedDidDocumentResolver({ + async resolve(did) { + calls++; + return { + "@context": [], + id: did, + verificationMethod: [], + }; + }, + }); + + await resolver.resolve(issuer); + await resolver.resolve(issuer); + expect(calls).toBe(1); + + await resolver.resolve(issuer, { noCache: true }); + expect(calls).toBe(2); + await resolver.resolve(issuer); + expect(calls).toBe(2); + }); + it("only lets an issuer notify its own record URIs", async () => { const { app } = await setup(); const jwt = await token("com.example.notifyOfUpdate");