diff --git a/.changeset/calm-winds-connect.md b/.changeset/calm-winds-connect.md new file mode 100644 index 0000000..1d9be89 --- /dev/null +++ b/.changeset/calm-winds-connect.md @@ -0,0 +1,5 @@ +--- +"@atmo-dev/contrail": patch +--- + +Allow config-backed `contrail dev` and `contrail connect` to continue after confirmation when configured Lexicons cannot yet be resolved, omitting incomplete schema closures and generating untyped values for affected records. diff --git a/packages/contrail/README.md b/packages/contrail/README.md index e30e3dd..859475b 100644 --- a/packages/contrail/README.md +++ b/packages/contrail/README.md @@ -176,7 +176,11 @@ contrail dev Run `contrail init` without a directory to write the config in the current directory. The command refuses to replace a config in any standard auto-detected location. -Without a Wrangler config this creates or resumes `.contrail/dev.sqlite`, resolves missing configured record/ref Lexicons from the AT Protocol network without overwriting project-owned schemas, runs PDS backfill, serves the complete public Contrail discovery/XRPC/Lexicon surface at `http://127.0.0.1:8787`, and runs bounded Jetstream ingestion every minute. It never creates or changes a deployment provider lock. Add `.contrail/` to the project ignore file. Existing Wrangler projects retain the prior D1 development behavior automatically. Useful SQLite options include: +Without a Wrangler config this creates or resumes `.contrail/dev.sqlite`, resolves missing configured record/ref Lexicons from the AT Protocol network without overwriting project-owned schemas, runs PDS backfill, serves the complete public Contrail discovery/XRPC/Lexicon surface at `http://127.0.0.1:8787`, and runs bounded Jetstream ingestion every minute. It never creates or changes a deployment provider lock. Add `.contrail/` to the project ignore file. Existing Wrangler projects retain the prior D1 development behavior automatically. + +When a configured Lexicon does not exist yet or cannot be resolved, interactive `contrail dev` and config-backed `contrail connect` warn and ask whether to continue with an incomplete local bundle. Unresolved documents and schemas that depend on them are omitted, and directly affected record values are generated as `unknown`; collections using `validate: true` still require their complete Lexicon closure. The prompt defaults to no, and non-interactive use fails closed unless `--yes` is passed. + +Useful SQLite options include: ```bash contrail dev --fresh # reset local SQLite first diff --git a/packages/contrail/src/cli/commands/connect.ts b/packages/contrail/src/cli/commands/connect.ts index 5f92304..286fb01 100644 --- a/packages/contrail/src/cli/commands/connect.ts +++ b/packages/contrail/src/cli/commands/connect.ts @@ -40,6 +40,7 @@ import { prepareDevLexicons, } from "../dev-lexicons.js"; import { generateLexiconTypesWithAtcute } from "../atcute.js"; +import { confirmUnresolvedLexicons } from "../shared.js"; const MAX_DISCOVERY_BYTES = 10 * 1024 * 1024; const REQUEST_TIMEOUT_MS = 15_000; @@ -63,6 +64,7 @@ interface ConnectOptions { skipClient?: boolean; update?: boolean; allowInsecureHttp?: boolean; + yes?: boolean; } export interface ProviderDefinition { @@ -482,6 +484,9 @@ export async function connectConfigSource(options: { out: string; lock?: string; endpoint?: string; + confirmUnresolvedLexicons?: ( + nsids: readonly string[], + ) => boolean | Promise; }): Promise<{ manifest: PublicServiceManifest; definition: ProviderDefinition; @@ -520,12 +525,13 @@ export async function connectConfigSource(options: { const outputRoot = resolveInsideRoot(projectRoot, options.out); const providerKey = "source"; const providerRoot = resolveInsideRoot(outputRoot, providerKey); - const lexicons = prepareDevLexicons( + const lexicons = await prepareDevLexicons( sourceConfig, projectRoot, workspaceRoot, providerRoot, configProjectRoot(configPath), + { confirmUnresolved: options.confirmUnresolvedLexicons }, ); const description = await describePublicService( sourceConfig, @@ -785,6 +791,10 @@ export function registerConnect(cli: CAC): void { "--no-generate", "Resolve the API without generating types or a client module", ) + .option( + "--yes, -y", + "Continue when configured Lexicons cannot be resolved", + ) .action(async (source: string, options: ConnectOptions) => { const endpoint = endpointSource(source); let api: ProviderDefinition; @@ -815,6 +825,8 @@ export function registerConnect(cli: CAC): void { root: options.root, out: options.out, lock: options.lock, + confirmUnresolvedLexicons: (nsids) => + confirmUnresolvedLexicons(nsids, options.yes === true), }); api = result.definition; target = result.target; diff --git a/packages/contrail/src/cli/commands/dev.ts b/packages/contrail/src/cli/commands/dev.ts index 1759c89..de43e57 100644 --- a/packages/contrail/src/cli/commands/dev.ts +++ b/packages/contrail/src/cli/commands/dev.ts @@ -20,6 +20,7 @@ import { prepareDevLexicons, } from "../dev-lexicons.js"; import { + confirmUnresolvedLexicons, promptYesNo, resolveConfig, resolveValidationLexicons, @@ -246,12 +247,16 @@ async function runSqliteDev( root, options.clientLexicons ?? defaultConsumerLexiconRoot(root), ); - const lexicons = prepareDevLexicons( + const lexicons = await prepareDevLexicons( config, root, lexiconWorkspace, clientLexiconRoot, sourceRoot, + { + confirmUnresolved: (nsids) => + confirmUnresolvedLexicons(nsids, options.yes === true), + }, ); const db = createSqliteDatabase(databasePath); const contrail = new Contrail({ ...config, db, lexicons }); @@ -398,7 +403,7 @@ export function registerDev(cli: CAC): void { .option("--concurrency ", "PDS backfill identity concurrency", { default: 100, }) - .option("--yes, -y", "Accept Wrangler backfill prompts") + .option("--yes, -y", "Accept confirmation prompts") .option("--alluvium", "Use Alluvium base + archive for SQLite bootstrap") .option("--alluvium-endpoint ", "Alluvium HTTP origin", { default: "https://alluvium-v0.atmo.tools", diff --git a/packages/contrail/src/cli/dev-lexicons.ts b/packages/contrail/src/cli/dev-lexicons.ts index 2af0f27..3e7d951 100644 --- a/packages/contrail/src/cli/dev-lexicons.ts +++ b/packages/contrail/src/cli/dev-lexicons.ts @@ -1,5 +1,11 @@ -import { existsSync, writeFileSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { + existsSync, + mkdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { isNsid } from "@atcute/lexicons/syntax"; import type { ContrailConfig } from "../core/types.js"; import { generateLexicons } from "../lexicons/generate.js"; import { pullLexiconsWithAtcute } from "./atcute.js"; @@ -10,9 +16,83 @@ export function defaultConsumerLexiconRoot(projectRoot: string): string { : "src/contrail/lexicons"; } +function referencedNsids(document: object): string[] { + const references = new Set(); + const visit = (value: unknown, key?: string) => { + if (typeof value === "string" && (key === "ref" || key === "refs")) { + const nsid = value.split("#", 1)[0]; + if (nsid?.includes(".")) references.add(nsid); + return; + } + if (Array.isArray(value)) { + for (const child of value) { + visit(child, key === "refs" ? "refs" : undefined); + } + } else if (value && typeof value === "object") { + for (const [childKey, child] of Object.entries(value)) { + visit(child, childKey); + } + } + }; + visit(document); + return [...references]; +} + +/** Stage only source documents whose complete external reference closure is + * available. Generated methods are rebuilt against this directory so values + * backed by an incomplete source schema become `unknown` rather than retaining + * a dangling reference that breaks downstream type generation. */ +function stageCompleteSourceLexicons( + workspaceRoot: string, + result: ReturnType, +): string { + const generatedIds = new Set(Object.keys(result.generated)); + const sources = new Map(); + for (const document of result.lexicons) { + const id = (document as { id?: unknown }).id; + if (typeof id === "string" && !generatedIds.has(id)) { + sources.set(id, document); + } + } + + const available = new Set([...generatedIds, ...sources.keys()]); + let changed = true; + while (changed) { + changed = false; + for (const [id, document] of sources) { + if (referencedNsids(document).some((nsid) => !available.has(nsid))) { + sources.delete(id); + available.delete(id); + changed = true; + } + } + } + + const directory = join(workspaceRoot, "dev-complete"); + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { recursive: true }); + for (const [id, document] of sources) { + if (!isNsid(id)) throw new Error(`Invalid Lexicon NSID: ${id}`); + const path = join(directory, ...id.split(".")) + ".json"; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(document, null, 2)}\n`); + } + return directory; +} + /** Resolve the exact public Lexicon bundle used by a config-backed local * service. Project-owned documents win over remotely pulled dependencies. */ -export function prepareDevLexicons( +export interface PrepareDevLexiconsOptions { + /** Ask whether preparation may continue with documents that could not be + * resolved. Omitted means fail closed. */ + confirmUnresolved?: ( + nsids: readonly string[], + ) => boolean | Promise; + /** @internal Test seam for network resolution. */ + pullLexicons?: typeof pullLexiconsWithAtcute; +} + +export async function prepareDevLexicons( config: ContrailConfig, projectRoot: string, workspaceRoot: string, @@ -21,7 +101,8 @@ export function prepareDevLexicons( defaultConsumerLexiconRoot(projectRoot), ), configRoot = projectRoot, -): object[] { + options: PrepareDevLexiconsOptions = {}, +): Promise { const pulled = join(workspaceRoot, "dev-pulled"); const roots = [...new Set([projectRoot, configRoot])]; const localSourceDirs = [ @@ -58,8 +139,20 @@ export function prepareDevLexicons( const current = JSON.stringify(missing); if (current === attempted) { + if (await options.confirmUnresolved?.(missing)) { + const complete = stageCompleteSourceLexicons(workspaceRoot, result); + return generateLexicons({ + config, + rootDir: workspaceRoot, + outputDir: output, + sourceDirs: [complete], + surface: "public", + writeAtcuteConfig: false, + quiet: true, + }).lexicons; + } throw new Error( - `Could not resolve required development Lexicons: ${missing.join(", ")}`, + `Could not resolve configured development Lexicons: ${missing.join(", ")}`, ); } attempted = current; @@ -84,7 +177,7 @@ export function prepareDevLexicons( )};\n`, ); console.log(`lexicons: resolving ${missing.join(", ")}`); - pullLexiconsWithAtcute(workspaceRoot, pullConfig); + (options.pullLexicons ?? pullLexiconsWithAtcute)(workspaceRoot, pullConfig); } throw new Error("Development Lexicon reference discovery did not converge"); } diff --git a/packages/contrail/src/cli/shared.ts b/packages/contrail/src/cli/shared.ts index 4ca8b12..3ca39ca 100644 --- a/packages/contrail/src/cli/shared.ts +++ b/packages/contrail/src/cli/shared.ts @@ -81,3 +81,20 @@ export async function promptYesNo( rl.close(); } } + +/** Warn before omitting unresolved documents from a temporary source bundle. */ +export async function confirmUnresolvedLexicons( + nsids: readonly string[], + autoYes: boolean, +): Promise { + console.warn("\nwarning: could not resolve these configured Lexicons:"); + for (const nsid of nsids) console.warn(` - ${nsid}`); + console.warn( + "Continuing omits them and schemas that depend on them from the local " + + "bundle. Directly affected record values will be generated as `unknown`.", + ); + console.warn( + "Collections with `validate: true` still cannot start without their complete Lexicon closure.", + ); + return promptYesNo("continue without these Lexicons?", false, autoYes); +} diff --git a/packages/contrail/tests/dev-lexicons.test.ts b/packages/contrail/tests/dev-lexicons.test.ts index 3f1c32b..0558a0e 100644 --- a/packages/contrail/tests/dev-lexicons.test.ts +++ b/packages/contrail/tests/dev-lexicons.test.ts @@ -32,7 +32,7 @@ function writeLexicon(root: string, nsid: string, document: object): void { } describe("contrail dev Lexicon preparation", () => { - it("reuses an unpublished source Lexicon from the stable Svelte consumer root", () => { + it("reuses an unpublished source Lexicon from the stable Svelte consumer root", async () => { const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); roots.push(root); const workspace = join(root, ".contrail"); @@ -62,7 +62,7 @@ describe("contrail dev Lexicon preparation", () => { review: { collection }, }, }; - const lexicons = prepareDevLexicons(config, root, workspace); + const lexicons = await prepareDevLexicons(config, root, workspace); const ids = lexicons.map( (document) => (document as { id?: string }).id, ); @@ -71,4 +71,153 @@ describe("contrail dev Lexicon preparation", () => { expect(ids).toContain("test.svelte.review.getRecord"); expect(ids).toContain("test.svelte.review.listRecords"); }); + + it("can continue with an unresolved collection Lexicon and generate unknown values", async () => { + const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); + roots.push(root); + const workspace = join(root, ".contrail"); + mkdirSync(workspace, { recursive: true }); + + const collection = "social.example.future.record"; + const config: ContrailConfig = { + namespace: "test.future", + profiles: [], + collections: { + future: { collection }, + }, + }; + const confirmed: string[][] = []; + const lexicons = await prepareDevLexicons( + config, + root, + workspace, + undefined, + undefined, + { + pullLexicons: () => {}, + confirmUnresolved: (nsids) => { + confirmed.push([...nsids]); + return true; + }, + }, + ); + + expect(confirmed).toEqual([[collection]]); + expect( + lexicons.some( + (document) => (document as { id?: string }).id === collection, + ), + ).toBe(false); + const list = lexicons.find( + (document) => + (document as { id?: string }).id === "test.future.future.listRecords", + ) as any; + expect(list.defs.record.properties.value).toEqual({ type: "unknown" }); + }); + + it("drops schemas with unresolved dependencies before generating the incomplete bundle", async () => { + const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); + roots.push(root); + const workspace = join(root, ".contrail"); + mkdirSync(workspace, { recursive: true }); + + const collection = "social.example.future.record"; + const dependency = "social.example.future.defs"; + const missing = "social.example.future.missing"; + writeLexicon(root, collection, { + lexicon: 1, + id: collection, + defs: { + main: { + type: "record", + key: "tid", + record: { + type: "object", + properties: { + subject: { type: "ref", ref: `${dependency}#main` }, + }, + }, + }, + }, + }); + writeLexicon(root, dependency, { + lexicon: 1, + id: dependency, + defs: { + main: { + type: "object", + properties: { + nested: { type: "ref", ref: `${missing}#main` }, + }, + }, + }, + }); + const confirmed: string[][] = []; + const lexicons = await prepareDevLexicons( + { + namespace: "test.future", + profiles: [], + collections: { future: { collection } }, + }, + root, + workspace, + undefined, + undefined, + { + pullLexicons: () => {}, + confirmUnresolved: (nsids) => { + confirmed.push([...nsids]); + return true; + }, + }, + ); + + expect(confirmed).toEqual([[missing]]); + expect( + lexicons.some((document) => { + const id = (document as { id?: string }).id; + return id === collection || id === dependency; + }), + ).toBe(false); + const list = lexicons.find( + (document) => + (document as { id?: string }).id === "test.future.future.listRecords", + ) as any; + expect(list.defs.record.properties.value).toEqual({ type: "unknown" }); + + const available = new Set( + lexicons.map((document) => (document as { id?: string }).id), + ); + expect( + JSON.stringify(lexicons).includes(`\"ref\":\"${missing}#main\"`), + ).toBe(false); + expect(available.has(missing)).toBe(false); + }); + + it("still fails closed when continuation is declined", async () => { + const root = mkdtempSync(join(tmpdir(), "contrail-dev-lexicons-")); + roots.push(root); + const workspace = join(root, ".contrail"); + mkdirSync(workspace, { recursive: true }); + + await expect( + prepareDevLexicons( + { + namespace: "test.future", + profiles: [], + collections: { + future: { collection: "social.example.future.record" }, + }, + }, + root, + workspace, + undefined, + undefined, + { + pullLexicons: () => {}, + confirmUnresolved: () => false, + }, + ), + ).rejects.toThrow("Could not resolve configured development Lexicons"); + }); });