diff --git a/.changeset/create-worker-lexicons-endpoint.md b/.changeset/create-worker-lexicons-endpoint.md new file mode 100644 index 0000000..7ae8ac3 --- /dev/null +++ b/.changeset/create-worker-lexicons-endpoint.md @@ -0,0 +1,32 @@ +--- +"@atmo-dev/contrail": minor +"@atmo-dev/contrail-lexicons": minor +--- + +two new DX pieces: + +**`@atmo-dev/contrail/worker`** exports `createWorker(config, options?)` — a prebuilt Cloudflare Workers entry that collapses the ~12-line `{ fetch, scheduled }` boilerplate to one line: + +```ts +import { createWorker } from "@atmo-dev/contrail/worker"; +import { config } from "./contrail.config"; +import { lexicons } from "../lexicons/generated"; + +export default createWorker(config, { lexicons }); +``` + +options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). + +**`/xrpc/.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: + +- `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. +- Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/.lexicons`. +- From a consumer app: + ```bash + contrail-lex pull-service https://my-contrail.dev/xrpc/com.example.lexicons + # or + contrail-lex pull-service https://my-contrail.dev --namespace com.example + ``` + Fetches the manifest, writes each lexicon under `lexicons/pulled/`. Then `npx lex-cli generate` emits TS types. + +Path 1 of 4 of a set of DX improvements — path 2 (consumer typegen) works end-to-end but assumes the operator has regenerated. Paths 3 (one-command deploy) and 4 (fully vendored worker) are deferred. diff --git a/README.md b/README.md index a360ae0..64d09f0 100644 --- a/README.md +++ b/README.md @@ -37,28 +37,18 @@ export const config: ContrailConfig = { }; ``` -**`src/worker.ts`** — serves XRPC requests, pulls new records every minute: +**`src/worker.ts`** — four lines. `createWorker` wires up fetch + scheduled + lazy init: ```ts -import { Contrail } from "@atmo-dev/contrail"; -import { createHandler } from "@atmo-dev/contrail/server"; +import { createWorker } from "@atmo-dev/contrail/worker"; import { config } from "./contrail.config"; +import { lexicons } from "../lexicons/generated"; -const contrail = new Contrail(config); -const handle = createHandler(contrail); -let ready = false; - -export default { - async fetch(req: Request, env: { DB: D1Database }) { - if (!ready) { await contrail.init(env.DB); ready = true; } // create tables once - return handle(req, env.DB); // xrpc routes - }, - async scheduled(_ev: ScheduledEvent, env: { DB: D1Database }, ctx: ExecutionContext) { - ctx.waitUntil(contrail.ingest({}, env.DB)); // pull new records - }, -}; +export default createWorker(config, { lexicons }); ``` +`lexicons/generated/` is produced by `contrail-lex generate`; passing `{ lexicons }` exposes them at `/lexicons` so consumer apps can typegen against your deployed service. Drop it if you don't need that. + and a d1 binding + cron in `wrangler.jsonc`: ```jsonc diff --git a/apps/cloudflare-workers/lexicons/generated/index.ts b/apps/cloudflare-workers/lexicons/generated/index.ts new file mode 100644 index 0000000..568ba45 --- /dev/null +++ b/apps/cloudflare-workers/lexicons/generated/index.ts @@ -0,0 +1,19 @@ +// Auto-generated by @atmo-dev/contrail-lexicons. Do not edit. +// Pass `lexicons` to `createWorker(config, { lexicons })` to expose them +// at `/xrpc/.lexicons` for consumer apps to typegen against. + +import _0 from "../pulled/app/bsky/actor/profile.json"; +import _1 from "../pulled/community/lexicon/calendar/event.json"; +import _2 from "../pulled/community/lexicon/location/address.json"; +import _3 from "../pulled/community/lexicon/location/fsq.json"; +import _4 from "../pulled/community/lexicon/location/geo.json"; +import _5 from "../pulled/community/lexicon/location/hthree.json"; +import _6 from "./com/example/event/getRecord.json"; +import _7 from "./com/example/event/listRecords.json"; +import _8 from "./com/example/getCursor.json"; +import _9 from "./com/example/getOverview.json"; +import _10 from "./com/example/getProfile.json"; +import _11 from "./com/example/notifyOfUpdate.json"; +import _12 from "./com/example/permissionSet.json"; + +export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12]; diff --git a/apps/cloudflare-workers/src/worker.ts b/apps/cloudflare-workers/src/worker.ts index 8d9a216..35003cf 100644 --- a/apps/cloudflare-workers/src/worker.ts +++ b/apps/cloudflare-workers/src/worker.ts @@ -1,17 +1,5 @@ -import { Contrail } from "@atmo-dev/contrail"; -import { createHandler } from "@atmo-dev/contrail/server"; +import { createWorker } from "@atmo-dev/contrail/worker"; import { config } from "./contrail.config"; +import { lexicons } from "../lexicons/generated"; -const contrail = new Contrail(config); -const handle = createHandler(contrail); -let ready = false; - -export default { - async fetch(req: Request, env: { DB: D1Database }) { - if (!ready) { await contrail.init(env.DB); ready = true; } // create tables once - return handle(req, env.DB); // xrpc routes - }, - async scheduled(_ev: ScheduledEvent, env: { DB: D1Database }, ctx: ExecutionContext) { - ctx.waitUntil(contrail.ingest({}, env.DB)); // pull new records - }, -}; +export default createWorker(config, { lexicons }); diff --git a/apps/group-chat/lexicons/generated/index.ts b/apps/group-chat/lexicons/generated/index.ts new file mode 100644 index 0000000..aaa46b9 --- /dev/null +++ b/apps/group-chat/lexicons/generated/index.ts @@ -0,0 +1,65 @@ +// Auto-generated by @atmo-dev/contrail-lexicons. Do not edit. +// Pass `lexicons` to `createWorker(config, { lexicons })` to expose them +// at `/xrpc/.lexicons` for consumer apps to typegen against. + +import _0 from "../custom/tools/atmo/chat/channel.json"; +import _1 from "../custom/tools/atmo/chat/message.json"; +import _2 from "../custom/tools/atmo/chat/server.json"; +import _3 from "../pulled/app/bsky/actor/profile.json"; +import _4 from "./tools/atmo/chat/channel/getRecord.json"; +import _5 from "./tools/atmo/chat/channel/listRecords.json"; +import _6 from "./tools/atmo/chat/channel/watchRecords.json"; +import _7 from "./tools/atmo/chat/community/adopt.json"; +import _8 from "./tools/atmo/chat/community/defs.json"; +import _9 from "./tools/atmo/chat/community/delete.json"; +import _10 from "./tools/atmo/chat/community/deleteRecord.json"; +import _11 from "./tools/atmo/chat/community/getHealth.json"; +import _12 from "./tools/atmo/chat/community/list.json"; +import _13 from "./tools/atmo/chat/community/mint.json"; +import _14 from "./tools/atmo/chat/community/putRecord.json"; +import _15 from "./tools/atmo/chat/community/reauth.json"; +import _16 from "./tools/atmo/chat/community/space/create.json"; +import _17 from "./tools/atmo/chat/community/space/delete.json"; +import _18 from "./tools/atmo/chat/community/space/deleteRecord.json"; +import _19 from "./tools/atmo/chat/community/space/grant.json"; +import _20 from "./tools/atmo/chat/community/space/listMembers.json"; +import _21 from "./tools/atmo/chat/community/space/putRecord.json"; +import _22 from "./tools/atmo/chat/community/space/resync.json"; +import _23 from "./tools/atmo/chat/community/space/revoke.json"; +import _24 from "./tools/atmo/chat/community/space/setAccessLevel.json"; +import _25 from "./tools/atmo/chat/getCursor.json"; +import _26 from "./tools/atmo/chat/getOverview.json"; +import _27 from "./tools/atmo/chat/getProfile.json"; +import _28 from "./tools/atmo/chat/invite/create.json"; +import _29 from "./tools/atmo/chat/invite/defs.json"; +import _30 from "./tools/atmo/chat/invite/list.json"; +import _31 from "./tools/atmo/chat/invite/redeem.json"; +import _32 from "./tools/atmo/chat/invite/revoke.json"; +import _33 from "./tools/atmo/chat/message/getRecord.json"; +import _34 from "./tools/atmo/chat/message/listRecords.json"; +import _35 from "./tools/atmo/chat/message/watchRecords.json"; +import _36 from "./tools/atmo/chat/notifyOfUpdate.json"; +import _37 from "./tools/atmo/chat/permissionSet.json"; +import _38 from "./tools/atmo/chat/realtime/subscribe.json"; +import _39 from "./tools/atmo/chat/realtime/ticket.json"; +import _40 from "./tools/atmo/chat/server/getRecord.json"; +import _41 from "./tools/atmo/chat/server/listRecords.json"; +import _42 from "./tools/atmo/chat/server/watchRecords.json"; +import _43 from "./tools/atmo/chat/space/addMember.json"; +import _44 from "./tools/atmo/chat/space/createSpace.json"; +import _45 from "./tools/atmo/chat/space/defs.json"; +import _46 from "./tools/atmo/chat/space/deleteRecord.json"; +import _47 from "./tools/atmo/chat/space/getBlob.json"; +import _48 from "./tools/atmo/chat/space/getRecord.json"; +import _49 from "./tools/atmo/chat/space/getSpace.json"; +import _50 from "./tools/atmo/chat/space/leaveSpace.json"; +import _51 from "./tools/atmo/chat/space/listBlobs.json"; +import _52 from "./tools/atmo/chat/space/listMembers.json"; +import _53 from "./tools/atmo/chat/space/listRecords.json"; +import _54 from "./tools/atmo/chat/space/listSpaces.json"; +import _55 from "./tools/atmo/chat/space/putRecord.json"; +import _56 from "./tools/atmo/chat/space/removeMember.json"; +import _57 from "./tools/atmo/chat/space/uploadBlob.json"; +import _58 from "./tools/atmo/chat/spaceExt/whoami.json"; + +export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58]; diff --git a/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts b/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts new file mode 100644 index 0000000..e1faba6 --- /dev/null +++ b/apps/sveltekit-cloudflare-workers/lexicons/generated/index.ts @@ -0,0 +1,15 @@ +// Auto-generated by @atmo-dev/contrail-lexicons. Do not edit. +// Pass `lexicons` to `createWorker(config, { lexicons })` to expose them +// at `/xrpc/.lexicons` for consumer apps to typegen against. + +import _0 from "../pulled/app/bsky/actor/profile.json"; +import _1 from "../pulled/xyz/statusphere/status.json"; +import _2 from "./statusphere/app/getCursor.json"; +import _3 from "./statusphere/app/getOverview.json"; +import _4 from "./statusphere/app/getProfile.json"; +import _5 from "./statusphere/app/notifyOfUpdate.json"; +import _6 from "./statusphere/app/permissionSet.json"; +import _7 from "./statusphere/app/status/getRecord.json"; +import _8 from "./statusphere/app/status/listRecords.json"; + +export const lexicons: object[] = [_0, _1, _2, _3, _4, _5, _6, _7, _8]; diff --git a/docs/03-lexicons.md b/docs/03-lexicons.md index 09cd550..d06c4e8 100644 --- a/docs/03-lexicons.md +++ b/docs/03-lexicons.md @@ -11,12 +11,13 @@ pnpm add -D @atmo-dev/contrail-lexicons @atcute/lex-cli ## CLI ```bash -contrail-lex generate # emit lexicon JSON from your Contrail config -contrail-lex pull # wraps `lex-cli pull` (fetch external lexicons) -contrail-lex types # wraps `lex-cli generate` (JSON → TS types) -contrail-lex all # generate → pull → generate → pull → types -contrail-lex all --no-types # same, skip the type step -contrail-lex publish # publish lexicons to your PDS (add --dry-run to preview) +contrail-lex generate # emit lexicon JSON from your Contrail config +contrail-lex pull # wraps `lex-cli pull` (fetch external lexicons) +contrail-lex types # wraps `lex-cli generate` (JSON → TS types) +contrail-lex all # generate → pull → generate → pull → types +contrail-lex all --no-types # same, skip the type step +contrail-lex publish # publish lexicons to your PDS (add --dry-run to preview) +contrail-lex pull-service # consume a deployed contrail's /lexicons endpoint ``` Config is auto-detected at `contrail.config.ts`, `src/contrail.config.ts`, `src/lib/contrail.config.ts`, or `app/contrail.config.ts`. Override with `--config `. @@ -36,6 +37,25 @@ src/lexicon-types/ # TS types from lex-cli generate `lexicons/custom/`, `lexicons/generated/`, and `lexicons/pulled/` should be **committed** — that way CI doesn't need network access. `lex.config.js` and `src/lexicon-types/` are regenerated on demand and safe to gitignore. +`lexicons/generated/index.ts` is also emitted on every run — a barrel that imports every lexicon the deployment speaks (generated + pulled + custom). Pass it to `createWorker(config, { lexicons })` to expose them at `/xrpc/.lexicons` on your deployed service; consumer apps can then `pull-service` against it. + +## Consuming a deployed contrail + +If you're building a frontend that talks to someone's (or your own) deployed contrail, you don't need the backend's source code to get typed XRPC calls. Have the operator pass `{ lexicons }` to `createWorker`, then: + +```bash +contrail-lex pull-service https://my-contrail.dev/xrpc/com.example.lexicons +# or: contrail-lex pull-service https://my-contrail.dev --namespace com.example + +npx lex-cli generate +``` + +`pull-service` hits the manifest endpoint, writes each lexicon under `lexicons/pulled/.json`, and `lex-cli generate` emits TypeScript types for `@atcute/client`. Override the output dir with `--out `. + +The manifest includes the service's generated lexicons *plus* any external NSIDs the generator `$ref`s (e.g., `app.bsky.actor.profile`, `community.lexicon.calendar.event`) — so typegen resolves cleanly with no additional fetching from bsky / atproto registries. + +No PDS setup required, no DNS, no published lexicon records — just an HTTP endpoint and typegen. + ## Publishing Once your lexicons are generated and committed, publish them as `com.atproto.lexicon.schema` records on your PDS so other apps can resolve them: diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 7253821..39493dd 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -31,6 +31,10 @@ "types": "./dist/workers/backfill.d.ts", "import": "./dist/workers/backfill.js" }, + "./worker": { + "types": "./dist/worker/index.d.ts", + "import": "./dist/worker/index.js" + }, "./cli-config": { "types": "./dist/cli-config.d.ts", "import": "./dist/cli-config.js" diff --git a/packages/contrail/src/cli-config.ts b/packages/contrail/src/cli-config.ts index 6482319..c422d08 100644 --- a/packages/contrail/src/cli-config.ts +++ b/packages/contrail/src/cli-config.ts @@ -46,7 +46,9 @@ export function findConfigFile(root: string, explicit?: string): string | null { /** Load a config file via jiti — handles TS + ESM + CJS transparently, * no tsx/ts-node hook required. Accepts either a named export `config` or - * a default export. Throws if neither is present. */ + * a default export. Validates the result has the minimum `ContrailConfig` + * shape (`namespace` + `collections`) so misnamed exports throw at load + * time rather than producing confusing "undefined.namespace" errors later. */ export async function loadConfig(path: string): Promise { const jiti = createJiti(import.meta.url, { interopDefault: true }); const mod = (await jiti.import(path)) as { config?: unknown; default?: unknown }; @@ -54,5 +56,11 @@ export async function loadConfig(path: string): Promise { if (!config || typeof config !== "object") { throw new Error(`Config at ${path} did not export a \`config\` object`); } + if (!("namespace" in config) || !("collections" in config)) { + throw new Error( + `Config at ${path} did not export a valid \`config\` — missing required fields (namespace, collections). ` + + `Make sure your file does \`export const config: ContrailConfig = { namespace: "…", collections: {…} }\`.` + ); + } return config as T; } diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index bf714fd..413161c 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -38,6 +38,11 @@ export interface CreateAppOptions { spacesDb?: Database; /** Full spaces context override (escape hatch for tests). */ spacesCtx?: SpacesContext | null; + /** Lexicon JSONs to serve at `/lexicons` so consumer apps can fetch + + * typegen against this deployment. Emit with `contrail-lex generate` — + * its `lexicons/generated/index.ts` exports the right shape. If omitted, + * the endpoint returns `404`. */ + lexicons?: object[]; } export function createApp( @@ -54,6 +59,16 @@ export function createApp( const ns = config.namespace; + // Lexicon manifest — lets consumer apps fetch every lexicon this + // deployment speaks (generated + pulled + custom) over HTTP and + // typegen clients, without needing a PDS or DNS resolution. Only + // registered when the caller passed bundled lexicons at build time + // via `contrail-lex generate`. + if (options.lexicons && options.lexicons.length > 0) { + const lexicons = options.lexicons; + app.get(`/xrpc/${ns}.lexicons`, (c) => c.json({ lexicons })); + } + app.get(`/xrpc/${ns}.getProfile`, async (c) => { const actor = c.req.query("actor"); if (!actor) return c.json({ error: "actor parameter required" }, 400); diff --git a/packages/contrail/src/server.ts b/packages/contrail/src/server.ts index 3e1bcf8..fd1eb03 100644 --- a/packages/contrail/src/server.ts +++ b/packages/contrail/src/server.ts @@ -16,16 +16,24 @@ import { markInProcess } from "./core/spaces/in-process"; * For most cases, prefer `contrail.handler()` directly when DBs are bound at * construction time. */ +export interface CreateHandlerOptions { + /** Bundled lexicon JSONs — if provided, served at `/lexicons` so + * consumer apps can fetch + typegen against this deployment. Generate + * with `contrail-lex generate` and import from `lexicons/generated`. */ + lexicons?: object[]; +} + export function createHandler( - contrail: Contrail + contrail: Contrail, + options: CreateHandlerOptions = {} ): (request: Request, db?: Database, spacesDb?: Database) => Promise { // When no per-request DBs are provided, build the app once and reuse it. let cached: ((request: Request) => Promise) | null = null; return (request: Request, db?: Database, spacesDb?: Database) => { if (db || spacesDb) { - return contrail.handler({ db, spacesDb })(request); + return contrail.handler({ db, spacesDb, lexicons: options.lexicons })(request); } - cached ??= contrail.handler(); + cached ??= contrail.handler({ lexicons: options.lexicons }); return cached(request); }; } diff --git a/packages/contrail/src/worker/index.ts b/packages/contrail/src/worker/index.ts new file mode 100644 index 0000000..b0d2ed1 --- /dev/null +++ b/packages/contrail/src/worker/index.ts @@ -0,0 +1,69 @@ +/** + * Prebuilt Cloudflare Workers entrypoint for a contrail deployment. + * + * Collapses the ~12-line boilerplate of `new Contrail()` + `createHandler()` + + * `{ fetch, scheduled }` down to: + * + * import { createWorker } from "@atmo-dev/contrail/worker"; + * import { config } from "./contrail.config"; + * import { lexicons } from "./lexicons/generated"; // optional, enables /lexicons + * export default createWorker(config, { lexicons }); + * + * The handler lazily inits the DB schema on first request per isolate, + * registers every XRPC route, and runs `contrail.ingest()` on the + * `scheduled` event. Pass `binding` if your D1 binding isn't named `DB`; + * pass `onInit` for app-specific one-shot setup that needs the DB. + */ +import { Contrail } from "../contrail.js"; +import { createHandler } from "../server.js"; +import type { ContrailConfig, Database } from "../core/types.js"; + +export interface CreateWorkerOptions { + /** D1 binding name in wrangler env. Default: `"DB"`. */ + binding?: string; + /** Bundled lexicon JSON — if provided, exposes them at `/lexicons` + * so consumer apps can typegen against the deployed service. Generate + * with `contrail-lex generate` (emits `lexicons/generated/index.ts`). */ + lexicons?: object[]; + /** Runs once per isolate, after schema init, before handling the first + * request. Use for app-specific setup that needs a live DB handle. */ + onInit?: (env: Record, db: Database) => void | Promise; +} + +type WorkerEnv = Record; + +export function createWorker( + config: ContrailConfig, + options: CreateWorkerOptions = {} +) { + const binding = options.binding ?? "DB"; + const contrail = new Contrail(config); + const handle = createHandler(contrail, { + lexicons: options.lexicons, + }); + + let ready = false; + const ensureReady = async (env: WorkerEnv, db: Database): Promise => { + if (ready) return; + await contrail.init(db); + await options.onInit?.(env, db); + ready = true; + }; + + return { + async fetch(request: Request, env: WorkerEnv): Promise { + const db = env[binding] as Database; + await ensureReady(env, db); + return (await handle(request, db)) as Response; + }, + async scheduled( + _event: ScheduledEvent, + env: WorkerEnv, + ctx: ExecutionContext + ): Promise { + const db = env[binding] as Database; + await ensureReady(env, db); + ctx.waitUntil(contrail.ingest({}, db)); + }, + }; +} diff --git a/packages/contrail/tests/cli-config.test.ts b/packages/contrail/tests/cli-config.test.ts new file mode 100644 index 0000000..35ce9e4 --- /dev/null +++ b/packages/contrail/tests/cli-config.test.ts @@ -0,0 +1,157 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + findConfigFile, + loadConfig, + CONFIG_CANDIDATES, +} from "../src/cli-config"; + +describe("findConfigFile", () => { + let root: string; + + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "contrail-cli-config-")); + }); + afterAll(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it("returns null when no candidates exist", () => { + expect(findConfigFile(root)).toBeNull(); + }); + + it("finds root-level contrail.config.ts", () => { + const path = join(root, "contrail.config.ts"); + writeFileSync(path, "export const config = {};"); + expect(findConfigFile(root)).toBe(path); + rmSync(path); + }); + + it("finds src/contrail.config.ts", () => { + mkdirSync(join(root, "src"), { recursive: true }); + const path = join(root, "src", "contrail.config.ts"); + writeFileSync(path, "export const config = {};"); + expect(findConfigFile(root)).toBe(path); + rmSync(path); + }); + + it("finds src/lib/contrail.config.ts", () => { + mkdirSync(join(root, "src", "lib"), { recursive: true }); + const path = join(root, "src", "lib", "contrail.config.ts"); + writeFileSync(path, "export const config = {};"); + expect(findConfigFile(root)).toBe(path); + rmSync(path); + }); + + it("finds app/contrail.config.ts", () => { + mkdirSync(join(root, "app"), { recursive: true }); + const path = join(root, "app", "contrail.config.ts"); + writeFileSync(path, "export const config = {};"); + expect(findConfigFile(root)).toBe(path); + rmSync(path); + }); + + it("returns the first match in candidate order", () => { + // Both src/ and app/ versions exist; src/ wins because it's earlier in the list. + mkdirSync(join(root, "src"), { recursive: true }); + mkdirSync(join(root, "app"), { recursive: true }); + const winner = join(root, "src", "contrail.config.ts"); + const loser = join(root, "app", "contrail.config.ts"); + writeFileSync(winner, ""); + writeFileSync(loser, ""); + expect(findConfigFile(root)).toBe(winner); + rmSync(winner); + rmSync(loser); + }); + + it("respects an explicit override path (relative to root)", () => { + const explicit = join(root, "weird.config.ts"); + writeFileSync(explicit, ""); + expect(findConfigFile(root, "weird.config.ts")).toBe(explicit); + rmSync(explicit); + }); + + it("returns null for an explicit path that doesn't exist", () => { + expect(findConfigFile(root, "doesnotexist.ts")).toBeNull(); + }); + + it("supports both .ts and .js variants in candidates", () => { + expect(CONFIG_CANDIDATES).toContain("contrail.config.ts"); + expect(CONFIG_CANDIDATES).toContain("contrail.config.js"); + expect(CONFIG_CANDIDATES).toContain("src/contrail.config.ts"); + }); +}); + +describe("loadConfig", () => { + let root: string; + + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), "contrail-load-config-")); + }); + afterAll(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it("loads a TS file with `export const config`", async () => { + const path = join(root, "named.config.ts"); + writeFileSync( + path, + `export const config = { namespace: "com.example", collections: {} };` + ); + const cfg = await loadConfig(path); + expect(cfg).toEqual({ namespace: "com.example", collections: {} }); + }); + + it("falls back to default export when no `config` named export", async () => { + const path = join(root, "default.config.ts"); + writeFileSync( + path, + `export default { namespace: "com.default", collections: {} };` + ); + const cfg = await loadConfig(path); + expect(cfg).toEqual({ namespace: "com.default", collections: {} }); + }); + + it("prefers named `config` over default export when both exist", async () => { + const path = join(root, "both.config.ts"); + writeFileSync( + path, + `export const config = { namespace: "named", collections: {} }; + export default { namespace: "default", collections: {} };` + ); + const cfg = await loadConfig<{ namespace: string }>(path); + expect(cfg.namespace).toBe("named"); + }); + + it("throws when no config-shaped export is found", async () => { + const path = join(root, "broken.config.ts"); + writeFileSync(path, `export const notConfig = "wrong";`); + await expect(loadConfig(path)).rejects.toThrow(/did not export a valid `config`/); + }); + + it("throws when config exports an object missing required fields", async () => { + const path = join(root, "missingfields.config.ts"); + writeFileSync(path, `export const config = { foo: "bar" };`); + await expect(loadConfig(path)).rejects.toThrow( + /missing required fields \(namespace, collections\)/ + ); + }); + + it("throws when default export is a primitive (not an object)", async () => { + const path = join(root, "primitive.config.ts"); + writeFileSync(path, `export default "just a string";`); + await expect(loadConfig(path)).rejects.toThrow(/did not export a `config` object/); + }); + + it("loads .js files too", async () => { + const path = join(root, "vanilla.config.js"); + writeFileSync( + path, + `export const config = { namespace: "js.example", collections: {} };` + ); + const cfg = await loadConfig(path); + expect(cfg).toEqual({ namespace: "js.example", collections: {} }); + }); +}); diff --git a/packages/contrail/tests/lexicons-endpoint.test.ts b/packages/contrail/tests/lexicons-endpoint.test.ts new file mode 100644 index 0000000..47a4208 --- /dev/null +++ b/packages/contrail/tests/lexicons-endpoint.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from "vitest"; +import { createApp } from "../src/core/router"; +import { createTestDbWithSchema, TEST_CONFIG } from "./helpers"; + +describe("/xrpc/.lexicons endpoint", () => { + it("returns the bundled lexicons when passed", async () => { + const db = await createTestDbWithSchema(); + const lexicons = [ + { lexicon: 1, id: "com.example.event.listRecords", defs: { main: { type: "query" } } }, + { lexicon: 1, id: "com.example.event.getRecord", defs: { main: { type: "query" } } }, + ]; + const app = createApp(db, TEST_CONFIG, { lexicons }); + + const res = await app.fetch(new Request("http://localhost/xrpc/com.example.lexicons")); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body).toEqual({ lexicons }); + }); + + it("404s when no lexicons were passed (route not registered)", async () => { + const db = await createTestDbWithSchema(); + const app = createApp(db, TEST_CONFIG); // no lexicons + + const res = await app.fetch(new Request("http://localhost/xrpc/com.example.lexicons")); + expect(res.status).toBe(404); + }); + + it("404s when lexicons array is empty (treated as not-passed)", async () => { + const db = await createTestDbWithSchema(); + const app = createApp(db, TEST_CONFIG, { lexicons: [] }); + + const res = await app.fetch(new Request("http://localhost/xrpc/com.example.lexicons")); + expect(res.status).toBe(404); + }); + + it("uses the config's namespace in the path", async () => { + const db = await createTestDbWithSchema(); + const lexicons = [{ lexicon: 1, id: "x" }]; + const app = createApp(db, TEST_CONFIG, { lexicons }); + + // Wrong namespace → 404 + const wrong = await app.fetch(new Request("http://localhost/xrpc/wrong.ns.lexicons")); + expect(wrong.status).toBe(404); + + // Right namespace → 200 + const right = await app.fetch(new Request("http://localhost/xrpc/com.example.lexicons")); + expect(right.status).toBe(200); + }); +}); diff --git a/packages/contrail/tests/refresh.test.ts b/packages/contrail/tests/refresh.test.ts new file mode 100644 index 0000000..2f9179c --- /dev/null +++ b/packages/contrail/tests/refresh.test.ts @@ -0,0 +1,199 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { refresh } from "../src/core/refresh"; +import { applyEvents, createTestDbWithSchema, makeEvent, TEST_CONFIG } from "./helpers"; +import type { Database } from "../src/core/types"; + +// We mock the PDS client so refresh() can be exercised without network IO. +// Each test sets the desired pageRecords for a given (did, collection) via +// the `pages` map below. +const pages = new Map>(); + +vi.mock("../src/core/client", () => ({ + getClient: vi.fn(async (did: string) => ({ + get: async ( + _method: string, + opts: { params: { repo: string; collection: string; cursor?: string } } + ) => { + const key = `${opts.params.repo}|${opts.params.collection}`; + // Single page per (did, collection); cursor triggers empty page = done. + if (opts.params.cursor) return { ok: true, data: { records: [], cursor: undefined } }; + const records = pages.get(key) ?? []; + return { ok: true, data: { records, cursor: undefined } }; + }, + })), + getPDS: vi.fn(), +})); + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const EVENT_NSID = "community.lexicon.calendar.event"; + +function aliceEventUri(rkey: string): string { + return `at://${ALICE}/${EVENT_NSID}/${rkey}`; +} + +async function registerKnownDid(db: Database, did: string): Promise { + // refresh() enumerates DIDs from the `backfills` table. + await db + .prepare( + "INSERT INTO backfills (did, collection, completed) VALUES (?, ?, 1) ON CONFLICT DO NOTHING" + ) + .bind(did, EVENT_NSID) + .run(); +} + +describe("refresh", () => { + beforeEach(() => { + pages.clear(); + }); + + it("classifies an unseen-by-DB record as missing", async () => { + const db = await createTestDbWithSchema(); + await registerKnownDid(db, ALICE); + + pages.set(`${ALICE}|${EVENT_NSID}`, [ + { + uri: aliceEventUri("new1"), + cid: "bafy-new", + value: { name: "Brand new", startsAt: "2026-04-01T10:00:00Z" }, + }, + ]); + + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 0 }); + expect(result.total.missing).toBe(1); + expect(result.total.staleUpdates).toBe(0); + expect(result.total.inSync).toBe(0); + expect(result.usersScanned).toBe(1); + }); + + it("classifies a same-CID record as in-sync", async () => { + const db = await createTestDbWithSchema(); + await registerKnownDid(db, ALICE); + + // Seed DB with a record at this URI. + await applyEvents(db, [ + makeEvent({ + did: ALICE, + rkey: "k1", + uri: aliceEventUri("k1"), + cid: "bafy-same", + record: { name: "Already here" }, + }), + ]); + + // PDS returns same URI + same CID. + pages.set(`${ALICE}|${EVENT_NSID}`, [ + { uri: aliceEventUri("k1"), cid: "bafy-same", value: { name: "Already here" } }, + ]); + + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 0 }); + expect(result.total.missing).toBe(0); + expect(result.total.staleUpdates).toBe(0); + expect(result.total.inSync).toBe(1); + }); + + it("classifies a different-CID record as a stale update (outside ignore window)", async () => { + const db = await createTestDbWithSchema(); + await registerKnownDid(db, ALICE); + + // Seed DB with an OLD record (indexed_at way in the past). + const dayAgoUs = (Date.now() - 86_400_000) * 1000; + await applyEvents(db, [ + makeEvent({ + did: ALICE, + rkey: "k2", + uri: aliceEventUri("k2"), + cid: "bafy-old", + time_us: dayAgoUs, + indexed_at: dayAgoUs, + record: { name: "Was online" }, + }), + ]); + + // PDS returns same URI but different CID. + pages.set(`${ALICE}|${EVENT_NSID}`, [ + { uri: aliceEventUri("k2"), cid: "bafy-NEW", value: { name: "Now in-person" } }, + ]); + + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 60_000 }); + expect(result.total.staleUpdates).toBe(1); + expect(result.total.missing).toBe(0); + expect(result.total.inSync).toBe(0); + }); + + it("skips stale-update classification when DB row is within the ignore window", async () => { + const db = await createTestDbWithSchema(); + await registerKnownDid(db, ALICE); + + // Seed with a record indexed JUST NOW. + const nowUs = Date.now() * 1000; + await applyEvents(db, [ + makeEvent({ + did: ALICE, + rkey: "k3", + uri: aliceEventUri("k3"), + cid: "bafy-recent", + time_us: nowUs, + indexed_at: nowUs, + record: { name: "Recent" }, + }), + ]); + + // PDS returns different CID — but DB row is fresh, so it counts as in-sync. + pages.set(`${ALICE}|${EVENT_NSID}`, [ + { uri: aliceEventUri("k3"), cid: "bafy-different", value: { name: "Recent v2" } }, + ]); + + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 60_000 }); + expect(result.total.staleUpdates).toBe(0); + expect(result.total.inSync).toBe(1); + }); + + it("aggregates stats per collection", async () => { + const db = await createTestDbWithSchema(); + await registerKnownDid(db, ALICE); + await registerKnownDid(db, BOB); + + pages.set(`${ALICE}|${EVENT_NSID}`, [ + { uri: aliceEventUri("a1"), cid: "x", value: {} }, + { uri: aliceEventUri("a2"), cid: "y", value: {} }, + ]); + pages.set(`${BOB}|${EVENT_NSID}`, [ + { uri: `at://${BOB}/${EVENT_NSID}/b1`, cid: "z", value: {} }, + ]); + + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 0 }); + expect(result.byCollection[EVENT_NSID]).toBeDefined(); + expect(result.byCollection[EVENT_NSID].missing).toBe(3); + expect(result.usersScanned).toBe(2); + }); + + it("counts a user as failed when getClient throws, and continues with others", async () => { + const { getClient } = await import("../src/core/client"); + (getClient as unknown as ReturnType).mockImplementationOnce( + async () => { + throw new Error("PDS unreachable"); + } + ); + + const db = await createTestDbWithSchema(); + await registerKnownDid(db, ALICE); + await registerKnownDid(db, BOB); + pages.set(`${BOB}|${EVENT_NSID}`, [ + { uri: `at://${BOB}/${EVENT_NSID}/x`, cid: "c", value: {} }, + ]); + + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 0, concurrency: 1 }); + expect(result.usersFailed).toBe(1); + expect(result.usersScanned).toBe(1); + // The non-failing user's records still classified. + expect(result.total.missing).toBeGreaterThanOrEqual(1); + }); + + it("returns elapsed time and the configured ignore window", async () => { + const db = await createTestDbWithSchema(); + const result = await refresh(db, TEST_CONFIG, { ignoreWindowMs: 30_000 }); + expect(result.elapsedMs).toBeGreaterThanOrEqual(0); + expect(result.ignoreWindowMs).toBe(30_000); + }); +}); diff --git a/packages/contrail/tests/worker.test.ts b/packages/contrail/tests/worker.test.ts new file mode 100644 index 0000000..e31b6fd --- /dev/null +++ b/packages/contrail/tests/worker.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, vi } from "vitest"; +import { createWorker } from "../src/worker"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import type { ContrailConfig } from "../src/core/types"; + +const MINIMAL_CONFIG: ContrailConfig = { + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { startsAt: { type: "range" } }, + }, + }, +}; + +describe("createWorker", () => { + it("returns an object with fetch + scheduled handlers", () => { + const worker = createWorker(MINIMAL_CONFIG); + expect(typeof worker.fetch).toBe("function"); + expect(typeof worker.scheduled).toBe("function"); + }); + + it("inits the DB schema lazily on the first fetch (and only once)", async () => { + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(MINIMAL_CONFIG); + const env = { DB: db }; + + // Before first fetch: schema not present yet. + const tables = await db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='cursor'") + .first<{ name: string }>(); + expect(tables).toBeNull(); + + await worker.fetch(new Request("http://localhost/health"), env); + + // After first fetch: schema present. + const after = await db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='cursor'") + .first<{ name: string }>(); + expect(after?.name).toBe("cursor"); + + // Second fetch shouldn't re-init (idempotent regardless, but verify + // onInit fires only once per isolate via a probe). + const onInit = vi.fn(); + const w2 = createWorker(MINIMAL_CONFIG, { onInit }); + await w2.fetch(new Request("http://localhost/health"), env); + await w2.fetch(new Request("http://localhost/health"), env); + await w2.fetch(new Request("http://localhost/health"), env); + expect(onInit).toHaveBeenCalledTimes(1); + }); + + it("respects a custom binding name", async () => { + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(MINIMAL_CONFIG, { binding: "MY_DB" }); + const env = { MY_DB: db }; + + const res = await worker.fetch(new Request("http://localhost/health"), env); + expect(res.status).toBe(200); + }); + + it("serves /xrpc/.lexicons when lexicons are passed", async () => { + const db = createSqliteDatabase(":memory:"); + const lexicons = [{ lexicon: 1, id: "com.example.foo" }]; + const worker = createWorker(MINIMAL_CONFIG, { lexicons }); + const env = { DB: db }; + + const res = await worker.fetch( + new Request("http://localhost/xrpc/com.example.lexicons"), + env + ); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ lexicons }); + }); + + it("does not serve /xrpc/.lexicons when lexicons are omitted", async () => { + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(MINIMAL_CONFIG); + const env = { DB: db }; + + const res = await worker.fetch( + new Request("http://localhost/xrpc/com.example.lexicons"), + env + ); + expect(res.status).toBe(404); + }); + + it("scheduled handler hands the ingest promise to ctx.waitUntil", async () => { + const db = createSqliteDatabase(":memory:"); + const worker = createWorker(MINIMAL_CONFIG); + const env = { DB: db }; + + const waitUntil = vi.fn(); + const ctx = { waitUntil, passThroughOnException: vi.fn() } as unknown as ExecutionContext; + + // Schedule returns once init + waitUntil have been called. The actual + // ingest is a long-running promise that would try to connect to a real + // Jetstream — we don't drain it; we just verify the wire-up. + await worker.scheduled({} as ScheduledEvent, env, ctx); + + expect(waitUntil).toHaveBeenCalledTimes(1); + expect(waitUntil.mock.calls[0][0]).toBeInstanceOf(Promise); + }); +}); diff --git a/packages/contrail/tsup.config.ts b/packages/contrail/tsup.config.ts index fc11602..7cd50c0 100644 --- a/packages/contrail/tsup.config.ts +++ b/packages/contrail/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ "src/adapters/sqlite.ts", "src/adapters/postgres.ts", "src/workers/backfill.ts", + "src/worker/index.ts", "src/cli.ts", "src/cli-config.ts", ], diff --git a/packages/lexicons/src/cli.ts b/packages/lexicons/src/cli.ts index 7c42bff..081c01a 100644 --- a/packages/lexicons/src/cli.ts +++ b/packages/lexicons/src/cli.ts @@ -9,14 +9,15 @@ * contrail-lex types # wraps `lex-cli generate` * contrail-lex all [--no-types] [--config ...] # generate → pull → types * contrail-lex publish [handle] [password] # publish lexicons to a PDS + * contrail-lex pull-service # fetch lexicons from a deployed contrail * * Config auto-detects at ./contrail.config.ts, ./app/config.ts, or * ./src/lib/contrail/config.ts (first match wins); override with --config. * The file must default-export or named-export `config: ContrailConfig`. */ import { spawnSync } from "node:child_process"; -import { existsSync } from "node:fs"; -import { resolve, join } from "node:path"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { resolve, join, dirname } from "node:path"; import { findConfigFile, loadConfig, @@ -25,17 +26,18 @@ import { import { generateLexicons } from "./generate.js"; import { publishLexicons } from "./publish.js"; -type Subcommand = "generate" | "pull" | "types" | "all" | "publish" | "help"; +type Subcommand = "generate" | "pull" | "types" | "all" | "publish" | "pull-service" | "help"; const USAGE = `contrail-lex [options] Subcommands: - generate Emit lexicon JSON from Contrail config - pull Pull external lexicons (wraps \`lex-cli pull\`) - types Generate TS types from lexicon JSON (wraps \`lex-cli generate\`) - all generate → pull → generate → pull → types (full pipeline) - publish Publish lexicon JSON as com.atproto.lexicon.schema records on a PDS - help Print this message + generate Emit lexicon JSON from Contrail config + pull Pull external lexicons (wraps \`lex-cli pull\`) + types Generate TS types from lexicon JSON (wraps \`lex-cli generate\`) + all generate → pull → generate → pull → types (full pipeline) + publish Publish lexicon JSON as com.atproto.lexicon.schema records on a PDS + pull-service Fetch lexicons from a deployed contrail \`/lexicons\` endpoint + help Print this message Options: --config Path to Contrail config file. Default: auto-detect. @@ -44,6 +46,8 @@ Options: --generated-dir For \`publish\`: dir of JSON to publish. Default: lexicons/generated. --skip-confirm For \`publish\`: skip the "do you control these zones?" prompt. --dry-run For \`publish\`: print what would be published + the DNS records needed, no writes. + --out For \`pull-service\`: where to write fetched lexicons. Default: lexicons/pulled. + --namespace For \`pull-service\`: construct the URL as \`/xrpc/.lexicons\`. Or pass a full URL. Environment variables (for \`publish\`): LEXICON_ACCOUNT_IDENTIFIER handle or DID (falls back to positional arg 1) @@ -58,6 +62,8 @@ function parseArgs(argv: string[]): { generatedDir: string; skipConfirm: boolean; dryRun: boolean; + out?: string; + namespace?: string; positional: string[]; } { const args = argv.slice(2); @@ -68,6 +74,8 @@ function parseArgs(argv: string[]): { let generatedDir = "lexicons/generated"; let skipConfirm = false; let dryRun = false; + let out: string | undefined; + let namespace: string | undefined; const positional: string[] = []; for (let i = 0; i < args.length; i++) { const a = args[i]; @@ -77,11 +85,13 @@ function parseArgs(argv: string[]): { else if (a === "--generated-dir") generatedDir = args[++i]; else if (a === "--skip-confirm") skipConfirm = true; else if (a === "--dry-run") dryRun = true; + else if (a === "--out") out = args[++i]; + else if (a === "--namespace") namespace = args[++i]; else if (a === "-h" || a === "--help") - return { cmd: "help", root, withTypes: true, generatedDir, skipConfirm, dryRun, positional }; + return { cmd: "help", root, withTypes: true, generatedDir, skipConfirm, dryRun, out, namespace, positional }; else positional.push(a); } - return { cmd, config, root, withTypes, generatedDir, skipConfirm, dryRun, positional }; + return { cmd, config, root, withTypes, generatedDir, skipConfirm, dryRun, out, namespace, positional }; } @@ -142,8 +152,80 @@ async function cmdPublish( return result.failed.length ? 1 : 0; } +interface LexiconDoc { + id?: string; + [k: string]: unknown; +} + +async function cmdPullService( + root: string, + outDir: string, + namespace: string | undefined, + positional: string[] +): Promise { + const url = positional[0]; + if (!url) { + console.error("usage: contrail-lex pull-service [--namespace ]"); + console.error( + " is either a full URL to the lexicon endpoint\n" + + " e.g. https://my-contrail.dev/xrpc/com.example.lexicons\n" + + " or a base URL combined with --namespace:\n" + + " contrail-lex pull-service https://my-contrail.dev --namespace com.example" + ); + return 1; + } + let endpoint: string; + if (url.includes("/xrpc/")) { + endpoint = url; + } else if (namespace) { + endpoint = `${url.replace(/\/$/, "")}/xrpc/${namespace}.lexicons`; + } else { + console.error( + "pull-service: pass a full URL containing /xrpc/.lexicons, or use --namespace to construct one." + ); + return 1; + } + console.log(`fetching ${endpoint}…`); + + let docs: LexiconDoc[]; + try { + const res = await fetch(endpoint); + if (!res.ok) { + console.error(`request failed: ${res.status} ${res.statusText}`); + return 1; + } + const body = (await res.json()) as { lexicons?: LexiconDoc[] }; + docs = body.lexicons ?? []; + } catch (err) { + console.error(`fetch error: ${(err as Error).message}`); + return 1; + } + + if (docs.length === 0) { + console.error( + "no lexicons in response. the service must pass { lexicons } to createWorker() " + + "(emit with `contrail-lex generate` and import from ./lexicons/generated)." + ); + return 1; + } + + const absOut = resolve(root, outDir); + mkdirSync(absOut, { recursive: true }); + let written = 0; + for (const doc of docs) { + if (!doc.id || typeof doc.id !== "string") continue; + const filePath = join(absOut, ...doc.id.split(".")) + ".json"; + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, JSON.stringify(doc, null, 2) + "\n"); + written++; + } + console.log(`wrote ${written} lexicon(s) to ${absOut}`); + console.log("run `npx lex-cli generate` (or `contrail-lex types`) to emit TS types."); + return 0; +} + async function main(): Promise { - const { cmd, config, root, withTypes, generatedDir, skipConfirm, dryRun, positional } = + const { cmd, config, root, withTypes, generatedDir, skipConfirm, dryRun, out, namespace, positional } = parseArgs(process.argv); if (cmd === "help") { @@ -155,6 +237,8 @@ async function main(): Promise { if (cmd === "types") return runLexCli(["generate"], root); if (cmd === "publish") return cmdPublish(root, generatedDir, positional, skipConfirm, dryRun); + if (cmd === "pull-service") + return cmdPullService(root, out ?? "lexicons/pulled", namespace, positional); const configPath = findConfigFile(root, config); if (!configPath) { diff --git a/packages/lexicons/src/generate.ts b/packages/lexicons/src/generate.ts index bf57b6a..cdf95d6 100644 --- a/packages/lexicons/src/generate.ts +++ b/packages/lexicons/src/generate.ts @@ -6,7 +6,7 @@ */ import { writeFileSync, mkdirSync, rmSync, existsSync, readFileSync, readdirSync } from "fs"; -import { join } from "path"; +import { join, relative } from "path"; import type { ContrailConfig } from "@atmo-dev/contrail"; /** Return the sorted list of XRPC method NSIDs (queries + procedures) in a @@ -1036,6 +1036,40 @@ export function generateLexicons(options: GenerateOptions): Record.lexicons` — the pulled ones are needed so consumer typegen + // can resolve `$ref`s from the generated schemas. + if (outputDir) { + const paths = new Set(); + + // Generated JSONs live in outputDir itself. + for (const nsid of Object.keys(generated)) { + paths.add("./" + nsid.split(".").join("/") + ".json"); + } + + // Pulled + custom JSONs live in the configured lexiconDirs (outside outputDir). + for (const dir of lexiconDirs) { + if (!existsSync(dir)) continue; + for (const file of walkJson(dir)) { + let rel = relative(outputDir, file); + if (!rel.startsWith(".")) rel = "./" + rel; + paths.add(rel); + } + } + + const sorted = [...paths].sort(); + const imports = sorted.map((p, i) => `import _${i} from "${p}";`).join("\n"); + const exportArray = `export const lexicons: object[] = [${sorted.map((_, i) => `_${i}`).join(", ")}];\n`; + const header = + "// Auto-generated by @atmo-dev/contrail-lexicons. Do not edit.\n" + + "// Pass `lexicons` to `createWorker(config, { lexicons })` to expose them\n" + + "// at `/xrpc/.lexicons` for consumer apps to typegen against.\n\n"; + writeFileSync(join(outputDir, "index.ts"), header + imports + "\n\n" + exportArray); + } + log("\nDone!"); return generated; } diff --git a/packages/lexicons/tests/generate.test.ts b/packages/lexicons/tests/generate.test.ts index 28460ff..65b283c 100644 --- a/packages/lexicons/tests/generate.test.ts +++ b/packages/lexicons/tests/generate.test.ts @@ -1,5 +1,7 @@ -import { describe, it, expect, beforeAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { join } from "path"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "fs"; +import { tmpdir } from "os"; import { generateLexicons, extractXrpcMethods, listXrpcMethods } from "../src/generate"; import type { ContrailConfig } from "@atmo-dev/contrail"; @@ -255,3 +257,92 @@ describe("extractXrpcMethods / listXrpcMethods", () => { expect(methods).toContain("test.comm.realtime.subscribe"); }); }); + +describe("manifest emission (lexicons/generated/index.ts)", () => { + let workdir: string; + let outDir: string; + + beforeAll(() => { + workdir = mkdtempSync(join(tmpdir(), "contrail-manifest-")); + outDir = join(workdir, "lexicons", "generated"); + }); + afterAll(() => { + rmSync(workdir, { recursive: true, force: true }); + }); + + it("emits index.ts with imports for every generated lexicon", () => { + generateLexicons({ + config: BASIC_CONFIG, + rootDir: workdir, + outputDir: outDir, + lexiconDirs: [], // skip pulled/custom for this test + quiet: true, + }); + const manifest = readFileSync(join(outDir, "index.ts"), "utf-8"); + + // One import per generated NSID — at minimum: collection-level + admin endpoints. + expect(manifest).toMatch(/import _0 from "\.\/test\/app\/.*\.json";/); + expect(manifest).toContain('export const lexicons: object[] ='); + + // Sorted, so getCursor (admin) appears before post.* (collection). + const importLines = manifest.split("\n").filter((l) => l.startsWith("import ")); + const paths = importLines.map((l) => l.match(/from "(.*?)"/)?.[1] ?? ""); + expect(paths).toEqual([...paths].sort()); + }); + + it("includes pulled lexicons in the manifest (with relative paths)", () => { + // Set up a fake pulled dir with one lexicon. + const pulledDir = join(workdir, "lexicons", "pulled"); + mkdirSync(join(pulledDir, "app", "bsky", "actor"), { recursive: true }); + writeFileSync( + join(pulledDir, "app", "bsky", "actor", "profile.json"), + JSON.stringify({ lexicon: 1, id: "app.bsky.actor.profile", defs: {} }) + ); + + rmSync(outDir, { recursive: true, force: true }); + generateLexicons({ + config: BASIC_CONFIG, + rootDir: workdir, + outputDir: outDir, + lexiconDirs: [pulledDir], + quiet: true, + }); + const manifest = readFileSync(join(outDir, "index.ts"), "utf-8"); + + // Pulled lexicon path is relative from outputDir, so it walks `..`. + expect(manifest).toContain('import _0 from "../pulled/app/bsky/actor/profile.json";'); + // Generated lexicons are still there with `./` paths. + expect(manifest).toMatch(/import _\d+ from "\.\/test\/app\//); + }); + + it("dedupes if the same path is somehow listed twice", () => { + rmSync(outDir, { recursive: true, force: true }); + generateLexicons({ + config: BASIC_CONFIG, + rootDir: workdir, + outputDir: outDir, + lexiconDirs: [], + quiet: true, + }); + const manifest = readFileSync(join(outDir, "index.ts"), "utf-8"); + const importLines = manifest.split("\n").filter((l) => l.startsWith("import ")); + const paths = importLines.map((l) => l.match(/from "(.*?)"/)?.[1] ?? ""); + expect(new Set(paths).size).toBe(paths.length); + }); + + it("does not emit a manifest when outputDir is omitted (in-memory only)", () => { + const isolatedDir = mkdtempSync(join(tmpdir(), "contrail-no-output-")); + try { + generateLexicons({ + config: BASIC_CONFIG, + rootDir: isolatedDir, + // no outputDir + lexiconDirs: [], + quiet: true, + }); + expect(existsSync(join(isolatedDir, "lexicons"))).toBe(false); + } finally { + rmSync(isolatedDir, { recursive: true, force: true }); + } + }); +});