From dbcad9ee2152ee00248dc93ecfdfd556827096a2 Mon Sep 17 00:00:00 2001 From: Roscoe Rubin-Rottenberg Date: Fri, 3 Apr 2026 18:36:31 -0400 Subject: [PATCH] revert xrpc-server and add backwards compat to lex-gen --- deno.lock | 10 +- lex-gen/cmd/build.ts | 2 +- lex-gen/cmd/gen-api.ts | 81 ++ lex-gen/cmd/gen-md.ts | 72 ++ lex-gen/cmd/gen-server.ts | 85 ++ lex-gen/cmd/gen-ts-obj.ts | 49 + lex-gen/cmd/index.ts | 6 +- lex-gen/codegen/client.ts | 652 +++++++++++++ lex-gen/codegen/common.ts | 299 ++++++ lex-gen/codegen/lex-gen.ts | 1060 ++++++++++++++++++++++ lex-gen/codegen/server.ts | 503 ++++++++++ lex-gen/codegen/util.ts | 108 +++ lex-gen/config.ts | 142 +++ lex-gen/deno.json | 4 +- lex-gen/mdgen/index.ts | 78 ++ lex-gen/mod.ts | 18 +- lex-gen/pull.ts | 163 ++++ lex-gen/types.ts | 48 + lex-gen/util.ts | 290 ++++++ xrpc-server/server.ts | 700 +------------- xrpc-server/tests/lex-router-api_test.ts | 205 ----- xrpc-server/types.ts | 105 +-- 22 files changed, 3702 insertions(+), 978 deletions(-) create mode 100644 lex-gen/cmd/gen-api.ts create mode 100644 lex-gen/cmd/gen-md.ts create mode 100644 lex-gen/cmd/gen-server.ts create mode 100644 lex-gen/cmd/gen-ts-obj.ts create mode 100644 lex-gen/codegen/client.ts create mode 100644 lex-gen/codegen/common.ts create mode 100644 lex-gen/codegen/lex-gen.ts create mode 100644 lex-gen/codegen/server.ts create mode 100644 lex-gen/codegen/util.ts create mode 100644 lex-gen/config.ts create mode 100644 lex-gen/mdgen/index.ts create mode 100644 lex-gen/pull.ts create mode 100644 lex-gen/types.ts create mode 100644 lex-gen/util.ts delete mode 100644 xrpc-server/tests/lex-router-api_test.ts diff --git a/deno.lock b/deno.lock index a552538..4607e5e 100644 --- a/deno.lock +++ b/deno.lock @@ -33,6 +33,7 @@ "jsr:@std/text@~1.0.7": "1.0.16", "jsr:@ts-morph/common@0.27": "0.27.0", "jsr:@ts-morph/ts-morph@26": "26.0.0", + "jsr:@zod/zod@^4.1.11": "4.3.6", "jsr:@zod/zod@^4.1.13": "4.3.6", "npm:@atproto/crypto@*": "0.1.0", "npm:@did-plc/lib@^0.0.4": "0.0.4", @@ -45,6 +46,7 @@ "npm:key-encoder@^2.0.3": "2.0.3", "npm:multiformats@^13.4.1": "13.4.1", "npm:p-queue@^8.1.1": "8.1.1", + "npm:prettier@^3.6.2": "3.8.1", "npm:rate-limiter-flexible@9": "9.0.0", "npm:ws@^8.18.0": "8.18.3" }, @@ -883,6 +885,10 @@ "xtend" ] }, + "prettier@3.8.1": { + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "bin": true + }, "process-warning@3.0.0": { "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, @@ -1147,7 +1153,9 @@ "jsr:@std/fs@^1.0.19", "jsr:@std/jsonc@^1.0.1", "jsr:@std/path@^1.1.2", - "jsr:@ts-morph/ts-morph@26" + "jsr:@ts-morph/ts-morph@26", + "jsr:@zod/zod@^4.1.11", + "npm:prettier@^3.6.2" ] }, "lexicon": { diff --git a/lex-gen/cmd/build.ts b/lex-gen/cmd/build.ts index 191f4a1..0d74867 100644 --- a/lex-gen/cmd/build.ts +++ b/lex-gen/cmd/build.ts @@ -13,7 +13,7 @@ const command = new Command() .option( "-o, --out ", "output directory for generated TS files", - { required: true, default: "./src/lexicons" }, + { required: true, default: "./lex" }, ) .option("--clear", "clear output directory before generating files", { default: false, diff --git a/lex-gen/cmd/gen-api.ts b/lex-gen/cmd/gen-api.ts new file mode 100644 index 0000000..7fed63b --- /dev/null +++ b/lex-gen/cmd/gen-api.ts @@ -0,0 +1,81 @@ +import { Command } from "@cliffy/command"; +import { + applyFileDiff, + genFileDiff, + printFileDiff, + readAllLexicons, + shouldPullLexicons, +} from "../util.ts"; +import { genClientApi } from "../codegen/client.ts"; +import { formatGeneratedFiles } from "../codegen/util.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; + +const command = new Command() + .description("Generate a TS client API") + .option("--js", "use .js extension for imports instead of .ts") + .option("-o, --outdir ", "dir path to write to") + .option("-i, --input ", "paths of lexicon files to include") + .option("--config ", "path to config file") + .action( + async ({ outdir, input, js, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalOutdir = outdir ?? config?.outdir; + const finalInput = input ?? config?.files; + + if (!finalOutdir) { + console.error("outdir is required (provide via -o/--outdir or config)"); + if (typeof Deno !== "undefined") { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalInput || finalInput.length === 0) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (typeof Deno !== "undefined") { + Deno.exit(1); + } else { + process.exit(1); + } + } + + const filesProvidedViaCli = input !== undefined; + const needsPull = shouldPullLexicons( + config, + filesProvidedViaCli, + finalInput, + ); + if (needsPull && config?.pull) { + await pullLexicons(config.pull); + } + + const useJs = js ?? false; + const importSuffix = config?.modules?.importSuffix; + const mappings = config?.mappings; + const lexicons = readAllLexicons(finalInput); + const api = await genClientApi(lexicons, { + useJsExtension: useJs, + importSuffix: importSuffix, + mappings: mappings, + }); + const diff = genFileDiff(finalOutdir, api); + console.log("This will write the following files:"); + printFileDiff(diff); + applyFileDiff(diff); + if (typeof Deno !== "undefined") { + await formatGeneratedFiles(finalOutdir); + } + console.log("API generated."); + + if (needsPull && config?.pull) { + cleanupPullDirectory(config.pull); + } + }, + ); + +export default command; diff --git a/lex-gen/cmd/gen-md.ts b/lex-gen/cmd/gen-md.ts new file mode 100644 index 0000000..c67a6c9 --- /dev/null +++ b/lex-gen/cmd/gen-md.ts @@ -0,0 +1,72 @@ +import { Command } from "@cliffy/command"; +import { readAllLexicons, shouldPullLexicons } from "../util.ts"; +import * as mdGen from "../mdgen/index.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; + +const isDeno = typeof Deno !== "undefined"; + +const command = new Command() + .description("Generate markdown documentation") + .option("-o, --output ", "Output file path") + .option("-i, --input ", "Input file path") + .option("--config ", "path to config file") + .action( + async ({ output, input, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalOutput = output ?? + (config?.outdir ? `${config.outdir}/docs.md` : undefined); + const finalInput = input ?? config?.files?.[0]; + + if (!finalOutput) { + console.error("output is required (provide via -o/--output or config)"); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalInput) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalOutput.endsWith(".md")) { + console.error( + "Must supply the path to a .md file", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + const filesProvidedViaCli = input !== undefined; + const needsPull = shouldPullLexicons( + config, + filesProvidedViaCli, + [finalInput], + ); + if (needsPull && config?.pull) { + await pullLexicons(config.pull); + } + + const lexicons = readAllLexicons(finalInput); + await mdGen.process(finalOutput, lexicons); + + if (needsPull && config?.pull) { + cleanupPullDirectory(config.pull); + } + }, + ); + +export default command; diff --git a/lex-gen/cmd/gen-server.ts b/lex-gen/cmd/gen-server.ts new file mode 100644 index 0000000..21e8114 --- /dev/null +++ b/lex-gen/cmd/gen-server.ts @@ -0,0 +1,85 @@ +import { Command } from "@cliffy/command"; +import { + applyFileDiff, + genFileDiff, + printFileDiff, + readAllLexicons, + shouldPullLexicons, +} from "../util.ts"; +import { formatGeneratedFiles } from "../codegen/util.ts"; +import { genServerApi } from "../codegen/server.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; + +const isDeno = typeof Deno !== "undefined"; + +const command = new Command() + .description("Generate a TS server API") + .option("--js", "use .js extension for imports instead of .ts") + .option("-o, --outdir ", "dir path to write to") + .option("-i, --input ", "paths of lexicon files to include") + .option("--config ", "path to config file") + .action( + async ({ outdir, input, js, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalOutdir = outdir ?? config?.outdir; + const finalInput = input ?? config?.files; + + if (!finalOutdir) { + console.error("outdir is required (provide via -o/--outdir or config)"); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalInput || finalInput.length === 0) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + const filesProvidedViaCli = input !== undefined; + const needsPull = shouldPullLexicons( + config, + filesProvidedViaCli, + finalInput, + ); + if (needsPull && config?.pull) { + await pullLexicons(config.pull); + } + + const useJs = js ?? false; + const importSuffix = config?.modules?.importSuffix; + const mappings = config?.mappings; + console.log("Generating API..."); + const lexicons = readAllLexicons(finalInput); + const api = await genServerApi(lexicons, { + useJsExtension: useJs, + importSuffix: importSuffix, + mappings: mappings, + }); + console.log("API generated."); + const diff = genFileDiff(finalOutdir, api); + console.log("This will write the following files:"); + printFileDiff(diff); + applyFileDiff(diff); + if (typeof Deno !== "undefined") { + await formatGeneratedFiles(finalOutdir); + } + console.log("API generated."); + + if (needsPull && config?.pull) { + cleanupPullDirectory(config.pull); + } + }, + ); + +export default command; diff --git a/lex-gen/cmd/gen-ts-obj.ts b/lex-gen/cmd/gen-ts-obj.ts new file mode 100644 index 0000000..3e89a23 --- /dev/null +++ b/lex-gen/cmd/gen-ts-obj.ts @@ -0,0 +1,49 @@ +import { Command } from "@cliffy/command"; +import { genTsObj, readAllLexicons, shouldPullLexicons } from "../util.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; + +const isDeno = typeof Deno !== "undefined"; + +const command = new Command() + .description("Generate a TS file that exports an array of lexicons") + .option("-i, --input ", "paths of the lexicon files to include") + .option("--config ", "path to config file") + .action(async ({ input, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalInput = input ?? config?.files; + + if (!finalInput || finalInput.length === 0) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + const filesProvidedViaCli = input !== undefined; + const finalInputArray = Array.isArray(finalInput) + ? finalInput + : [finalInput]; + const needsPull = shouldPullLexicons( + config, + filesProvidedViaCli, + finalInputArray, + ); + if (needsPull && config?.pull) { + await pullLexicons(config.pull); + } + + const lexicons = readAllLexicons(finalInput); + console.log(genTsObj(lexicons)); + + if (needsPull && config?.pull) { + cleanupPullDirectory(config.pull); + } + }); + +export default command; diff --git a/lex-gen/cmd/index.ts b/lex-gen/cmd/index.ts index edb68bb..b0eb559 100644 --- a/lex-gen/cmd/index.ts +++ b/lex-gen/cmd/index.ts @@ -1,3 +1,7 @@ import build from "./build.ts"; +import genMd from "./gen-md.ts"; +import genApi from "./gen-api.ts"; +import genServer from "./gen-server.ts"; +import genTsObj from "./gen-ts-obj.ts"; -export { build }; +export { build, genApi, genMd, genServer, genTsObj }; diff --git a/lex-gen/codegen/client.ts b/lex-gen/codegen/client.ts new file mode 100644 index 0000000..0ec6a9d --- /dev/null +++ b/lex-gen/codegen/client.ts @@ -0,0 +1,652 @@ +import { + IndentationText, + Project, + type SourceFile, + VariableDeclarationKind, +} from "ts-morph"; +import { type LexiconDoc, Lexicons, type LexRecord } from "@atp/lexicon"; +import { NSID } from "@atp/syntax"; +import type { GeneratedAPI } from "../types.ts"; +import { gen, lexiconsTs, utilTs } from "./common.ts"; +import { + collectExternalImports, + genCommonImports, + genImports, + genRecord, + genUserType, + genXrpcInput, + genXrpcOutput, + genXrpcParams, + resolveExternalImport, +} from "./lex-gen.ts"; +import { + type CodeGenOptions, + type DefTreeNode, + lexiconsToDefTree, + schemasToNsidTokens, + toCamelCase, + toScreamingSnakeCase, + toTitleCase, +} from "./util.ts"; + +const ATP_METHODS = { + list: "com.atproto.repo.listRecords", + get: "com.atproto.repo.getRecord", + create: "com.atproto.repo.createRecord", + put: "com.atproto.repo.putRecord", + delete: "com.atproto.repo.deleteRecord", +}; + +export async function genClientApi( + lexiconDocs: LexiconDoc[], + options?: CodeGenOptions, +): Promise { + const project = new Project({ + useInMemoryFileSystem: true, + manipulationSettings: { indentationText: IndentationText.TwoSpaces }, + }); + const api: GeneratedAPI = { files: [] }; + const lexicons = new Lexicons(lexiconDocs); + const nsidTree = lexiconsToDefTree(lexiconDocs); + const nsidTokens = schemasToNsidTokens(lexiconDocs); + for (const lexiconDoc of lexiconDocs) { + api.files.push(await lexiconTs(project, lexicons, lexiconDoc, options)); + } + api.files.push(await utilTs(project)); + api.files.push(await lexiconsTs(project, lexiconDocs, options)); + api.files.push( + await indexTs(project, lexiconDocs, nsidTree, nsidTokens, options), + ); + return api; +} + +const indexTs = ( + project: Project, + lexiconDocs: LexiconDoc[], + nsidTree: DefTreeNode[], + nsidTokens: Record, + options?: CodeGenOptions, +) => + gen(project, "/index.ts", (file) => { + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); + //= import { XrpcClient, type FetchHandler, type FetchHandlerOptions } from '@atp/xrpc' + file.addImportDeclaration({ + moduleSpecifier: "@atp/xrpc", + namedImports: [ + { name: "XrpcClient" }, + { name: "FetchHandler", isTypeOnly: true }, + { name: "FetchHandlerOptions", isTypeOnly: true }, + ], + }); + //= import {schemas} from './lexicons.ts' + file.addImportDeclaration({ + moduleSpecifier: `./lexicons${importExtension}`, + namedImports: [{ name: "schemas" }], + }); + + //= import { type OmitKey, type Un$Typed } from './util.ts' + file.addImportDeclaration({ + moduleSpecifier: `./util${importExtension}`, + isTypeOnly: true, + namedImports: [ + { name: "OmitKey" }, + { name: "Un$Typed" }, + ], + }); + + // collect and import external lexicon references + const externalImports = collectExternalImports(lexiconDocs, options); + const mappings = options?.mappings; + for (const [nsid, types] of externalImports) { + const mapping = resolveExternalImport(nsid, mappings); + if (mapping) { + if (typeof mapping.imports === "string") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: mapping.imports, + namedImports: [{ name: toTitleCase(nsid), isTypeOnly: true }], + }); + } else { + const result = mapping.imports(nsid); + if (result.type === "namespace") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namespaceImport: toTitleCase(nsid), + }); + } else { + const namedImports = Array.from(types).map((typeName) => ({ + name: toTitleCase(typeName), + isTypeOnly: true, + })); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namedImports, + }); + } + } + } + } + + // generate type imports and re-exports + for (const lexicon of lexiconDocs) { + const moduleSpecifier = `./types/${ + lexicon.id.split(".").join("/") + }${importExtension}`; + + const defs = Object.values(lexicon.defs); + const hasRecord = defs.some((d) => d.type === "record"); + const hasQueryOrProc = defs.some( + (d) => d.type === "query" || d.type === "procedure", + ); + const needsValue = defs.some( + (d) => + (d.type === "query" || d.type === "procedure") && d.errors?.length, + ); + + if (hasRecord || hasQueryOrProc) { + file.addImportDeclaration({ + moduleSpecifier, + isTypeOnly: !needsValue, + namespaceImport: toTitleCase(lexicon.id), + }); + } + + file + .addExportDeclaration({ moduleSpecifier }) + .setNamespaceExport(toTitleCase(lexicon.id)); + } + + // generate token enums + for (const nsidAuthority in nsidTokens) { + // export const {THE_AUTHORITY} = { + // {Name}: "{authority.the.name}" + // } + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: toScreamingSnakeCase(nsidAuthority), + initializer: [ + "{", + ...nsidTokens[nsidAuthority].map( + (nsidName) => + `${toTitleCase(nsidName)}: "${nsidAuthority}.${nsidName}",`, + ), + "}", + ].join("\n"), + }, + ], + }); + } + + //= export class AtpBaseClient {...} + const clientCls = file.addClass({ + name: "AtpBaseClient", + isExported: true, + extends: "XrpcClient", + }); + + for (const ns of nsidTree) { + //= ns: NS + clientCls.addProperty({ + name: ns.propName, + type: ns.className, + }); + } + + //= constructor (options: FetchHandler | FetchHandlerOptions) { + //= super(options, schemas) + //= {namespace declarations} + //= } + clientCls.addConstructor({ + parameters: [ + { name: "options", type: "FetchHandler | FetchHandlerOptions" }, + ], + statements: [ + "super(options, schemas)", + ...nsidTree.map( + (ns) => `this.${ns.propName} = new ${ns.className}(this)`, + ), + ], + }); + + //= /** @deprecated use `this` instead */ + //= get xrpc(): XrpcClient { + //= return this + //= } + clientCls + .addGetAccessor({ + name: "xrpc", + returnType: "XrpcClient", + statements: ["return this"], + }) + .addJsDoc("@deprecated use `this` instead"); + + // generate classes for the schemas + for (const ns of nsidTree) { + genNamespaceCls(file, ns); + } + }); + +function genNamespaceCls(file: SourceFile, ns: DefTreeNode) { + //= export class {ns}NS {...} + const cls = file.addClass({ + name: ns.className, + isExported: true, + }); + //= _client: XrpcClient + cls.addProperty({ + name: "_client", + type: "XrpcClient", + }); + + for (const userType of ns.userTypes) { + if (userType.def.type !== "record") { + continue; + } + //= type: TypeRecord + const name = NSID.parse(userType.nsid).name || ""; + cls.addProperty({ + name: toCamelCase(name), + type: `${toTitleCase(userType.nsid)}Record`, + }); + } + + for (const child of ns.children) { + //= child: ChildNS + cls.addProperty({ + name: child.propName, + type: child.className, + }); + + // recurse + genNamespaceCls(file, child); + } + + //= constructor(public client: XrpcClient) { + //= this._client = client + //= {child namespace prop declarations} + //= {record prop declarations} + //= } + cls.addConstructor({ + parameters: [ + { + name: "client", + type: "XrpcClient", + }, + ], + statements: [ + `this._client = client`, + ...ns.children.map( + (ns) => `this.${ns.propName} = new ${ns.className}(client)`, + ), + ...ns.userTypes + .filter((ut) => ut.def.type === "record") + .map((ut) => { + const name = NSID.parse(ut.nsid).name || ""; + return `this.${toCamelCase(name)} = new ${ + toTitleCase( + ut.nsid, + ) + }Record(client)`; + }), + ], + }); + + // methods + for (const userType of ns.userTypes) { + if (userType.def.type !== "query" && userType.def.type !== "procedure") { + continue; + } + const isGetReq = userType.def.type === "query"; + const moduleName = toTitleCase(userType.nsid); + const name = toCamelCase(NSID.parse(userType.nsid).name || ""); + const method = cls.addMethod({ + name, + returnType: `Promise<${moduleName}.Response>`, + }); + if (isGetReq) { + method.addParameter({ + name: "params?", + type: `${moduleName}.QueryParams`, + }); + } else if (userType.def.type === "procedure") { + method.addParameter({ + name: "data?", + type: `${moduleName}.InputSchema`, + }); + } + method.addParameter({ + name: "opts?", + type: `${moduleName}.CallOptions`, + }); + method.setBodyText( + [ + `return this._client`, + isGetReq + ? `.call('${userType.nsid}', params, undefined, opts)` + : `.call('${userType.nsid}', opts?.qp, data, opts)`, + userType.def.errors?.length + // Only add a catch block if there are custom errors + ? ` .catch((e) => { throw ${moduleName}.toKnownErr(e) })` + : "", + ].join("\n"), + ); + } + + // record api classes + for (const userType of ns.userTypes) { + if (userType.def.type !== "record") { + continue; + } + genRecordCls(file, userType.nsid, userType.def); + } +} + +function genRecordCls(file: SourceFile, nsid: string, lexRecord: LexRecord) { + //= export class {type}Record {...} + const cls = file.addClass({ + name: `${toTitleCase(nsid)}Record`, + isExported: true, + }); + //= _client: XrpcClient + cls.addProperty({ + name: "_client", + type: "XrpcClient", + }); + + //= constructor(client: XrpcClient) { + //= this._client = client + //= } + const cons = cls.addConstructor(); + cons.addParameter({ + name: "client", + type: "XrpcClient", + }); + cons.setBodyText(`this._client = client`); + + // methods + const typeModule = toTitleCase(nsid); + { + //= list() + const method = cls.addMethod({ + isAsync: true, + name: "list", + returnType: + `Promise<{cursor?: string, records: ({uri: string, value: ${typeModule}.Record})[]}>`, + }); + method.addParameter({ + name: "params", + type: `OmitKey<${ + toTitleCase(ATP_METHODS.list) + }.QueryParams, "collection">`, + }); + method.setBodyText( + [ + `const res = await this._client.call('${ATP_METHODS.list}', { collection: '${nsid}', ...params })`, + `return res.data`, + ].join("\n"), + ); + } + { + //= get() + const method = cls.addMethod({ + isAsync: true, + name: "get", + returnType: + `Promise<{uri: string, cid: string, value: ${typeModule}.Record}>`, + }); + method.addParameter({ + name: "params", + type: `OmitKey<${ + toTitleCase(ATP_METHODS.get) + }.QueryParams, "collection">`, + }); + method.setBodyText( + [ + `const res = await this._client.call('${ATP_METHODS.get}', { collection: '${nsid}', ...params })`, + `return res.data`, + ].join("\n"), + ); + } + { + //= create() + const method = cls.addMethod({ + isAsync: true, + name: "create", + returnType: "Promise<{uri: string, cid: string}>", + }); + method.addParameter({ + name: "params", + type: `OmitKey<${ + toTitleCase( + ATP_METHODS.create, + ) + }.InputSchema, "collection" | "record">`, + }); + method.addParameter({ + name: "record", + type: `Un$Typed<${typeModule}.Record>`, + }); + method.addParameter({ + name: "headers?", + type: `Record`, + }); + const maybeRkeyPart = lexRecord.key?.startsWith("literal:") + ? `rkey: '${lexRecord.key.replace("literal:", "")}', ` + : ""; + method.setBodyText( + [ + `const collection = '${nsid}'`, + `const res = await this._client.call('${ATP_METHODS.create}', undefined, { collection, ${maybeRkeyPart}...params, record: { ...record, $type: collection} }, {encoding: 'application/json', headers })`, + `return res.data`, + ].join("\n"), + ); + } + // { + // //= put() + // const method = cls.addMethod({ + // isAsync: true, + // name: 'put', + // returnType: 'Promise<{uri: string, cid: string}>', + // }) + // method.addParameter({ + // name: 'params', + // type: `OmitKey<${toTitleCase(ATP_METHODS.put)}.InputSchema, "collection" | "record">`, + // }) + // method.addParameter({ + // name: 'record', + // type: `${typeModule}.Record`, + // }) + // method.addParameter({ + // name: 'headers?', + // type: `Record`, + // }) + // method.setBodyText( + // [ + // `record.$type = '${userType.nsid}'`, + // `const res = await this._client.call('${ATP_METHODS.put}', undefined, { collection: '${userType.nsid}', record, ...params }, {encoding: 'application/json', headers})`, + // `return res.data`, + // ].join('\n'), + // ) + // } + { + //= delete() + const method = cls.addMethod({ + isAsync: true, + name: "delete", + returnType: "Promise", + }); + method.addParameter({ + name: "params", + type: `OmitKey<${ + toTitleCase( + ATP_METHODS.delete, + ) + }.InputSchema, "collection">`, + }); + method.addParameter({ + name: "headers?", + type: `Record`, + }); + + method.setBodyText( + [ + `await this._client.call('${ATP_METHODS.delete}', undefined, { collection: '${nsid}', ...params }, { headers })`, + ].join("\n"), + ); + } +} + +const lexiconTs = ( + project: Project, + lexicons: Lexicons, + lexiconDoc: LexiconDoc, + options?: CodeGenOptions, +) => + gen( + project, + `/types/${lexiconDoc.id.split(".").join("/")}.ts`, + (file) => { + // Filter out subscriptions as they are not currently generated for client + const filteredDefs = Object.fromEntries( + Object.entries(lexiconDoc.defs).filter(([_, def]) => + def.type !== "subscription" + ), + ); + const filteredDoc = { ...lexiconDoc, defs: filteredDefs }; + + const main = filteredDoc.defs.main; + if ( + main?.type === "query" || + main?.type === "procedure" + ) { + const needsXrpcError = (main.type === "query" || + main.type === "procedure") && main.errors?.length; + + //= import {HeadersMap, XRPCError} from '@atp/xrpc' + file.addImportDeclaration({ + moduleSpecifier: "@atp/xrpc", + isTypeOnly: !needsXrpcError, + namedImports: needsXrpcError + ? [{ name: "HeadersMap", isTypeOnly: true }, { name: "XRPCError" }] + : [{ name: "HeadersMap" }], + }); + } + + genCommonImports(file, lexiconDoc.id, filteredDoc); + + const imports: Map> = new Map(); + for (const defId in filteredDoc.defs) { + const def = filteredDoc.defs[defId]; + const lexUri = `${lexiconDoc.id}#${defId}`; + if (defId === "main") { + if (def.type === "query" || def.type === "procedure") { + genXrpcParams(file, lexicons, lexUri, false); + genXrpcInput(file, imports, lexicons, lexUri, false, options); + genXrpcOutput(file, imports, lexicons, lexUri, false, options); + genClientXrpcCommon(file, lexicons, lexUri); + } else if (def.type === "record") { + genRecord(file, imports, lexicons, lexUri, options); + } else { + genUserType(file, imports, lexicons, lexUri, options); + } + } else { + genUserType(file, imports, lexicons, lexUri, options); + } + } + genImports(file, imports, lexiconDoc.id, options); + return Promise.resolve(); + }, + ); + +function genClientXrpcCommon( + file: SourceFile, + lexicons: Lexicons, + lexUri: string, +) { + const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); + + //= export interface CallOptions {...} + const opts = file.addInterface({ + name: "CallOptions", + isExported: true, + }); + opts.addProperty({ name: "signal?", type: "AbortSignal" }); + opts.addProperty({ name: "headers?", type: "HeadersMap" }); + if (def.type === "procedure") { + opts.addProperty({ name: "qp?", type: "QueryParams" }); + } + if (def.type === "procedure" && def.input) { + let encodingType = "string"; + if (def.input.encoding !== "*/*") { + encodingType = def.input.encoding + .split(",") + .map((v) => `'${v.trim()}'`) + .join(" | "); + } + opts.addProperty({ + name: "encoding?", + type: encodingType, + }); + } + + // export interface Response {...} + const res = file.addInterface({ + name: "Response", + isExported: true, + }); + res.addProperty({ name: "success", type: "boolean" }); + res.addProperty({ name: "headers", type: "HeadersMap" }); + if (def.output?.schema) { + if (def.output.encoding?.includes(",")) { + res.addProperty({ name: "data", type: "OutputSchema | Uint8Array" }); + } else { + res.addProperty({ name: "data", type: "OutputSchema" }); + } + } else if (def.output?.encoding) { + res.addProperty({ name: "data", type: "Uint8Array" }); + } + + // export class {errcode}Error {...} + const customErrors: { name: string; cls: string }[] = []; + for (const error of def.errors || []) { + let name = toTitleCase(error.name); + if (!name.endsWith("Error")) name += "Error"; + const errCls = file.addClass({ + name, + extends: "XRPCError", + isExported: true, + }); + errCls.addConstructor({ + parameters: [{ name: "src", type: "XRPCError" }], + statements: [ + "super(src.status, src.error, src.message, src.headers, { cause: src })", + ], + }); + + customErrors.push({ name: error.name, cls: name }); + } + + // export function toKnownErr(err: any) {...} + file.addFunction({ + name: "toKnownErr", + isExported: true, + parameters: [{ name: "e", type: "unknown" }], + returnType: "unknown", + statements: customErrors.length + ? [ + "if (e instanceof XRPCError) {", + ...customErrors.map( + (err) => `if (e.error === '${err.name}') return new ${err.cls}(e)`, + ), + "}", + "return e", + ] + : ["return e"], + }); +} diff --git a/lex-gen/codegen/common.ts b/lex-gen/codegen/common.ts new file mode 100644 index 0000000..d8c642c --- /dev/null +++ b/lex-gen/codegen/common.ts @@ -0,0 +1,299 @@ +import { + type Project, + type SourceFile, + VariableDeclarationKind, +} from "ts-morph"; +import type { LexiconDoc } from "@atp/lexicon"; +import type { GeneratedFile } from "../types.ts"; +import type { CodeGenOptions } from "./util.ts"; +import { format, type Options as PrettierOptions } from "prettier"; + +const PRETTIER_OPTS: PrettierOptions = { + parser: "typescript", + tabWidth: 2, + semi: false, + singleQuote: true, + trailingComma: "all", +}; + +export const utilTs = ( + project: Project, +) => + gen(project, "/util.ts", (file) => { + file.replaceWithText(` +import type { ValidationResult } from '@atp/lexicon' + +export type OmitKey = { + [K2 in keyof T as K2 extends K ? never : K2]: T[K2] +} + +export type $Typed = V & { $type: T } +export type Un$Typed = OmitKey + +export type $Type = Hash extends 'main' + ? Id + : \`\${Id}#\${Hash}\` + +function isObject(v: V): v is V & object { + return v != null && typeof v === 'object' +} + +function is$type( + $type: unknown, + id: Id, + hash: Hash, +): $type is $Type { + return hash === 'main' + ? $type === id + : // $type === \`\${id}#\${hash}\` + typeof $type === 'string' && + $type.length === id.length + 1 + hash.length && + $type.charCodeAt(id.length) === 35 /* '#' */ && + $type.startsWith(id) && + $type.endsWith(hash) +} +${ + /** + * The construct below allows to properly distinguish open unions. Consider + * the following example: + * + * ```ts + * type Foo = { $type?: $Type<'foo', 'main'>; foo: string } + * type Bar = { $type?: $Type<'bar', 'main'>; bar: string } + * type OpenFooBarUnion = $Typed | $Typed | { $type: string } + * ``` + * + * In the context of lexicons, when there is a open union as shown above, the + * if `$type` if either `foo` or `bar`, then the object IS of type `Foo` or + * `Bar`. + * + * ```ts + * declare const obj1: OpenFooBarUnion + * if (is$typed(obj1, 'foo', 'main')) { + * obj1.$type // $Type<'foo', 'main'> + * obj1.foo // string + * } + * ``` + * + * Similarly, if an object is of type `unknown`, then the `is$typed` function + * should only return assurance about the `$type` property, which is what it + * actually checks: + * + * ```ts + * declare const obj2: unknown + * if (is$typed(obj2, 'foo', 'main')) { + * obj2.$type // $Type<'foo', 'main'> + * // @ts-expect-error + * obj2.foo + * } + * ``` + * + * The construct bellow is what makes these two scenarios possible. + */ + ""} +export type $TypedObject = V extends { + $type: $Type +} + ? V + : V extends { $type?: string } + ? V extends { $type?: infer T extends $Type } + ? V & { $type: T } + : never + : V & { $type: $Type } + +export function is$typed( + v: V, + id: Id, + hash: Hash, +): v is $TypedObject { + return isObject(v) && '$type' in v && is$type(v.$type, id, hash) +} + +export function maybe$typed( + v: V, + id: Id, + hash: Hash, +): v is V & object & { $type?: $Type } { + return ( + isObject(v) && + ('$type' in v + ? v.$type === undefined || is$type(v.$type, id, hash) + : true) + ) +} + +export type Validator = (v: unknown) => ValidationResult +export type ValidatorParam = + V extends Validator ? R : never + +/** + * Utility function that allows to convert a "validate*" utility function into a + * type predicate. + */ +export function asPredicate(validate: V) { + return function (v: T): v is T & ValidatorParam { + return validate(v).success + } +} +`); + }); + +export const lexiconsTs = ( + project: Project, + lexiconDocs: LexiconDoc[], + options?: CodeGenOptions, +) => + gen(project, "/lexicons.ts", (file) => { + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); + const nsidToEnum = (nsid: string): string => { + return nsid + .split(".") + .map((word) => word[0].toUpperCase() + word.slice(1)) + .join(""); + }; + + //= import { type LexiconDoc, Lexicons } from '@atp/lexicon' + file + .addImportDeclaration({ + moduleSpecifier: "@atp/lexicon", + }) + .addNamedImports([ + { name: "LexiconDoc", isTypeOnly: true }, + { name: "Lexicons" }, + { name: "ValidationError" }, + { name: "ValidationResult", isTypeOnly: true }, + ]); + + //= import { is$typed, maybe$typed, type $Typed } from "./util${extension}" + file + .addImportDeclaration({ moduleSpecifier: `./util${importExtension}` }) + .addNamedImports([ + { name: "is$typed" }, + { name: "maybe$typed" }, + ]); + + //= export const schemaDict = {...} as const satisfies Record + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: "schemaDict", + initializer: JSON.stringify( + lexiconDocs.reduce( + (acc, cur) => ({ + ...acc, + [nsidToEnum(cur.id)]: cur, + }), + {}, + ), + null, + 2, + ) + " as Record", + }, + ], + }); + + //= export const schemas = Object.values(schemaDict) satisfies LexiconDoc[] + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: "schemas", + initializer: "Object.values(schemaDict) satisfies LexiconDoc[]", + }, + ], + }); + + //= export const lexicons: Lexicons = new Lexicons(schemas) + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: "lexicons", + type: "Lexicons", + initializer: "new Lexicons(schemas)", + }, + ], + }); + + file.addFunction({ + isExported: true, + name: "validate", + overloads: [ + { + typeParameters: ["T extends { $type: string }"], + parameters: [ + { name: "v", type: "unknown" }, + { name: "id", type: "string" }, + { name: "hash", type: "string" }, + { name: "requiredType", type: "true" }, + ], + returnType: "ValidationResult", + }, + { + typeParameters: ["T extends { $type?: string }"], + parameters: [ + { name: "v", type: "unknown" }, + { name: "id", type: "string" }, + { name: "hash", type: "string" }, + { name: "requiredType", type: "false", hasQuestionToken: true }, + ], + returnType: "ValidationResult", + }, + ], + parameters: [ + { name: "v", type: "unknown" }, + { name: "id", type: "string" }, + { name: "hash", type: "string" }, + { name: "requiredType", type: "boolean", hasQuestionToken: true }, + ], + statements: [ + // If $type is present, make sure it is valid before validating the rest of the object + "return (requiredType ? is$typed : maybe$typed)(v, id, hash) ? lexicons.validate(`${id}#${hash}`, v) : { success: false, error: new ValidationError(`Must be an object with \"${hash === 'main' ? id : `${id}#${hash}`}\" $type property`) }", + ], + returnType: "ValidationResult", + }); + + //= export const ids = {...} + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: "ids", + initializer: `{${ + lexiconDocs + .map( + (lex) => + `\n ${nsidToEnum(lex.id)}: ${JSON.stringify(lex.id)},`, + ) + .join("") + }\n} as const`, + }, + ], + }); + }); + +export async function gen( + project: Project, + path: string, + gen: (file: SourceFile) => void | Promise, +): Promise { + const file = project.createSourceFile(path); + gen(file); + await file.save(); // Save in the "in memory" file system + let content = `${banner()}${file.getFullText()}`; + if (!(typeof Deno !== "undefined")) { + content = await format(content, PRETTIER_OPTS); + } + + return { path, content }; +} + +function banner() { + return `/**\n * GENERATED CODE - DO NOT MODIFY\n */\n`; +} diff --git a/lex-gen/codegen/lex-gen.ts b/lex-gen/codegen/lex-gen.ts new file mode 100644 index 0000000..efb98a4 --- /dev/null +++ b/lex-gen/codegen/lex-gen.ts @@ -0,0 +1,1060 @@ +import { relative as getRelativePath } from "@std/path"; +import { type JSDoc, type SourceFile, VariableDeclarationKind } from "ts-morph"; +import type { + LexArray, + LexBlob, + LexBytes, + LexCidLink, + Lexicons, + LexIpldType, + LexObject, + LexPrimitive, + LexToken, +} from "@atp/lexicon"; +import { + type CodeGenOptions, + toCamelCase, + toScreamingSnakeCase, + toTitleCase, +} from "./util.ts"; +import type { LexiconDoc, LexUserType } from "@atp/lexicon"; +import type { ImportMapping } from "../types.ts"; + +interface Commentable { + addJsDoc: ({ description }: { description: string }) => JSDoc; +} +export function genComment( + commentable: T, + def: { description?: string }, +): T { + if (def.description) { + commentable.addJsDoc({ description: def.description }); + } + return commentable; +} + +export function genCommonImports( + file: SourceFile, + baseNsid: string, + lexiconDoc: LexiconDoc, + options?: CodeGenOptions, +) { + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); + const needsBlobRef = Object.values(lexiconDoc.defs).some((def: LexUserType) => + def.type === "blob" || + (def.type === "object" && + Object.values((def as LexObject).properties || {}).some((prop) => + "type" in prop && (prop.type === "blob" || + (prop.type === "array" && "items" in prop && + prop.items.type === "blob")) + )) || + (def.type === "array" && def.items.type === "blob") || + // Check record schema for blobs + (def.type === "record" && + Object.values(def.record.properties || {}).some((prop) => + "type" in prop && (prop.type === "blob" || + (prop.type === "array" && "items" in prop && + prop.items.type === "blob")) + )) || + // Check output schema for blobs + (def.type === "query" || def.type === "procedure") && + def.output?.schema?.type === "object" && + Object.values(def.output.schema.properties || {}).some((prop) => + "type" in prop && (prop.type === "blob" || + (prop.type === "array" && "items" in prop && + prop.items.type === "blob")) + ) + ); + + const needsCID = Object.values(lexiconDoc.defs).some((def: LexUserType) => + def.type === "cid-link" || + (def.type === "object" && + Object.values((def as LexObject).properties || {}).some((prop) => + "type" in prop && prop.type === "cid-link" + )) || + (def.type === "array" && def.items.type === "cid-link") || + // Check record schema for cid-links + (def.type === "record" && + Object.values(def.record.properties || {}).some((prop) => + "type" in prop && (prop.type === "cid-link" || + (prop.type === "array" && "items" in prop && + prop.items.type === "cid-link")) + )) || + // Check output schema for cid-links + (def.type === "query" || def.type === "procedure") && + def.output?.schema?.type === "object" && + Object.values(def.output.schema.properties || {}).some((prop) => + "type" in prop && (prop.type === "cid-link" || + (prop.type === "array" && "items" in prop && + prop.items.type === "cid-link")) + ) + ); + + const needsTypedValidation = Object.values(lexiconDoc.defs).some(( + def: LexUserType, + ) => def.type === "record" || def.type === "object"); + + const needsId = Object.values(lexiconDoc.defs).some(( + def: LexUserType, + ) => def.type === "token") || needsTypedValidation; + + const needsUnionType = Object.values(lexiconDoc.defs).some( + (def: LexUserType) => { + // Check direct array unions + if (def.type === "array" && def.items.type === "union") return true; + + // Check object property unions + if (def.type === "object") { + return Object.values((def as LexObject).properties || {}).some((prop) => + prop.type === "union" || + (prop.type === "array" && prop.items?.type === "union") + ); + } + + // Check record property unions + if (def.type === "record") { + return Object.values(def.record.properties || {}).some((prop) => + "type" in prop && ( + prop.type === "union" || + (prop.type === "array" && "items" in prop && + prop.items.type === "union") + ) + ); + } + + // Check procedure input/output schemas + if (def.type === "procedure") { + // Check input schema + if (def.input?.schema?.type === "union") return true; + if (def.input?.schema?.type === "object") { + return Object.values(def.input.schema.properties || {}).some((prop) => + "type" in prop && ( + prop.type === "union" || + (prop.type === "array" && "items" in prop && + prop.items.type === "union") + ) + ); + } + // Check output schema + if (def.output?.schema?.type === "union") return true; + if (def.output?.schema?.type === "object") { + return Object.values(def.output.schema.properties || {}).some(( + prop, + ) => + "type" in prop && ( + prop.type === "union" || + (prop.type === "array" && "items" in prop && + prop.items.type === "union") + ) + ); + } + } + + // Check query output schemas + if (def.type === "query") { + if (def.output?.schema?.type === "union") return true; + if (def.output?.schema?.type === "object") { + return Object.values(def.output.schema.properties || {}).some(( + prop, + ) => + "type" in prop && ( + prop.type === "union" || + (prop.type === "array" && "items" in prop && + prop.items.type === "union") + ) + ); + } + } + + // Check subscription message schemas + if (def.type === "subscription") { + if (def.message?.schema?.type === "union") return true; + if (def.message?.schema?.type === "object") { + return Object.values(def.message.schema.properties || {}).some(( + prop, + ) => + "type" in prop && ( + prop.type === "union" || + (prop.type === "array" && "items" in prop && + prop.items.type === "union") + ) + ); + } + } + + return false; + }, + ); + + //= import {BlobRef} from '@atp/lexicon' + if (needsBlobRef) { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: "@atp/lexicon", + namedImports: [{ name: "BlobRef" }], + }); + } + + //= import {CID} from 'multiformats/cid' + if (needsCID) { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: "multiformats/cid", + namedImports: [{ name: "CID" }], + }); + } + + const utilPath = `${ + baseNsid + .split(".") + .map((_str) => "..") + .join("/") + }/util${importExtension}`; + + if (needsTypedValidation) { + //= import { validate as _validate } from '../../lexicons.ts' + file + .addImportDeclaration({ + moduleSpecifier: `${ + baseNsid + .split(".") + .map((_str) => "..") + .join("/") + }/lexicons${importExtension}`, + }) + .addNamedImports([{ name: "validate", alias: "_validate" }]); + + //= import type { ValidationResult } from '@atp/lexicon' + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: "@atp/lexicon", + namedImports: [{ name: "ValidationResult" }], + }); + + // tsc adds protection against circular imports, which hurts bundle size. + // Since we know that lexicon.ts and util.ts do not depend on the file being + // generated, we can safely bypass this protection. + // Note that we are not using `import * as util from '../../util'` because + // typescript will emit is own helpers for the import, which we want to avoid. + file.addVariableStatement({ + isExported: false, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { name: "is$typed", initializer: "_is$typed" }, + { name: "validate", initializer: "_validate" }, + ], + }); + } + + const utilImports: Array< + { name: string; alias?: string; isTypeOnly?: boolean } + > = []; + if (needsTypedValidation) { + utilImports.push({ name: "is$typed", alias: "_is$typed" }); + } + if (needsUnionType) { + utilImports.push({ name: "$Typed", isTypeOnly: true }); + } + + if (utilImports.length > 0) { + const allTypeOnly = utilImports.every((imp) => imp.isTypeOnly); + if (allTypeOnly) { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: utilPath, + namedImports: utilImports.map((imp) => ({ + name: imp.name, + alias: imp.alias, + })), + }); + } else { + file + .addImportDeclaration({ + moduleSpecifier: utilPath, + }) + .addNamedImports(utilImports); + } + } + + if (needsId) { + //= const id = "{baseNsid}" + file.addVariableStatement({ + isExported: false, // Do not export to allow tree-shaking + declarationKind: VariableDeclarationKind.Const, + declarations: [{ name: "id", initializer: JSON.stringify(baseNsid) }], + }); + } +} + +export function collectExternalImports( + lexiconDocs: LexiconDoc[], + options?: CodeGenOptions, +): Map> { + const imports: Map> = new Map(); + const mappings = options?.mappings; + + // Check if any records exist (which use ATP_METHODS) + const hasRecords = lexiconDocs.some((lexiconDoc) => + Object.values(lexiconDoc.defs).some((def) => def.type === "record") + ); + + // Record classes use ATP_METHODS which may need external imports + // Note: put is commented out in genRecordCls, so we don't import it + if (hasRecords) { + const atpMethods = [ + "com.atproto.repo.listRecords", + "com.atproto.repo.getRecord", + "com.atproto.repo.createRecord", + "com.atproto.repo.deleteRecord", + ]; + for (const methodNsid of atpMethods) { + const mapping = resolveExternalImport(methodNsid, mappings); + if (mapping) { + if (!imports.has(methodNsid)) { + imports.set(methodNsid, new Set()); + } + // These methods use QueryParams, InputSchema, etc. + imports.get(methodNsid)!.add("main"); + } + } + } + return imports; +} + +export function genImports( + file: SourceFile, + imports: Map>, + baseNsid: string, + options?: CodeGenOptions, +) { + const startPath = "/" + baseNsid.split(".").slice(0, -1).join("/"); + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); + const mappings = options?.mappings; + + for (const [nsid, types] of imports) { + const mapping = resolveExternalImport(nsid, mappings); + if (mapping) { + if (typeof mapping.imports === "string") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: mapping.imports, + namedImports: [{ name: toTitleCase(nsid), isTypeOnly: true }], + }); + } else { + const result = mapping.imports(nsid); + if (result.type === "namespace") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namespaceImport: toTitleCase(nsid), + }); + } else { + const namedImports = Array.from(types).map((typeName) => ({ + name: toTitleCase(typeName), + isTypeOnly: true, + })); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namedImports, + }); + } + } + } else { + const targetPath = "/" + nsid.split(".").join("/") + importExtension; + let resolvedPath = getRelativePath(startPath, targetPath); + if (!resolvedPath.startsWith(".")) { + resolvedPath = `./${resolvedPath}`; + } + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: resolvedPath, + namespaceImport: toTitleCase(nsid), + }); + } + } +} + +export function genUserType( + file: SourceFile, + imports: Map>, + lexicons: Lexicons, + lexUri: string, + options?: CodeGenOptions, +) { + const def = lexicons.getDefOrThrow(lexUri); + switch (def.type) { + case "array": + genArray(file, imports, lexUri, def, options); + break; + case "token": + genToken(file, lexUri, def); + break; + case "object": { + const ifaceName: string = toTitleCase(getHash(lexUri)); + genObject(file, imports, lexUri, def, ifaceName, { + typeProperty: true, + }, options); + genObjHelpers(file, lexUri, ifaceName, { + requireTypeProperty: false, + }); + break; + } + + case "blob": + case "bytes": + case "cid-link": + case "boolean": + case "integer": + case "string": + case "unknown": + genPrimitiveOrBlob(file, lexUri, def); + break; + + default: + throw new Error( + `genLexUserType() called with wrong definition type (${def.type}) in ${lexUri}`, + ); + } +} + +function genObject( + file: SourceFile, + imports: Map>, + lexUri: string, + def: LexObject, + ifaceName: string, + { + defaultsArePresent = true, + allowUnknownProperties = false, + typeProperty = false, + }: { + defaultsArePresent?: boolean; + allowUnknownProperties?: boolean; + typeProperty?: boolean | "required"; + } = {}, + options?: CodeGenOptions, +) { + const iface = file.addInterface({ + name: ifaceName, + isExported: true, + }); + genComment(iface, def); + + if (typeProperty) { + const hash = getHash(lexUri); + const baseNsid = stripScheme(stripHash(lexUri)); + + //= $type?: + iface.addProperty({ + name: typeProperty === "required" ? `$type` : `$type?`, + type: + // Not using $Type here because it is less readable than a plain string + // `$Type<${JSON.stringify(baseNsid)}, ${JSON.stringify(hash)}>` + hash === "main" + ? JSON.stringify(`${baseNsid}`) + : JSON.stringify(`${baseNsid}#${hash}`), + }); + } + + const nullableProps = new Set(def.nullable); + if (def.properties) { + for (const propKey in def.properties) { + const propDef = def.properties[propKey]; + const propNullable = nullableProps.has(propKey); + const req = def.required?.includes(propKey) || + (defaultsArePresent && + "default" in propDef && + propDef.default !== undefined); + if (propDef.type === "ref" || propDef.type === "union") { + //= propName: External|External + const types = propDef.type === "union" + ? propDef.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ) + : [ + refToType( + propDef.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ), + ]; + if (propDef.type === "union" && !propDef.closed) { + types.push("{ $type: string }"); + } + iface.addProperty({ + name: `${propKey}${req ? "" : "?"}`, + type: makeType(types, { nullable: propNullable }), + }); + continue; + } else { + if (propDef.type === "array") { + //= propName: type[] + let propAst; + if (propDef.items.type === "ref") { + propAst = iface.addProperty({ + name: `${propKey}${req ? "" : "?"}`, + type: makeType( + refToType( + propDef.items.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ), + { + nullable: propNullable, + array: true, + }, + ), + }); + } else if (propDef.items.type === "union") { + const types = propDef.items.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ); + if (!propDef.items.closed) { + types.push("{ $type: string }"); + } + propAst = iface.addProperty({ + name: `${propKey}${req ? "" : "?"}`, + type: makeType(types, { + nullable: propNullable, + array: true, + }), + }); + } else { + propAst = iface.addProperty({ + name: `${propKey}${req ? "" : "?"}`, + type: makeType(primitiveOrBlobToType(propDef.items), { + nullable: propNullable, + array: true, + }), + }); + } + genComment(propAst, propDef); + } else { + //= propName: type + genComment( + iface.addProperty({ + name: `${propKey}${req ? "" : "?"}`, + type: makeType(primitiveOrBlobToType(propDef), { + nullable: propNullable, + }), + }), + propDef, + ); + } + } + } + + if (allowUnknownProperties) { + //= [k: string]: unknown + iface.addIndexSignature({ + keyName: "k", + keyType: "string", + returnType: "unknown", + }); + } + } +} + +export function genToken(file: SourceFile, lexUri: string, def: LexToken) { + //= /** */ + //= export const = `${id}#` + genComment( + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: toScreamingSnakeCase(getHash(lexUri)), + type: "string", + initializer: `\`\${id}#${getHash(lexUri)}\``, + }, + ], + }), + def, + ); +} + +export function genArray( + file: SourceFile, + imports: Map>, + lexUri: string, + def: LexArray, + options?: CodeGenOptions, +) { + if (def.items.type === "ref") { + file.addTypeAlias({ + name: toTitleCase(getHash(lexUri)), + type: `${ + refToType( + def.items.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ) + }[]`, + isExported: true, + }); + } else if (def.items.type === "union") { + const types = def.items.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ); + if (!def.items.closed) { + types.push("{ $type: string }"); + } + file.addTypeAlias({ + name: toTitleCase(getHash(lexUri)), + type: `(${types.join("|")})[]`, + isExported: true, + }); + } else { + genComment( + file.addTypeAlias({ + name: toTitleCase(getHash(lexUri)), + type: `${primitiveOrBlobToType(def.items)}[]`, + isExported: true, + }), + def, + ); + } +} + +export function genPrimitiveOrBlob( + file: SourceFile, + lexUri: string, + def: LexPrimitive | LexBlob | LexIpldType, +) { + genComment( + file.addTypeAlias({ + name: toTitleCase(getHash(lexUri)), + type: primitiveOrBlobToType(def), + isExported: true, + }), + def, + ); +} + +export function genXrpcParams( + file: SourceFile, + lexicons: Lexicons, + lexUri: string, + defaultsArePresent = true, +) { + const def = lexicons.getDefOrThrow(lexUri, [ + "query", + "subscription", + "procedure", + ]); + + // @NOTE We need to use a `type` here instead of an `interface` because we + // need the generated type to be used as generic type parameter like this: + // + // type QueryParams = {} // Generated by this function + // + // type MyUtil

= (...) + // type NsType = MyUtil // ERROR if `NS.QueryParams` is an `interface` + // + // Second line will fail if `NS.QueryParams` is an `interface` that does + // not explicitly extend `xrpcServer.QueryParam`, or have a string index + // signature that encompasses `xrpcServer.QueryParam`. + + //= export type QueryParams = {...} + if ( + def.parameters && def.parameters.properties && + Object.keys(def.parameters.properties).length > 0 + ) { + genComment( + file.addTypeAlias({ + name: "QueryParams", + isExported: true, + type: `{ + ${ + Object.entries(def.parameters.properties) + .map(([paramKey, paramDef]) => { + const req = def.parameters!.required?.includes(paramKey) || + (defaultsArePresent && + "default" in paramDef && + paramDef.default !== undefined); + const jsDoc = paramDef.description + ? `/** ${paramDef.description} */\n` + : ""; + return `${jsDoc}${paramKey}${req ? "" : "?"}: ${ + paramDef.type === "array" + ? primitiveToType(paramDef.items) + "[]" + : primitiveToType(paramDef) + }`; + }) + .join("\n") + } + }`, + }), + def.parameters, + ); + } else { + file.addTypeAlias({ + name: "QueryParams", + isExported: true, + type: "globalThis.Record", + }); + } +} + +export function genXrpcInput( + file: SourceFile, + imports: Map>, + lexicons: Lexicons, + lexUri: string, + defaultsArePresent = true, + options?: CodeGenOptions, +) { + const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); + + if (def.type === "procedure" && def.input?.schema) { + if (def.input.schema.type === "ref" || def.input.schema.type === "union") { + //= export type InputSchema = ... + + const types = def.input.schema.type === "union" + ? def.input.schema.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ) + : [ + refToType( + def.input.schema.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ), + ]; + + if (def.input.schema.type === "union" && !def.input.schema.closed) { + types.push("{ $type: string }"); + } + file.addTypeAlias({ + name: "InputSchema", + type: types.join("|"), + isExported: true, + }); + } else { + //= export interface InputSchema {...} + genObject(file, imports, lexUri, def.input.schema, `InputSchema`, { + defaultsArePresent, + }, options); + } + } else if (def.type === "procedure" && def.input?.encoding) { + //= export type InputSchema = string | Uint8Array | Blob + file.addTypeAlias({ + isExported: true, + name: "InputSchema", + type: "string | Uint8Array | Blob", + }); + } else { + //= export type InputSchema = undefined + file.addTypeAlias({ + isExported: true, + name: "InputSchema", + type: "undefined", + }); + } +} + +export function genXrpcOutput( + file: SourceFile, + imports: Map>, + lexicons: Lexicons, + lexUri: string, + defaultsArePresent = true, + options?: CodeGenOptions, +) { + const def = lexicons.getDefOrThrow(lexUri, [ + "query", + "subscription", + "procedure", + ]); + + const schema = def.type === "subscription" + ? def.message?.schema + : def.output?.schema; + if (schema) { + if (schema.type === "ref" || schema.type === "union") { + //= export type OutputSchema = ... + const types = schema.type === "union" + ? schema.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ) + : [ + refToType( + schema.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ), + ]; + if (schema.type === "union" && !schema.closed) { + types.push("{ $type: string }"); + } + file.addTypeAlias({ + name: "OutputSchema", + type: types.join("|"), + isExported: true, + }); + } else { + // Check if schema is empty (no properties) + const isEmpty = !schema.properties || + Object.keys(schema.properties).length === 0; + if (isEmpty) { + //= export type OutputSchema = Record + file.addTypeAlias({ + name: "OutputSchema", + type: "globalThis.Record", + isExported: true, + }); + } else { + //= export interface OutputSchema {...} + genObject(file, imports, lexUri, schema, `OutputSchema`, { + defaultsArePresent, + }, options); + } + } + } +} + +export function genRecord( + file: SourceFile, + imports: Map>, + lexicons: Lexicons, + lexUri: string, + options?: CodeGenOptions, +) { + const def = lexicons.getDefOrThrow(lexUri, ["record"]); + + //= export interface Record {...} + genObject(file, imports, lexUri, def.record, "Record", { + defaultsArePresent: true, + allowUnknownProperties: true, + typeProperty: "required", + }, options); + + //= export function isRecord(v: unknown): v is Record {...} + genObjHelpers(file, lexUri, "Record", { + requireTypeProperty: true, + }); + + const hash = getHash(lexUri); + if (hash === "main") { + //= export type Main = Record + file.addTypeAlias({ + name: "Main", + type: "Record", + isExported: true, + }); + } +} + +function genObjHelpers( + file: SourceFile, + lexUri: string, + ifaceName: string, + { + requireTypeProperty, + }: { + requireTypeProperty: boolean; + }, +) { + const hash = getHash(lexUri); + + const hashVar = `hash${ifaceName}`; + + file.addVariableStatement({ + isExported: false, + declarationKind: VariableDeclarationKind.Const, + declarations: [{ name: hashVar, initializer: JSON.stringify(hash) }], + }); + + const isX = toCamelCase(`is-${ifaceName}`); + + //= export function is{X}(v: V): v is {ifaceName} & V {...} + file + .addFunction({ + name: isX, + typeParameters: [{ name: `V` }], + parameters: [{ name: `v`, type: `V` }], + returnType: `v is ${ifaceName} & V`, + isExported: true, + }) + .setBodyText(`return is$typed(v, id, ${hashVar})`); + + const validateX = toCamelCase(`validate-${ifaceName}`); + + //= export function validate{X}(v: V): ValidationResult<{ifaceName} & V> {...} + file + .addFunction({ + name: validateX, + typeParameters: [{ name: `V` }], + parameters: [{ name: `v`, type: `V` }], + returnType: `ValidationResult<${ifaceName} & V>`, + isExported: true, + }) + .setBodyText( + `return validate<${ifaceName} & V>(v, id, ${hashVar}${ + requireTypeProperty ? ", true" : "" + })`, + ); +} + +export function stripScheme(uri: string): string { + if (uri.startsWith("lex:")) return uri.slice(4); + return uri; +} + +export function stripHash(uri: string): string { + return uri.split("#")[0] || ""; +} + +export function getHash(uri: string): string { + return uri.split("#").pop() || ""; +} + +export function ipldToType(def: LexCidLink | LexBytes) { + if (def.type === "bytes") { + return "Uint8Array"; + } + return "CID"; +} + +function refToUnionType( + ref: string, + lexUri: string, + imports: Map>, + mappings?: ImportMapping[], +): string { + const baseNsid = stripScheme(stripHash(lexUri)); + return `$Typed<${refToType(ref, baseNsid, imports, mappings)}>`; +} + +export function resolveExternalImport( + nsid: string, + mappings?: ImportMapping[], +): ImportMapping | undefined { + if (!mappings) return undefined; + return mappings.find((mapping) => { + return mapping.nsid.some((pattern) => { + if (pattern.endsWith(".*")) { + return nsid.startsWith(pattern.slice(0, -1)); + } + return nsid === pattern; + }); + }); +} + +function refToType( + ref: string, + baseNsid: string, + imports: Map>, + mappings?: ImportMapping[], +): string { + let [refBase, refHash] = ref.split("#"); + refBase = stripScheme(refBase); + if (!refHash) refHash = "main"; + + // internal + if (!refBase || baseNsid === refBase) { + return toTitleCase(refHash); + } + + // external - check if there's a mapping + const mapping = resolveExternalImport(refBase, mappings); + if (mapping) { + if (!imports.has(refBase)) { + imports.set(refBase, new Set()); + } + const types = imports.get(refBase)!; + types.add(refHash); + + if (typeof mapping.imports === "string") { + // String mapping means namespace import + return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; + } else { + const result = mapping.imports(refBase); + if (result.type === "namespace") { + return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; + } else { + // Named import - return just the type name + return toTitleCase(refHash); + } + } + } + + // external - no mapping, use relative import + if (!imports.has(refBase)) { + imports.set(refBase, new Set()); + } + return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; +} + +export function primitiveOrBlobToType( + def: LexBlob | LexPrimitive | LexIpldType, +): string { + switch (def.type) { + case "blob": + return "BlobRef"; + case "bytes": + return "Uint8Array"; + case "cid-link": + return "CID"; + default: + return primitiveToType(def); + } +} + +export function primitiveToType(def: LexPrimitive): string { + switch (def.type) { + case "string": + if (def.knownValues?.length) { + return `${ + def.knownValues + .map((v) => JSON.stringify(v)) + .join(" | ") + } | (string & globalThis.Record)`; + } else if (def.enum) { + return def.enum.map((v) => JSON.stringify(v)).join(" | "); + } else if (def.const) { + return JSON.stringify(def.const); + } + return "string"; + case "integer": + if (def.enum) { + return def.enum.map((v) => JSON.stringify(v)).join(" | "); + } else if (def.const) { + return JSON.stringify(def.const); + } + return "number"; + case "boolean": + if (def.const) { + return JSON.stringify(def.const); + } + return "boolean"; + case "unknown": + // @TODO Should we use "object" here ? + // the "Record" identifier from typescript get overwritten by the Record + // interface created by lex-cli. + return "{ [_ in string]: unknown }"; // Record + default: + throw new Error(`Unexpected primitive type: ${JSON.stringify(def)}`); + } +} + +function makeType( + _types: string | string[], + opts?: { array?: boolean; nullable?: boolean }, +) { + const types = ([] as string[]).concat(_types); + if (opts?.nullable) types.push("null"); + const arr = opts?.array ? "[]" : ""; + if (types.length === 1) return `(${types[0]})${arr}`; + if (arr) return `(${types.join(" | ")})${arr}`; + return types.join(" | "); +} diff --git a/lex-gen/codegen/server.ts b/lex-gen/codegen/server.ts new file mode 100644 index 0000000..1ac8ac2 --- /dev/null +++ b/lex-gen/codegen/server.ts @@ -0,0 +1,503 @@ +import { + IndentationText, + Project, + type SourceFile, + VariableDeclarationKind, +} from "ts-morph"; +import { type LexiconDoc, Lexicons } from "@atp/lexicon"; +import { NSID } from "@atp/syntax"; +import type { GeneratedAPI } from "../types.ts"; +import { gen, lexiconsTs, utilTs } from "./common.ts"; +import { + collectExternalImports, + genCommonImports, + genImports, + genRecord, + genUserType, + genXrpcInput, + genXrpcOutput, + genXrpcParams, + resolveExternalImport, +} from "./lex-gen.ts"; +import { + type CodeGenOptions, + type DefTreeNode, + lexiconsToDefTree, + schemasToNsidTokens, + toCamelCase, + toScreamingSnakeCase, + toTitleCase, +} from "./util.ts"; + +export async function genServerApi( + lexiconDocs: LexiconDoc[], + options?: CodeGenOptions, +): Promise { + const project = new Project({ + useInMemoryFileSystem: true, + manipulationSettings: { indentationText: IndentationText.TwoSpaces }, + }); + const api: GeneratedAPI = { files: [] }; + const lexicons = new Lexicons(lexiconDocs); + const nsidTree = lexiconsToDefTree(lexiconDocs); + const nsidTokens = schemasToNsidTokens(lexiconDocs); + for (const lexiconDoc of lexiconDocs) { + api.files.push(await lexiconTs(project, lexicons, lexiconDoc, options)); + } + api.files.push(await utilTs(project)); + api.files.push(await lexiconsTs(project, lexiconDocs)); + api.files.push( + await indexTs(project, lexiconDocs, nsidTree, nsidTokens, options), + ); + return api; +} + +const indexTs = ( + project: Project, + lexiconDocs: LexiconDoc[], + nsidTree: DefTreeNode[], + nsidTokens: Record, + options?: CodeGenOptions, +) => + gen(project, "/index.ts", (file) => { + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); + + // Check if there are any subscription types + const hasSubscriptions = lexiconDocs.some((doc) => + doc.defs.main?.type === "subscription" + ); + + //= import {createServer as createXrpcServer, Server as XrpcServer} from '@atp/xrpc-server' + const namedImports = [ + { name: "Auth", isTypeOnly: true }, + { name: "Options", alias: "XrpcOptions", isTypeOnly: true }, + { name: "Server", alias: "XrpcServer", isTypeOnly: true }, + { name: "MethodConfigOrHandler", isTypeOnly: true }, + { name: "createServer", alias: "createXrpcServer" }, + ]; + + if (hasSubscriptions) { + namedImports.splice(3, 0, { + name: "StreamConfigOrHandler", + isTypeOnly: true, + }); + } + + file.addImportDeclaration({ + moduleSpecifier: "@atp/xrpc-server", + namedImports, + }); + //= import {schemas} from './lexicons.ts' + file + .addImportDeclaration({ + moduleSpecifier: `./lexicons${importExtension}`, + }) + .addNamedImport({ + name: "schemas", + }); + + // collect and import external lexicon references + const externalImports = collectExternalImports(lexiconDocs, options); + const mappings = options?.mappings; + for (const [nsid, types] of externalImports) { + const mapping = resolveExternalImport(nsid, mappings); + if (mapping) { + if (typeof mapping.imports === "string") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: mapping.imports, + namedImports: [{ name: toTitleCase(nsid), isTypeOnly: true }], + }); + } else { + const result = mapping.imports(nsid); + if (result.type === "namespace") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namespaceImport: toTitleCase(nsid), + }); + } else { + const namedImports = Array.from(types).map((typeName) => ({ + name: toTitleCase(typeName), + isTypeOnly: true, + })); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namedImports, + }); + } + } + } + } + + // generate type imports + for (const lexiconDoc of lexiconDocs) { + if ( + lexiconDoc.defs.main?.type !== "query" && + lexiconDoc.defs.main?.type !== "subscription" && + lexiconDoc.defs.main?.type !== "procedure" + ) { + continue; + } + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: `./types/${ + lexiconDoc.id.split(".").join("/") + }${importExtension}`, + namespaceImport: toTitleCase(lexiconDoc.id), + }); + } + + // generate token enums + for (const nsidAuthority in nsidTokens) { + // export const {THE_AUTHORITY} = { + // {Name}: "{authority.the.name}" + // } + file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: toScreamingSnakeCase(nsidAuthority), + initializer: [ + "{", + ...nsidTokens[nsidAuthority].map( + (nsidName) => + `${toTitleCase(nsidName)}: "${nsidAuthority}.${nsidName}",`, + ), + "}", + ].join("\n"), + }, + ], + }); + } + + //= export function createServer(options?: XrpcOptions) { ... } + const createServerFn = file.addFunction({ + name: "createServer", + returnType: "Server", + parameters: [ + { name: "options", type: "XrpcOptions", hasQuestionToken: true }, + ], + isExported: true, + }); + createServerFn.setBodyText(`return new Server(options)`); + + //= export class Server {...} + const serverCls = file.addClass({ + name: "Server", + isExported: true, + }); + //= xrpc: XrpcServer = createXrpcServer(methodSchemas) + serverCls.addProperty({ + name: "xrpc", + type: "XrpcServer", + }); + + // generate classes for the schemas + for (const ns of nsidTree) { + //= ns: NS + serverCls.addProperty({ + name: ns.propName, + type: ns.className, + }); + + // class... + genNamespaceCls(file, ns); + } + + //= constructor (options?: XrpcOptions) { + //= this.xrpc = createXrpcServer(schemas, options) + //= {namespace declarations} + //= } + serverCls + .addConstructor({ + parameters: [ + { name: "options", type: "XrpcOptions", hasQuestionToken: true }, + ], + }) + .setBodyText( + [ + "this.xrpc = createXrpcServer(schemas, options)", + ...nsidTree.map( + (ns) => `this.${ns.propName} = new ${ns.className}(this)`, + ), + ].join("\n"), + ); + }); + +function genNamespaceCls(file: SourceFile, ns: DefTreeNode) { + //= export class {ns}NS {...} + const cls = file.addClass({ + name: ns.className, + isExported: true, + }); + //= _server: Server + cls.addProperty({ + name: "_server", + type: "Server", + }); + + for (const child of ns.children) { + //= child: ChildNS + cls.addProperty({ + name: child.propName, + type: child.className, + }); + + // recurse + genNamespaceCls(file, child); + } + + //= constructor(server: Server) { + //= this._server = server + //= {child namespace declarations} + //= } + const cons = cls.addConstructor(); + cons.addParameter({ + name: "server", + type: "Server", + }); + cons.setBodyText( + [ + `this._server = server`, + ...ns.children.map( + (ns) => `this.${ns.propName} = new ${ns.className}(server)`, + ), + ].join("\n"), + ); + + // methods + for (const userType of ns.userTypes) { + if ( + userType.def.type !== "query" && + userType.def.type !== "subscription" && + userType.def.type !== "procedure" + ) { + continue; + } + const moduleName = toTitleCase(userType.nsid); + const name = toCamelCase(NSID.parse(userType.nsid).name || ""); + const isSubscription = userType.def.type === "subscription"; + const method = cls.addMethod({ + name, + typeParameters: [ + { + name: "A", + constraint: "Auth", + default: "void", + }, + ], + }); + method.addParameter({ + name: "cfg", + type: isSubscription + ? `StreamConfigOrHandler< + A, + ${moduleName}.QueryParams, + ${moduleName}.HandlerOutput, + >` + : `MethodConfigOrHandler< + A, + ${moduleName}.QueryParams, + ${moduleName}.HandlerInput, + ${moduleName}.HandlerOutput, + >`, + }); + const methodType = isSubscription ? "streamMethod" : "method"; + method.setBodyText( + [ + `const nsid = '${userType.nsid}' // @ts-ignore - dynamically generated`, + `return this._server.xrpc.${methodType}(nsid, cfg)`, + ].join("\n"), + ); + } +} + +const lexiconTs = ( + project: Project, + lexicons: Lexicons, + lexiconDoc: LexiconDoc, + options?: CodeGenOptions, +) => + gen( + project, + `/types/${lexiconDoc.id.split(".").join("/")}.ts`, + (file) => { + const main = lexiconDoc.defs.main; + if (main?.type === "query" || main?.type === "procedure") { + const streamingInput = main?.type === "procedure" && + main.input?.encoding && + !main.input.schema; + const streamingOutput = main.output?.encoding && !main.output.schema; + if (streamingInput || streamingOutput) { + //= ReadableStream is a web standard API + // No import needed for ReadableStream + } + } + + genCommonImports(file, lexiconDoc.id, lexiconDoc); + + const imports: Map> = new Map(); + for (const defId in lexiconDoc.defs) { + const def = lexiconDoc.defs[defId]; + const lexUri = `${lexiconDoc.id}#${defId}`; + if (defId === "main") { + if (def.type === "query" || def.type === "procedure") { + genXrpcParams(file, lexicons, lexUri); + genXrpcInput(file, imports, lexicons, lexUri, false, options); + genXrpcOutput(file, imports, lexicons, lexUri, false, options); + genServerXrpcMethod(file, lexicons, lexUri); + } else if (def.type === "subscription") { + genXrpcParams(file, lexicons, lexUri); + genXrpcOutput(file, imports, lexicons, lexUri, false, options); + genServerXrpcStreaming(file, lexicons, lexUri); + } else if (def.type === "record") { + genRecord(file, imports, lexicons, lexUri, options); + } else { + genUserType(file, imports, lexicons, lexUri, options); + } + } else { + genUserType(file, imports, lexicons, lexUri, options); + } + } + genImports(file, imports, lexiconDoc.id, options); + }, + ); + +function genServerXrpcMethod( + file: SourceFile, + lexicons: Lexicons, + lexUri: string, +) { + const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); + + //= export interface HandlerInput {...} + if (def.type === "procedure" && def.input?.encoding) { + const handlerInput = file.addInterface({ + name: "HandlerInput", + isExported: true, + }); + + handlerInput.addProperty({ + name: "encoding", + type: def.input.encoding + .split(",") + .map((v) => `'${v.trim()}'`) + .join(" | "), + }); + handlerInput.addProperty({ + name: "body", + type: def.input.schema + ? def.input.encoding.includes(",") + ? "InputSchema | ReadableStream" + : "InputSchema" + : "ReadableStream", + }); + } else { + file.addTypeAlias({ + isExported: true, + name: "HandlerInput", + type: "void", + }); + } + + // export interface HandlerSuccess {...} + let hasHandlerSuccess = false; + if (def.output?.schema || def.output?.encoding) { + hasHandlerSuccess = true; + const handlerSuccess = file.addInterface({ + name: "HandlerSuccess", + isExported: true, + }); + + if (def.output.encoding) { + handlerSuccess.addProperty({ + name: "encoding", + type: def.output.encoding + .split(",") + .map((v) => `'${v.trim()}'`) + .join(" | "), + }); + } + if (def.output?.schema) { + if (def.output.encoding.includes(",")) { + handlerSuccess.addProperty({ + name: "body", + type: "OutputSchema | Uint8Array | ReadableStream", + }); + } else { + handlerSuccess.addProperty({ name: "body", type: "OutputSchema" }); + } + } else if (def.output?.encoding) { + handlerSuccess.addProperty({ + name: "body", + type: "Uint8Array | ReadableStream", + }); + } + handlerSuccess.addProperty({ + name: "headers?", + type: "{ [key: string]: string }", + }); + } + + // export interface HandlerError {...} + const handlerError = file.addInterface({ + name: "HandlerError", + isExported: true, + }); + handlerError.addProperties([ + { name: "status", type: "number" }, + { name: "message?", type: "string" }, + ]); + if (def.errors?.length) { + handlerError.addProperty({ + name: "error?", + type: def.errors.map((err) => `'${err.name}'`).join(" | "), + }); + } + + // export type HandlerOutput = ... + file.addTypeAlias({ + isExported: true, + name: "HandlerOutput", + type: `HandlerError | ${hasHandlerSuccess ? "HandlerSuccess" : "void"}`, + }); +} + +function genServerXrpcStreaming( + file: SourceFile, + lexicons: Lexicons, + lexUri: string, +) { + const def = lexicons.getDefOrThrow(lexUri, ["subscription"]); + + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: "@atp/xrpc-server", + namedImports: [{ name: "ErrorFrame" }], + }); + + // export type HandlerError = ... + file.addTypeAlias({ + name: "HandlerError", + isExported: true, + type: `ErrorFrame<${arrayToUnion(def.errors?.map((e) => e.name))}>`, + }); + + // export type HandlerOutput = ... + file.addTypeAlias({ + isExported: true, + name: "HandlerOutput", + type: `HandlerError | ${def.message?.schema ? "OutputSchema" : "void"}`, + }); +} + +function arrayToUnion(arr?: string[]) { + if (!arr?.length) { + return "never"; + } + return arr.map((item) => `'${item}'`).join(" | "); +} diff --git a/lex-gen/codegen/util.ts b/lex-gen/codegen/util.ts new file mode 100644 index 0000000..170c768 --- /dev/null +++ b/lex-gen/codegen/util.ts @@ -0,0 +1,108 @@ +import type { LexiconDoc, LexUserType } from "@atp/lexicon"; +import { NSID } from "@atp/syntax"; +import type { ImportMapping } from "../types.ts"; + +export interface CodeGenOptions { + useJsExtension?: boolean; + importSuffix?: string; + mappings?: ImportMapping[]; +} + +export interface DefTreeNodeUserType { + nsid: string; + def: LexUserType; +} + +export interface DefTreeNode { + name: string; + className: string; + propName: string; + children: DefTreeNode[]; + userTypes: DefTreeNodeUserType[]; +} + +export function lexiconsToDefTree(lexicons: LexiconDoc[]): DefTreeNode[] { + const tree: DefTreeNode[] = []; + for (const lexicon of lexicons) { + if (!lexicon.defs.main) { + continue; + } + const node = getOrCreateNode(tree, lexicon.id.split(".").slice(0, -1)); + node.userTypes.push({ nsid: lexicon.id, def: lexicon.defs.main }); + } + return tree; +} + +function getOrCreateNode(tree: DefTreeNode[], path: string[]): DefTreeNode { + let node: DefTreeNode | undefined; + for (let i = 0; i < path.length; i++) { + const segment = path[i]; + node = tree.find((v) => v.name === segment); + if (!node) { + node = { + name: segment, + className: `${toTitleCase(path.slice(0, i + 1).join("-"))}NS`, + propName: toCamelCase(segment), + children: [], + userTypes: [], + } as DefTreeNode; + tree.push(node); + } + tree = node.children; + } + if (!node) throw new Error(`Invalid schema path: ${path.join(".")}`); + return node; +} + +export function schemasToNsidTokens( + lexiconDocs: LexiconDoc[], +): Record { + const nsidTokens: Record = {}; + for (const lexiconDoc of lexiconDocs) { + const nsidp = NSID.parse(lexiconDoc.id); + if (!nsidp.name) continue; + for (const defId in lexiconDoc.defs) { + const def = lexiconDoc.defs[defId]; + if (def.type !== "token") continue; + const authority = nsidp.segments.slice(0, -1).join("."); + nsidTokens[authority] ??= []; + nsidTokens[authority].push( + nsidp.name + (defId === "main" ? "" : `#${defId}`), + ); + } + } + return nsidTokens; +} + +export function toTitleCase(v: string): string { + v = v.replace(/^([a-z])/gi, (_, g) => g.toUpperCase()); // upper-case first letter + v = v.replace(/[.#-]([a-z])/gi, (_, g) => g.toUpperCase()); // uppercase any dash, dot, or hash segments + return v.replace(/[.-]/g, ""); // remove lefover dashes or dots +} + +export function toCamelCase(v: string): string { + v = v.replace(/[.#-]([a-z])/gi, (_, g) => g.toUpperCase()); // uppercase any dash, dot, or hash segments + return v.replace(/[.-]/g, ""); // remove lefover dashes or dots +} + +export function toScreamingSnakeCase(v: string): string { + v = v.replace(/[.#-]+/gi, "_"); // convert dashes, dots, and hashes into underscores + return v.toUpperCase(); // and scream! +} + +export async function formatGeneratedFiles(outDir: string) { + console.log("Formatting generated files..."); + const cmd = new Deno.Command("deno", { + args: ["fmt", outDir], + cwd: Deno.cwd(), + }); + + const { code, stderr } = await cmd.output(); + + if (code !== 0) { + const errorMsg = new TextDecoder().decode(stderr); + console.warn(`Warning: deno fmt failed: ${errorMsg}`); + } else { + console.log("Files formatted successfully."); + } +} diff --git a/lex-gen/config.ts b/lex-gen/config.ts new file mode 100644 index 0000000..dfb684e --- /dev/null +++ b/lex-gen/config.ts @@ -0,0 +1,142 @@ +import { NSID } from "@atp/syntax"; +import { parse } from "@std/jsonc"; +import type { LexiconConfig } from "./types.ts"; + +function isValidLexiconPattern(pattern: string): boolean { + if (pattern.endsWith(".*")) { + try { + NSID.parse(`${pattern.slice(0, -2)}.x`); + return true; + } catch { + return false; + } + } + return NSID.isValid(pattern); +} + +function validateConfig(config: LexiconConfig): void { + if (!config.outdir || config.outdir.length === 0) { + throw new Error("outdir must not be empty"); + } + + if (!config.files || config.files.length === 0) { + throw new Error("files must include at least one glob pattern"); + } + + for (const file of config.files) { + if (!file || file.length === 0) { + throw new Error("files must not contain empty strings"); + } + } + + if (config.mappings) { + for (const mapping of config.mappings) { + if (!mapping.nsid || mapping.nsid.length === 0) { + throw new Error("mappings.nsid requires at least one pattern"); + } + + for (const pattern of mapping.nsid) { + if (!isValidLexiconPattern(pattern)) { + throw new Error( + `invalid NSID pattern: ${pattern} (must be valid NSID or end with .*)`, + ); + } + } + + if (typeof mapping.imports === "string") { + if (mapping.imports.length === 0) { + throw new Error("mappings.imports must not be empty"); + } + } else if (typeof mapping.imports !== "function") { + throw new Error("mappings.imports must be a string or function"); + } + } + } + + if (config.modules?.importSuffix !== undefined) { + if (config.modules.importSuffix.length === 0) { + throw new Error("modules.importSuffix must not be empty"); + } + } + + if (config.pull) { + if (!config.pull.outdir || config.pull.outdir.length === 0) { + throw new Error("pull.outdir must not be empty"); + } + + if (!config.pull.sources || config.pull.sources.length === 0) { + throw new Error("pull.sources must include at least one source"); + } + + for (const source of config.pull.sources) { + if (source.type === "git") { + if (!source.remote || source.remote.length === 0) { + throw new Error("pull.sources[].remote must not be empty"); + } + + if (source.ref !== undefined && source.ref.length === 0) { + throw new Error("pull.sources[].ref must not be empty"); + } + + if (!source.pattern || source.pattern.length === 0) { + throw new Error( + "pull.sources[].pattern must include at least one glob pattern", + ); + } + + for (const pattern of source.pattern) { + if (!pattern || pattern.length === 0) { + throw new Error( + "pull.sources[].pattern must not contain empty strings", + ); + } + } + } + } + } +} + +export function defineLexiconConfig(config: LexiconConfig): LexiconConfig { + validateConfig(config); + return config; +} + +export async function loadLexiconConfig( + configPath?: string, +): Promise { + if (!configPath) { + const possiblePaths = [ + "./lexicon.config.json", + "./lexicon.config.jsonc", + ]; + for (const path of possiblePaths) { + try { + if (typeof Deno !== "undefined") { + const stat = Deno.statSync(path); + if (stat.isFile) { + configPath = path; + break; + } + } + } catch { + continue; + } + } + } + + if (!configPath) { + return null; + } + + try { + const content = typeof Deno !== "undefined" + ? Deno.readTextFileSync(configPath) + : (await import("node:fs")).readFileSync(configPath, "utf-8"); + + const parsed = parse(content) as unknown as LexiconConfig; + return defineLexiconConfig(parsed); + } catch (error) { + console.warn(`Failed to load config from ${configPath}:`, error); + return null; + } +} diff --git a/lex-gen/deno.json b/lex-gen/deno.json index 4748353..864d88c 100644 --- a/lex-gen/deno.json +++ b/lex-gen/deno.json @@ -9,6 +9,8 @@ "@std/fs": "jsr:@std/fs@^1.0.19", "@std/jsonc": "jsr:@std/jsonc@^1.0.1", "@std/path": "jsr:@std/path@^1.1.2", - "ts-morph": "jsr:@ts-morph/ts-morph@^26.0.0" + "prettier": "npm:prettier@^3.6.2", + "ts-morph": "jsr:@ts-morph/ts-morph@^26.0.0", + "zod": "jsr:@zod/zod@^4.1.11" } } diff --git a/lex-gen/mdgen/index.ts b/lex-gen/mdgen/index.ts new file mode 100644 index 0000000..312f96e --- /dev/null +++ b/lex-gen/mdgen/index.ts @@ -0,0 +1,78 @@ +import { readFileSync } from "@std/fs/unstable-read-file"; +import { writeFileSync } from "@std/fs/unstable-write-file"; +import type { LexiconDoc } from "@atp/lexicon"; + +const INSERT_START = [ + "", + "", +]; +const INSERT_END = [ + "", +]; + +export async function process(outFilePath: string, lexicons: LexiconDoc[]) { + let existingContent = ""; + try { + existingContent = new TextDecoder().decode(readFileSync(outFilePath)); + } catch { + // ignore - no existing content + } + const fileLines: StringTree = existingContent.split("\n"); + + // find previously generated content + let startIndex = fileLines.findIndex((line) => matchesStart(line as string)); + let endIndex = fileLines.findIndex((line) => matchesEnd(line as string)); + if (startIndex === -1) { + startIndex = fileLines.length; + } + if (endIndex === -1) { + endIndex = fileLines.length; + } + + // generate & insert content + fileLines.splice(startIndex, endIndex - startIndex + 1, [ + INSERT_START, + await genMdLines(lexicons), + INSERT_END, + ]); + + writeFileSync(outFilePath, new TextEncoder().encode(merge(fileLines))); +} + +function genMdLines(lexicons: LexiconDoc[]): StringTree { + const doc: StringTree = []; + for (const lexicon of lexicons) { + console.log(lexicon.id); + const desc: StringTree = []; + if (lexicon.description) { + desc.push(lexicon.description, ``); + } + doc.push([ + `---`, + ``, + `## ${lexicon.id}`, + "", + desc, + "```json", + JSON.stringify(lexicon, null, 2), + "```", + ]); + } + return doc; +} + +type StringTree = (StringTree | string | undefined)[]; +function merge(arr: StringTree): string { + return arr + .flat(10) + .filter((v) => typeof v === "string") + .join("\n"); +} + +function matchesStart(line: string) { + return /