diff --git a/lex-gen/cmd/gen-api.ts b/lex-gen/cmd/gen-api.ts index a11d70a..f4b3e2a 100644 --- a/lex-gen/cmd/gen-api.ts +++ b/lex-gen/cmd/gen-api.ts @@ -7,28 +7,67 @@ import { } from "../util.ts"; import { genClientApi } from "../codegen/client.ts"; import { formatGeneratedFiles } from "../codegen/util.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; const command = new Command() .description("Generate a TS client API") .option("--js", "use .js extension for imports instead of .ts") - .option("-o, --outdir ", "dir path to write to", { required: true }) - .option("-i, --input ", "paths of lexicon files to include", { - required: true, - }) + .option("-o, --outdir ", "dir path to write to") + .option("-i, --input ", "paths of lexicon files to include") + .option("--config ", "path to config file") .action( - async ({ outdir, input, js }) => { - const lexicons = readAllLexicons(input); + async ({ outdir, input, js, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalOutdir = outdir ?? config?.outdir; + const finalInput = input ?? config?.files; + + if (!finalOutdir) { + console.error("outdir is required (provide via -o/--outdir or config)"); + if (typeof Deno !== "undefined") { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalInput || finalInput.length === 0) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (typeof Deno !== "undefined") { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (config?.pull) { + await pullLexicons(config.pull); + } + + const useJs = js ?? false; + const importSuffix = config?.modules?.importSuffix; + const mappings = config?.mappings; + const lexicons = readAllLexicons(finalInput); const api = await genClientApi(lexicons, { - useJsExtension: js, + useJsExtension: useJs, + importSuffix: importSuffix, + mappings: mappings, }); - const diff = genFileDiff(outdir, api); + const diff = genFileDiff(finalOutdir, api); console.log("This will write the following files:"); printFileDiff(diff); applyFileDiff(diff); if (typeof Deno !== "undefined") { - await formatGeneratedFiles(outdir); + await formatGeneratedFiles(finalOutdir); } console.log("API generated."); + + if (config?.pull) { + cleanupPullDirectory(config.pull); + } }, ); diff --git a/lex-gen/cmd/gen-md.ts b/lex-gen/cmd/gen-md.ts index 6918683..d4820eb 100644 --- a/lex-gen/cmd/gen-md.ts +++ b/lex-gen/cmd/gen-md.ts @@ -1,19 +1,47 @@ import { Command } from "@cliffy/command"; import { readAllLexicons } from "../util.ts"; import * as mdGen from "../mdgen/index.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; import process from "node:process"; const isDeno = typeof Deno !== "undefined"; const command = new Command() .description("Generate markdown documentation") - .option("-o, --output ", "Output file path", { required: true }) - .option("-i, --input ", "Input file path", { required: true }) + .option("-o, --output ", "Output file path") + .option("-i, --input ", "Input file path") + .option("--config ", "path to config file") .action( - async ({ output, input }) => { - if (!output.endsWith(".md")) { + async ({ output, input, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalOutput = output ?? + (config?.outdir ? `${config.outdir}/docs.md` : undefined); + const finalInput = input ?? config?.files?.[0]; + + if (!finalOutput) { + console.error("output is required (provide via -o/--output or config)"); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalInput) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalOutput.endsWith(".md")) { console.error( - "Must supply the path to a .md file as the first parameter", + "Must supply the path to a .md file", ); if (isDeno) { Deno.exit(1); @@ -21,8 +49,17 @@ const command = new Command() process.exit(1); } } - const lexicons = readAllLexicons(input); - await mdGen.process(output, lexicons); + + if (config?.pull) { + await pullLexicons(config.pull); + } + + const lexicons = readAllLexicons(finalInput); + await mdGen.process(finalOutput, lexicons); + + if (config?.pull) { + cleanupPullDirectory(config.pull); + } }, ); diff --git a/lex-gen/cmd/gen-server.ts b/lex-gen/cmd/gen-server.ts index 81f632a..7ead8c0 100644 --- a/lex-gen/cmd/gen-server.ts +++ b/lex-gen/cmd/gen-server.ts @@ -7,30 +7,71 @@ import { } from "../util.ts"; import { formatGeneratedFiles } from "../codegen/util.ts"; import { genServerApi } from "../codegen/server.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; + +const isDeno = typeof Deno !== "undefined"; const command = new Command() .description("Generate a TS server API") .option("--js", "use .js extension for imports instead of .ts") - .option("-o, --outdir ", "dir path to write to", { required: true }) - .option("-i, --input ", "paths of lexicon files to include", { - required: true, - }) + .option("-o, --outdir ", "dir path to write to") + .option("-i, --input ", "paths of lexicon files to include") + .option("--config ", "path to config file") .action( - async ({ outdir, input, js }) => { + async ({ outdir, input, js, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalOutdir = outdir ?? config?.outdir; + const finalInput = input ?? config?.files; + + if (!finalOutdir) { + console.error("outdir is required (provide via -o/--outdir or config)"); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (!finalInput || finalInput.length === 0) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (config?.pull) { + await pullLexicons(config.pull); + } + + const useJs = js ?? false; + const importSuffix = config?.modules?.importSuffix; + const mappings = config?.mappings; console.log("Generating API..."); - const lexicons = readAllLexicons(input); + const lexicons = readAllLexicons(finalInput); const api = await genServerApi(lexicons, { - useJsExtension: js, + useJsExtension: useJs, + importSuffix: importSuffix, + mappings: mappings, }); console.log("API generated."); - const diff = genFileDiff(outdir, api); + const diff = genFileDiff(finalOutdir, api); console.log("This will write the following files:"); printFileDiff(diff); applyFileDiff(diff); if (typeof Deno !== "undefined") { - await formatGeneratedFiles(outdir); + await formatGeneratedFiles(finalOutdir); } console.log("API generated."); + + if (config?.pull) { + cleanupPullDirectory(config.pull); + } }, ); diff --git a/lex-gen/cmd/gen-ts-obj.ts b/lex-gen/cmd/gen-ts-obj.ts index 7ff6a8c..2c4159d 100644 --- a/lex-gen/cmd/gen-ts-obj.ts +++ b/lex-gen/cmd/gen-ts-obj.ts @@ -1,14 +1,40 @@ import { Command } from "@cliffy/command"; import { genTsObj, readAllLexicons } from "../util.ts"; +import { loadLexiconConfig } from "../config.ts"; +import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; +import process from "node:process"; + +const isDeno = typeof Deno !== "undefined"; const command = new Command() .description("Generate a TS file that exports an array of lexicons") - .option("-i, --input ", "paths of the lexicon files to include", { - required: true, - }) - .action(({ input }) => { - const lexicons = readAllLexicons(input); + .option("-i, --input ", "paths of the lexicon files to include") + .option("--config ", "path to config file") + .action(async ({ input, config: configPath }) => { + const config = await loadLexiconConfig(configPath); + const finalInput = input ?? config?.files; + + if (!finalInput || finalInput.length === 0) { + console.error( + "input is required (provide via -i/--input or config.files)", + ); + if (isDeno) { + Deno.exit(1); + } else { + process.exit(1); + } + } + + if (config?.pull) { + await pullLexicons(config.pull); + } + + const lexicons = readAllLexicons(finalInput); console.log(genTsObj(lexicons)); + + if (config?.pull) { + cleanupPullDirectory(config.pull); + } }); export default command; diff --git a/lex-gen/codegen/client.ts b/lex-gen/codegen/client.ts index 6712615..1e49858 100644 --- a/lex-gen/codegen/client.ts +++ b/lex-gen/codegen/client.ts @@ -48,7 +48,7 @@ export async function genClientApi( const nsidTree = lexiconsToDefTree(lexiconDocs); const nsidTokens = schemasToNsidTokens(lexiconDocs); for (const lexiconDoc of lexiconDocs) { - api.files.push(await lexiconTs(project, lexicons, lexiconDoc)); + api.files.push(await lexiconTs(project, lexicons, lexiconDoc, options)); } api.files.push(await utilTs(project)); api.files.push(await lexiconsTs(project, lexiconDocs, options)); @@ -66,7 +66,8 @@ const indexTs = ( options?: CodeGenOptions, ) => gen(project, "/index.ts", (file) => { - const extension = options?.useJsExtension ? ".js" : ".ts"; + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); //= import { XrpcClient, type FetchHandler, type FetchHandlerOptions } from '@atp/xrpc' const xrpcImport = file.addImportDeclaration({ moduleSpecifier: "@atp/xrpc", @@ -78,7 +79,7 @@ const indexTs = ( ]); //= import {schemas} from './lexicons.ts' file - .addImportDeclaration({ moduleSpecifier: `./lexicons${extension}` }) + .addImportDeclaration({ moduleSpecifier: `./lexicons${importExtension}` }) .addNamedImports([{ name: "schemas" }]); // Check if any lexicon docs use cid-link types @@ -110,7 +111,7 @@ const indexTs = ( //= import { type OmitKey, type Un$Typed } from './util.ts' file - .addImportDeclaration({ moduleSpecifier: `./util${extension}` }) + .addImportDeclaration({ moduleSpecifier: `./util${importExtension}` }) .addNamedImports([ { name: "OmitKey", isTypeOnly: true }, { name: "Un$Typed", isTypeOnly: true }, @@ -120,7 +121,7 @@ const indexTs = ( for (const lexicon of lexiconDocs) { const moduleSpecifier = `./types/${ lexicon.id.split(".").join("/") - }${extension}`; + }${importExtension}`; file .addImportDeclaration({ moduleSpecifier }) .setNamespaceImport(toTitleCase(lexicon.id)); @@ -476,6 +477,7 @@ const lexiconTs = ( project: Project, lexicons: Lexicons, lexiconDoc: LexiconDoc, + options?: CodeGenOptions, ) => gen( project, @@ -499,28 +501,28 @@ const lexiconTs = ( genCommonImports(file, lexiconDoc.id, lexiconDoc); - const imports: Set = new Set(); + const imports: Map> = new Map(); for (const defId in lexiconDoc.defs) { const def = lexiconDoc.defs[defId]; const lexUri = `${lexiconDoc.id}#${defId}`; if (defId === "main") { if (def.type === "query" || def.type === "procedure") { genXrpcParams(file, lexicons, lexUri, false); - genXrpcInput(file, imports, lexicons, lexUri, false); - genXrpcOutput(file, imports, lexicons, lexUri); + genXrpcInput(file, imports, lexicons, lexUri, false, options); + genXrpcOutput(file, imports, lexicons, lexUri, false, options); genClientXrpcCommon(file, lexicons, lexUri); } else if (def.type === "subscription") { continue; } else if (def.type === "record") { - genRecord(file, imports, lexicons, lexUri); + genRecord(file, imports, lexicons, lexUri, options); } else { - genUserType(file, imports, lexicons, lexUri); + genUserType(file, imports, lexicons, lexUri, options); } } else { - genUserType(file, imports, lexicons, lexUri); + genUserType(file, imports, lexicons, lexUri, options); } } - genImports(file, imports, lexiconDoc.id); + genImports(file, imports, lexiconDoc.id, options); return Promise.resolve(); }, ); diff --git a/lex-gen/codegen/common.ts b/lex-gen/codegen/common.ts index cb1c9ca..d8c642c 100644 --- a/lex-gen/codegen/common.ts +++ b/lex-gen/codegen/common.ts @@ -21,7 +21,7 @@ export const utilTs = ( ) => gen(project, "/util.ts", (file) => { file.replaceWithText(` -import { type ValidationResult } from '@atp/lexicon' +import type { ValidationResult } from '@atp/lexicon' export type OmitKey = { [K2 in keyof T as K2 extends K ? never : K2]: T[K2] @@ -144,7 +144,8 @@ export const lexiconsTs = ( options?: CodeGenOptions, ) => gen(project, "/lexicons.ts", (file) => { - const extension = options?.useJsExtension ? ".js" : ".ts"; + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); const nsidToEnum = (nsid: string): string => { return nsid .split(".") @@ -166,7 +167,7 @@ export const lexiconsTs = ( //= import { is$typed, maybe$typed, type $Typed } from "./util${extension}" file - .addImportDeclaration({ moduleSpecifier: `./util${extension}` }) + .addImportDeclaration({ moduleSpecifier: `./util${importExtension}` }) .addNamedImports([ { name: "is$typed" }, { name: "maybe$typed" }, diff --git a/lex-gen/codegen/lex-gen.ts b/lex-gen/codegen/lex-gen.ts index 470f006..3e1b5e0 100644 --- a/lex-gen/codegen/lex-gen.ts +++ b/lex-gen/codegen/lex-gen.ts @@ -18,6 +18,7 @@ import { toTitleCase, } from "./util.ts"; import type { LexiconDoc, LexUserType } from "@atp/lexicon"; +import type { ImportMapping } from "../types.ts"; interface Commentable { addJsDoc: ({ description }: { description: string }) => JSDoc; @@ -38,7 +39,8 @@ export function genCommonImports( lexiconDoc: LexiconDoc, options?: CodeGenOptions, ) { - const extension = options?.useJsExtension ? ".js" : ".ts"; + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); const needsBlobRef = Object.values(lexiconDoc.defs).some((def: LexUserType) => def.type === "blob" || (def.type === "object" && @@ -93,6 +95,10 @@ export function genCommonImports( def: LexUserType, ) => def.type === "record" || def.type === "object"); + const needsId = Object.values(lexiconDoc.defs).some(( + def: LexUserType, + ) => def.type === "token") || needsTypedValidation; + const needsUnionType = Object.values(lexiconDoc.defs).some( (def: LexUserType) => { // Check direct array unions @@ -181,33 +187,31 @@ export function genCommonImports( }, ); - const needsIdConstant = Object.values(lexiconDoc.defs).some(( - def: LexUserType, - ) => - (def.type === "string" && - (def.enum?.length || def.const || def.knownValues?.length)) || - def.type === "record" || - def.type === "object" - ); - //= import {BlobRef} from '@atp/lexicon' if (needsBlobRef) { - file - .addImportDeclaration({ - moduleSpecifier: "@atp/lexicon", - }) - .addNamedImports([{ name: "BlobRef" }]); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: "@atp/lexicon", + namedImports: [{ name: "BlobRef" }], + }); } //= import {CID} from 'multiformats/cid' if (needsCID) { - file - .addImportDeclaration({ - moduleSpecifier: "multiformats/cid", - }) - .addNamedImports([{ name: "CID" }]); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: "multiformats/cid", + namedImports: [{ name: "CID" }], + }); } + const utilPath = `${ + baseNsid + .split(".") + .map((_str) => "..") + .join("/") + }/util${importExtension}`; + if (needsTypedValidation) { //= import { validate as _validate } from '../../lexicons.ts' file @@ -217,24 +221,10 @@ export function genCommonImports( .split(".") .map((_str) => "..") .join("/") - }/lexicons${extension}`, + }/lexicons${importExtension}`, }) .addNamedImports([{ name: "validate", alias: "_validate" }]); - //= import { is$typed as _is$typed } from '../[...]/util.ts' - file - .addImportDeclaration({ - moduleSpecifier: `${ - baseNsid - .split(".") - .map((_str) => "..") - .join("/") - }/util${extension}`, - }) - .addNamedImports([ - { name: "is$typed", alias: "_is$typed" }, - ]); - // tsc adds protection against circular imports, which hurts bundle size. // Since we know that lexicon.ts and util.ts do not depend on the file being // generated, we can safely bypass this protection. @@ -250,7 +240,37 @@ export function genCommonImports( }); } - if (needsIdConstant) { + const utilImports: Array< + { name: string; alias?: string; isTypeOnly?: boolean } + > = []; + if (needsTypedValidation) { + utilImports.push({ name: "is$typed", alias: "_is$typed" }); + } + if (needsUnionType) { + utilImports.push({ name: "$Typed", isTypeOnly: true }); + } + + if (utilImports.length > 0) { + const allTypeOnly = utilImports.every((imp) => imp.isTypeOnly); + if (allTypeOnly) { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: utilPath, + namedImports: utilImports.map((imp) => ({ + name: imp.name, + alias: imp.alias, + })), + }); + } else { + file + .addImportDeclaration({ + moduleSpecifier: utilPath, + }) + .addNamedImports(utilImports); + } + } + + if (needsId) { //= const id = "{baseNsid}" file.addVariableStatement({ isExported: false, // Do not export to allow tree-shaking @@ -258,57 +278,74 @@ export function genCommonImports( declarations: [{ name: "id", initializer: JSON.stringify(baseNsid) }], }); } - - if (needsUnionType) { - //= import { type $Typed } from '../[...]/util.ts' - file - .addImportDeclaration({ - moduleSpecifier: `${ - baseNsid - .split(".") - .map((_str) => "..") - .join("/") - }/util${extension}`, - }) - .addNamedImports([ - { name: "$Typed", isTypeOnly: true }, - ]); - } } export function genImports( file: SourceFile, - imports: Set, + imports: Map>, baseNsid: string, options?: CodeGenOptions, ) { const startPath = "/" + baseNsid.split(".").slice(0, -1).join("/"); - const extension = options?.useJsExtension ? ".js" : ".ts"; - - for (const nsid of imports) { - const targetPath = "/" + nsid.split(".").join("/") + extension; - let resolvedPath = getRelativePath(startPath, targetPath); - if (!resolvedPath.startsWith(".")) { - resolvedPath = `./${resolvedPath}`; + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); + const mappings = options?.mappings; + + for (const [nsid, types] of imports) { + const mapping = resolveExternalImport(nsid, mappings); + if (mapping) { + if (typeof mapping.imports === "string") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: mapping.imports, + namespaceImport: toTitleCase(nsid), + }); + } else { + const result = mapping.imports(nsid); + if (result.type === "namespace") { + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namespaceImport: toTitleCase(nsid), + }); + } else { + const namedImports = Array.from(types).map((typeName) => ({ + name: toTitleCase(typeName), + isTypeOnly: true, + })); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: result.from, + namedImports, + }); + } + } + } else { + const targetPath = "/" + nsid.split(".").join("/") + importExtension; + let resolvedPath = getRelativePath(startPath, targetPath); + if (!resolvedPath.startsWith(".")) { + resolvedPath = `./${resolvedPath}`; + } + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: resolvedPath, + namespaceImport: toTitleCase(nsid), + }); } - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: resolvedPath, - namespaceImport: toTitleCase(nsid), - }); } } export function genUserType( file: SourceFile, - imports: Set, + imports: Map>, lexicons: Lexicons, lexUri: string, + options?: CodeGenOptions, ) { const def = lexicons.getDefOrThrow(lexUri); switch (def.type) { case "array": - genArray(file, imports, lexUri, def); + genArray(file, imports, lexUri, def, options); break; case "token": genToken(file, lexUri, def); @@ -317,7 +354,7 @@ export function genUserType( const ifaceName: string = toTitleCase(getHash(lexUri)); genObject(file, imports, lexUri, def, ifaceName, { typeProperty: true, - }); + }, options); genObjHelpers(file, lexUri, ifaceName, { requireTypeProperty: false, }); @@ -343,7 +380,7 @@ export function genUserType( function genObject( file: SourceFile, - imports: Set, + imports: Map>, lexUri: string, def: LexObject, ifaceName: string, @@ -356,6 +393,7 @@ function genObject( allowUnknownProperties?: boolean; typeProperty?: boolean | "required"; } = {}, + options?: CodeGenOptions, ) { const iface = file.addInterface({ name: ifaceName, @@ -391,8 +429,17 @@ function genObject( if (propDef.type === "ref" || propDef.type === "union") { //= propName: External|External const types = propDef.type === "union" - ? propDef.refs.map((ref) => refToUnionType(ref, lexUri, imports)) - : [refToType(propDef.ref, stripScheme(stripHash(lexUri)), imports)]; + ? propDef.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ) + : [ + refToType( + propDef.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ), + ]; if (propDef.type === "union" && !propDef.closed) { types.push("{ $type: string }"); } @@ -413,6 +460,7 @@ function genObject( propDef.items.ref, stripScheme(stripHash(lexUri)), imports, + options?.mappings, ), { nullable: propNullable, @@ -422,7 +470,7 @@ function genObject( }); } else if (propDef.items.type === "union") { const types = propDef.items.refs.map((ref) => - refToUnionType(ref, lexUri, imports) + refToUnionType(ref, lexUri, imports, options?.mappings) ); if (!propDef.items.closed) { types.push("{ $type: string }"); @@ -490,9 +538,10 @@ export function genToken(file: SourceFile, lexUri: string, def: LexToken) { export function genArray( file: SourceFile, - imports: Set, + imports: Map>, lexUri: string, def: LexArray, + options?: CodeGenOptions, ) { if (def.items.type === "ref") { file.addTypeAlias({ @@ -502,13 +551,14 @@ export function genArray( def.items.ref, stripScheme(stripHash(lexUri)), imports, + options?.mappings, ) }[]`, isExported: true, }); } else if (def.items.type === "union") { const types = def.items.refs.map((ref) => - refToUnionType(ref, lexUri, imports) + refToUnionType(ref, lexUri, imports, options?.mappings) ); if (!def.items.closed) { types.push("{ $type: string }"); @@ -612,10 +662,11 @@ export function genXrpcParams( export function genXrpcInput( file: SourceFile, - imports: Set, + imports: Map>, lexicons: Lexicons, lexUri: string, defaultsArePresent = true, + options?: CodeGenOptions, ) { const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); @@ -625,13 +676,14 @@ export function genXrpcInput( const types = def.input.schema.type === "union" ? def.input.schema.refs.map((ref) => - refToUnionType(ref, lexUri, imports) + refToUnionType(ref, lexUri, imports, options?.mappings) ) : [ refToType( def.input.schema.ref, stripScheme(stripHash(lexUri)), imports, + options?.mappings, ), ]; @@ -647,7 +699,7 @@ export function genXrpcInput( //= export interface InputSchema {...} genObject(file, imports, lexUri, def.input.schema, `InputSchema`, { defaultsArePresent, - }); + }, options); } } else if (def.type === "procedure" && def.input?.encoding) { //= export type InputSchema = string | Uint8Array | Blob @@ -668,10 +720,11 @@ export function genXrpcInput( export function genXrpcOutput( file: SourceFile, - imports: Set, + imports: Map>, lexicons: Lexicons, lexUri: string, defaultsArePresent = true, + options?: CodeGenOptions, ) { const def = lexicons.getDefOrThrow(lexUri, [ "query", @@ -686,8 +739,17 @@ export function genXrpcOutput( if (schema.type === "ref" || schema.type === "union") { //= export type OutputSchema = ... const types = schema.type === "union" - ? schema.refs.map((ref) => refToUnionType(ref, lexUri, imports)) - : [refToType(schema.ref, stripScheme(stripHash(lexUri)), imports)]; + ? schema.refs.map((ref) => + refToUnionType(ref, lexUri, imports, options?.mappings) + ) + : [ + refToType( + schema.ref, + stripScheme(stripHash(lexUri)), + imports, + options?.mappings, + ), + ]; if (schema.type === "union" && !schema.closed) { types.push("{ $type: string }"); } @@ -711,7 +773,7 @@ export function genXrpcOutput( //= export interface OutputSchema {...} genObject(file, imports, lexUri, schema, `OutputSchema`, { defaultsArePresent, - }); + }, options); } } } @@ -719,9 +781,10 @@ export function genXrpcOutput( export function genRecord( file: SourceFile, - imports: Set, + imports: Map>, lexicons: Lexicons, lexUri: string, + options?: CodeGenOptions, ) { const def = lexicons.getDefOrThrow(lexUri, ["record"]); @@ -730,12 +793,22 @@ export function genRecord( defaultsArePresent: true, allowUnknownProperties: true, typeProperty: "required", - }); + }, options); //= export function isRecord(v: unknown): v is Record {...} genObjHelpers(file, lexUri, "Record", { requireTypeProperty: true, }); + + const hash = getHash(lexUri); + if (hash === "main") { + //= export type Main = Record + file.addTypeAlias({ + name: "Main", + type: "Record", + isExported: true, + }); + } } function genObjHelpers( @@ -810,18 +883,34 @@ export function ipldToType(def: LexCidLink | LexBytes) { function refToUnionType( ref: string, lexUri: string, - imports: Set, + imports: Map>, + mappings?: ImportMapping[], ): string { const baseNsid = stripScheme(stripHash(lexUri)); - return `$Typed<${refToType(ref, baseNsid, imports)}>`; + return `$Typed<${refToType(ref, baseNsid, imports, mappings)}>`; +} + +function resolveExternalImport( + nsid: string, + mappings?: ImportMapping[], +): ImportMapping | undefined { + if (!mappings) return undefined; + return mappings.find((mapping) => { + return mapping.nsid.some((pattern) => { + if (pattern.endsWith(".*")) { + return nsid.startsWith(pattern.slice(0, -1)); + } + return nsid === pattern; + }); + }); } function refToType( ref: string, baseNsid: string, - imports: Set, + imports: Map>, + mappings?: ImportMapping[], ): string { - // TODO: import external types! let [refBase, refHash] = ref.split("#"); refBase = stripScheme(refBase); if (!refHash) refHash = "main"; @@ -831,8 +920,33 @@ function refToType( return toTitleCase(refHash); } - // external - imports.add(refBase); + // external - check if there's a mapping + const mapping = resolveExternalImport(refBase, mappings); + if (mapping) { + if (!imports.has(refBase)) { + imports.set(refBase, new Set()); + } + const types = imports.get(refBase)!; + types.add(refHash); + + if (typeof mapping.imports === "string") { + // String mapping means namespace import + return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; + } else { + const result = mapping.imports(refBase); + if (result.type === "namespace") { + return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; + } else { + // Named import - return just the type name + return toTitleCase(refHash); + } + } + } + + // external - no mapping, use relative import + if (!imports.has(refBase)) { + imports.set(refBase, new Set()); + } return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; } diff --git a/lex-gen/codegen/server.ts b/lex-gen/codegen/server.ts index c329c05..2aba4d4 100644 --- a/lex-gen/codegen/server.ts +++ b/lex-gen/codegen/server.ts @@ -58,7 +58,8 @@ const indexTs = ( options?: CodeGenOptions, ) => gen(project, "/index.ts", (file) => { - const extension = options?.useJsExtension ? ".js" : ".ts"; + const importExtension = options?.importSuffix ?? + (options?.useJsExtension ? ".js" : ".ts"); // Check if there are any subscription types const hasSubscriptions = lexiconDocs.some((doc) => @@ -69,7 +70,7 @@ const indexTs = ( const namedImports = [ { name: "Auth", isTypeOnly: true }, { name: "Options", alias: "XrpcOptions", isTypeOnly: true }, - { name: "Server", alias: "XrpcServer" }, + { name: "Server", alias: "XrpcServer", isTypeOnly: true }, { name: "MethodConfigOrHandler", isTypeOnly: true }, { name: "createServer", alias: "createXrpcServer" }, ]; @@ -88,7 +89,7 @@ const indexTs = ( //= import {schemas} from './lexicons.ts' file .addImportDeclaration({ - moduleSpecifier: "./lexicons.ts", + moduleSpecifier: `./lexicons${importExtension}`, }) .addNamedImport({ name: "schemas", @@ -103,13 +104,13 @@ const indexTs = ( ) { continue; } - file - .addImportDeclaration({ - moduleSpecifier: `./types/${ - lexiconDoc.id.split(".").join("/") - }${extension}`, - }) - .setNamespaceImport(toTitleCase(lexiconDoc.id)); + file.addImportDeclaration({ + isTypeOnly: true, + moduleSpecifier: `./types/${ + lexiconDoc.id.split(".").join("/") + }${importExtension}`, + namespaceImport: toTitleCase(lexiconDoc.id), + }); } // generate token enums @@ -286,9 +287,7 @@ const lexiconTs = ( ) => gen( project, - `/types/${lexiconDoc.id.split(".").join("/")}${ - options?.useJsExtension ? ".js" : ".ts" - }`, + `/types/${lexiconDoc.id.split(".").join("/")}.ts`, (file) => { const main = lexiconDoc.defs.main; if (main?.type === "query" || main?.type === "procedure") { @@ -304,27 +303,27 @@ const lexiconTs = ( genCommonImports(file, lexiconDoc.id, lexiconDoc); - const imports: Set = new Set(); + const imports: Map> = new Map(); for (const defId in lexiconDoc.defs) { const def = lexiconDoc.defs[defId]; const lexUri = `${lexiconDoc.id}#${defId}`; if (defId === "main") { if (def.type === "query" || def.type === "procedure") { genXrpcParams(file, lexicons, lexUri); - genXrpcInput(file, imports, lexicons, lexUri); - genXrpcOutput(file, imports, lexicons, lexUri, false); + genXrpcInput(file, imports, lexicons, lexUri, false, options); + genXrpcOutput(file, imports, lexicons, lexUri, false, options); genServerXrpcMethod(file, lexicons, lexUri); } else if (def.type === "subscription") { genXrpcParams(file, lexicons, lexUri); - genXrpcOutput(file, imports, lexicons, lexUri, false); + genXrpcOutput(file, imports, lexicons, lexUri, false, options); genServerXrpcStreaming(file, lexicons, lexUri); } else if (def.type === "record") { - genRecord(file, imports, lexicons, lexUri); + genRecord(file, imports, lexicons, lexUri, options); } else { - genUserType(file, imports, lexicons, lexUri); + genUserType(file, imports, lexicons, lexUri, options); } } else { - genUserType(file, imports, lexicons, lexUri); + genUserType(file, imports, lexicons, lexUri, options); } } genImports(file, imports, lexiconDoc.id, options); @@ -439,6 +438,7 @@ function genServerXrpcStreaming( const def = lexicons.getDefOrThrow(lexUri, ["subscription"]); file.addImportDeclaration({ + isTypeOnly: true, moduleSpecifier: "@atp/xrpc-server", namedImports: [{ name: "ErrorFrame" }], }); diff --git a/lex-gen/codegen/util.ts b/lex-gen/codegen/util.ts index 0ad89bb..170c768 100644 --- a/lex-gen/codegen/util.ts +++ b/lex-gen/codegen/util.ts @@ -1,8 +1,11 @@ import type { LexiconDoc, LexUserType } from "@atp/lexicon"; import { NSID } from "@atp/syntax"; +import type { ImportMapping } from "../types.ts"; export interface CodeGenOptions { useJsExtension?: boolean; + importSuffix?: string; + mappings?: ImportMapping[]; } export interface DefTreeNodeUserType { diff --git a/lex-gen/config.ts b/lex-gen/config.ts new file mode 100644 index 0000000..b148894 --- /dev/null +++ b/lex-gen/config.ts @@ -0,0 +1,151 @@ +import { NSID } from "@atp/syntax"; +import type { LexiconConfig } from "./types.ts"; + +function isValidLexiconPattern(pattern: string): boolean { + if (pattern.endsWith(".*")) { + try { + NSID.parse(`${pattern.slice(0, -2)}.x`); + return true; + } catch { + return false; + } + } + return NSID.isValid(pattern); +} + +function validateConfig(config: LexiconConfig): void { + if (!config.outdir || config.outdir.length === 0) { + throw new Error("outdir must not be empty"); + } + + if (!config.files || config.files.length === 0) { + throw new Error("files must include at least one glob pattern"); + } + + for (const file of config.files) { + if (!file || file.length === 0) { + throw new Error("files must not contain empty strings"); + } + } + + if (config.mappings) { + for (const mapping of config.mappings) { + if (!mapping.nsid || mapping.nsid.length === 0) { + throw new Error("mappings.nsid requires at least one pattern"); + } + + for (const pattern of mapping.nsid) { + if (!isValidLexiconPattern(pattern)) { + throw new Error( + `invalid NSID pattern: ${pattern} (must be valid NSID or end with .*)`, + ); + } + } + + if (typeof mapping.imports === "string") { + if (mapping.imports.length === 0) { + throw new Error("mappings.imports must not be empty"); + } + } else if (typeof mapping.imports !== "function") { + throw new Error("mappings.imports must be a string or function"); + } + } + } + + if (config.modules?.importSuffix !== undefined) { + if (config.modules.importSuffix.length === 0) { + throw new Error("modules.importSuffix must not be empty"); + } + } + + if (config.pull) { + if (!config.pull.outdir || config.pull.outdir.length === 0) { + throw new Error("pull.outdir must not be empty"); + } + + if (!config.pull.sources || config.pull.sources.length === 0) { + throw new Error("pull.sources must include at least one source"); + } + + for (const source of config.pull.sources) { + if (source.type === "git") { + if (!source.remote || source.remote.length === 0) { + throw new Error("pull.sources[].remote must not be empty"); + } + + if (source.ref !== undefined && source.ref.length === 0) { + throw new Error("pull.sources[].ref must not be empty"); + } + + if (!source.pattern || source.pattern.length === 0) { + throw new Error( + "pull.sources[].pattern must include at least one glob pattern", + ); + } + + for (const pattern of source.pattern) { + if (!pattern || pattern.length === 0) { + throw new Error( + "pull.sources[].pattern must not contain empty strings", + ); + } + } + } + } + } +} + +export function defineLexiconConfig(config: LexiconConfig): LexiconConfig { + validateConfig(config); + return config; +} + +export async function loadLexiconConfig( + configPath?: string, +): Promise { + if (!configPath) { + const possiblePaths = [ + "./lexicon.config.ts", + "./lexicon.config.js", + "./lexicon.config.json", + ]; + for (const path of possiblePaths) { + try { + if (typeof Deno !== "undefined") { + const stat = Deno.statSync(path); + if (stat.isFile) { + configPath = path; + break; + } + } + } catch { + continue; + } + } + } + + if (!configPath) { + return null; + } + + try { + if (configPath.endsWith(".json")) { + const content = Deno.readTextFileSync(configPath); + const parsed = JSON.parse(content); + return defineLexiconConfig(parsed); + } else { + const module = await import( + new URL(configPath, `file://${Deno.cwd()}/`).href + ); + const config = module.default ?? module.config; + if (typeof config === "function") { + return defineLexiconConfig(config()); + } else { + return defineLexiconConfig(config); + } + } + } catch (error) { + console.warn(`Failed to load config from ${configPath}:`, error); + return null; + } +} diff --git a/lex-gen/mod.ts b/lex-gen/mod.ts index 2791e56..42eeef2 100644 --- a/lex-gen/mod.ts +++ b/lex-gen/mod.ts @@ -33,8 +33,19 @@ */ import { Command } from "@cliffy/command"; import { genApi, genMd, genServer, genTsObj } from "./cmd/index.ts"; +import { defineLexiconConfig, loadLexiconConfig } from "./config.ts"; import process from "node:process"; +export { defineLexiconConfig, loadLexiconConfig }; +export type { + GitSourceConfig, + ImportMapping, + LexiconConfig, + ModulesConfig, + PullConfig, + SourceConfig, +} from "./types.ts"; + const isDeno = typeof Deno !== "undefined"; await new Command() diff --git a/lex-gen/pull.ts b/lex-gen/pull.ts new file mode 100644 index 0000000..3130285 --- /dev/null +++ b/lex-gen/pull.ts @@ -0,0 +1,163 @@ +import { join } from "@std/path"; +import { existsSync } from "@std/fs"; +import { removeSync } from "@std/fs/unstable-remove"; +import { mkdirSync } from "@std/fs/unstable-mkdir"; +import { readFileSync } from "@std/fs/unstable-read-file"; +import { writeFileSync } from "@std/fs/unstable-write-file"; +import { readDirSync } from "@std/fs/unstable-read-dir"; +import { statSync } from "@std/fs/unstable-stat"; +import { globToRegExp } from "@std/path"; +import process from "node:process"; +import type { PullConfig } from "./types.ts"; + +function copyMatchingFiles( + sourceDir: string, + targetBase: string, + relativePath: string, + regex: RegExp, +): void { + try { + if (!existsSync(sourceDir)) return; + const entries = Array.from(readDirSync(sourceDir)); + for (const entry of entries) { + const sourcePath = join(sourceDir, entry.name); + const relPath = relativePath + ? join(relativePath, entry.name) + : entry.name; + const testPath = relPath.startsWith("/") ? relPath : `/${relPath}`; + + if (statSync(sourcePath).isDirectory) { + copyMatchingFiles(sourcePath, targetBase, relPath, regex); + } else if (entry.name.endsWith(".json")) { + if (regex.test(testPath) || regex.test(relPath)) { + const targetPath = join(targetBase, relPath); + mkdirSync(join(targetPath, ".."), { recursive: true }); + const content = readFileSync(sourcePath); + writeFileSync(targetPath, content); + } + } + } + } catch { + // skip + } +} + +export async function pullLexicons(config: PullConfig): Promise { + const cwd = typeof Deno !== "undefined" ? Deno.cwd() : process.cwd(); + const pullDir = join(cwd, config.outdir); + + if (config.clean && existsSync(pullDir)) { + console.log(`Cleaning ${pullDir}...`); + removeSync(pullDir); + } + + mkdirSync(pullDir, { recursive: true }); + + for (const source of config.sources) { + if (source.type === "git") { + await pullFromGit(source, pullDir); + } + } +} + +export function cleanupPullDirectory(config: PullConfig): void { + if (!config.clean) { + return; + } + + const cwd = typeof Deno !== "undefined" ? Deno.cwd() : process.cwd(); + const pullDir = join(cwd, config.outdir); + + if (existsSync(pullDir)) { + try { + removeSync(pullDir, { recursive: true }); + } catch { + // ignore cleanup errors + } + } +} + +async function pullFromGit( + source: { remote: string; ref?: string; pattern: string[] }, + targetDir: string, +): Promise { + const cwd = typeof Deno !== "undefined" ? Deno.cwd() : process.cwd(); + const tempDir = join(cwd, ".lex-gen-temp", crypto.randomUUID()); + + try { + console.log(`Cloning ${source.remote}...`); + const cloneArgs = [ + "clone", + "--depth", + "1", + "--filter=blob:none", + "--sparse", + ]; + + if (source.ref) { + cloneArgs.push(`--branch=${source.ref}`); + } + + cloneArgs.push(source.remote, tempDir); + + const cloneCmd = new Deno.Command("git", { + args: cloneArgs, + cwd, + }); + + const cloneResult = await cloneCmd.output(); + if (!cloneResult.success) { + const error = new TextDecoder().decode(cloneResult.stderr); + throw new Error(`Failed to clone repository: ${error}`); + } + + const sparseCheckoutCmd = new Deno.Command("git", { + args: ["sparse-checkout", "set", "--no-cone", ...source.pattern], + cwd: tempDir, + }); + + const sparseResult = await sparseCheckoutCmd.output(); + if (!sparseResult.success) { + const error = new TextDecoder().decode(sparseResult.stderr); + throw new Error(`Failed to set sparse checkout: ${error}`); + } + + const checkoutCmd = new Deno.Command("git", { + args: ["checkout"], + cwd: tempDir, + }); + + const checkoutResult = await checkoutCmd.output(); + if (!checkoutResult.success) { + const error = new TextDecoder().decode(checkoutResult.stderr); + throw new Error(`Failed to checkout files: ${error}`); + } + + for (const pattern of source.pattern) { + const normalizedPattern = pattern.startsWith("./") + ? pattern.slice(2) + : pattern; + const regex = globToRegExp(normalizedPattern, { + extended: true, + globstar: true, + }); + + copyMatchingFiles(tempDir, targetDir, "", regex); + } + } finally { + if (existsSync(tempDir)) { + removeSync(tempDir, { recursive: true }); + } + const tempParent = join(cwd, ".lex-gen-temp"); + if (existsSync(tempParent)) { + try { + const entries = Array.from(readDirSync(tempParent)); + if (entries.length === 0) { + removeSync(tempParent); + } + } catch { + // ignore + } + } + } +} diff --git a/lex-gen/types.ts b/lex-gen/types.ts index 09b995b..a4c43c9 100644 --- a/lex-gen/types.ts +++ b/lex-gen/types.ts @@ -12,3 +12,37 @@ export interface FileDiff { path: string; content?: string; } + +export interface GitSourceConfig { + type: "git"; + remote: string; + ref?: string; + pattern: string[]; +} + +export type SourceConfig = GitSourceConfig; + +export interface PullConfig { + outdir: string; + clean?: boolean; + sources: SourceConfig[]; +} + +export interface ImportMapping { + nsid: string[]; + imports: + | string + | ((nsid: string) => { type: "named" | "namespace"; from: string }); +} + +export interface ModulesConfig { + importSuffix?: string; +} + +export interface LexiconConfig { + outdir: string; + files: string[]; + mappings?: ImportMapping[]; + modules?: ModulesConfig; + pull?: PullConfig; +} diff --git a/lex-gen/util.ts b/lex-gen/util.ts index f8bc1eb..ba988b8 100644 --- a/lex-gen/util.ts +++ b/lex-gen/util.ts @@ -3,24 +3,90 @@ import { statSync } from "@std/fs/unstable-stat"; import { mkdirSync } from "@std/fs/unstable-mkdir"; import { writeFileSync } from "@std/fs/unstable-write-file"; import { existsSync } from "@std/fs"; -import { join } from "@std/path"; +import { globToRegExp, join } from "@std/path"; import { removeSync } from "@std/fs/unstable-remove"; import { readDirSync } from "@std/fs/unstable-read-dir"; import { colors } from "@cliffy/ansi/colors"; import { ZodError } from "zod"; import { type LexiconDoc, parseLexiconDoc } from "@atp/lexicon"; import type { FileDiff, GeneratedAPI } from "./types.ts"; +import process from "node:process"; type RecursiveZodError = { _errors?: string[]; [k: string]: RecursiveZodError | string[] | undefined; }; +export function expandGlobPatterns(patterns: string[]): string[] { + const files: string[] = []; + const cwd = typeof Deno !== "undefined" ? Deno.cwd() : process.cwd(); + + function walkDir( + dir: string, + relativePath: string, + regex: RegExp, + files: string[], + ): void { + try { + if (!existsSync(dir)) return; + const entries = Array.from(readDirSync(dir)); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + const relPath = relativePath + ? join(relativePath, entry.name) + : entry.name; + if (statSync(fullPath).isDirectory) { + walkDir(fullPath, relPath, regex, files); + } else if (entry.name.endsWith(".json")) { + const testPath = relPath.startsWith("/") ? relPath : `/${relPath}`; + if (regex.test(testPath) || regex.test(relPath)) { + files.push(fullPath); + } + } + } + } catch { + // skip + } + } + + for (const pattern of patterns) { + const normalizedPattern = pattern.startsWith("./") + ? pattern.slice(2) + : pattern; + const regex = globToRegExp(normalizedPattern, { + extended: true, + globstar: true, + }); + const basePath = normalizedPattern.split("*")[0] || + normalizedPattern.split("?")[0] || ""; + const searchDir = basePath.includes("/") + ? join( + cwd, + basePath.substring(0, basePath.lastIndexOf("/") || basePath.length), + ) + : cwd; + + walkDir(searchDir, "", regex, files); + } + + return Array.from(new Set(files)); +} + export function readAllLexicons(paths: string[] | string): LexiconDoc[] { const docs: LexiconDoc[] = []; - for (const path of Array.isArray(paths) ? paths : [paths]) { + const pathArray = Array.isArray(paths) ? paths : [paths]; + const expandedPaths: string[] = []; + + for (const path of pathArray) { + if (path.includes("*") || path.includes("?")) { + expandedPaths.push(...expandGlobPatterns([path])); + } else { + expandedPaths.push(path); + } + } + + for (const path of expandedPaths) { if (statSync(path).isDirectory) { - // If it's a directory, recursively read all .json files in it const entries = Array.from(readDirSync(path)); const subPaths = entries.map((entry) => join(path, entry.name)); docs.push(...readAllLexicons(subPaths));