diff --git a/deno.json b/deno.json index 1ba62c8..1353991 100644 --- a/deno.json +++ b/deno.json @@ -10,10 +10,11 @@ "xrpc", "xrpc-server", "sync", - "lex-gen" + "lex-gen", + "lex" ], "imports": { "@opentelemetry/api": "npm:@opentelemetry/api@^1.9.0", - "@std/assert": "jsr:@std/assert@^1.0.16" + "@std/assert": "jsr:@std/assert@^1.0.19" } } diff --git a/deno.lock b/deno.lock index 9b89a46..a552538 100644 --- a/deno.lock +++ b/deno.lock @@ -13,39 +13,38 @@ "jsr:@noble/curves@^2.0.1": "2.0.1", "jsr:@noble/hashes@2": "2.0.1", "jsr:@noble/hashes@^2.0.1": "2.0.1", - "jsr:@std/assert@^1.0.16": "1.0.16", + "jsr:@std/assert@^1.0.19": "1.0.19", "jsr:@std/bytes@^1.0.6": "1.0.6", "jsr:@std/cbor@~0.1.9": "0.1.9", "jsr:@std/encoding@^1.0.10": "1.0.10", "jsr:@std/encoding@~1.0.5": "1.0.10", "jsr:@std/fmt@~1.0.2": "1.0.8", - "jsr:@std/fs@1": "1.0.20", - "jsr:@std/fs@^1.0.19": "1.0.20", - "jsr:@std/fs@^1.0.20": "1.0.20", + "jsr:@std/fs@1": "1.0.23", + "jsr:@std/fs@^1.0.19": "1.0.23", + "jsr:@std/fs@^1.0.20": "1.0.23", "jsr:@std/internal@^1.0.12": "1.0.12", "jsr:@std/io@~0.224.9": "0.224.9", "jsr:@std/json@^1.0.2": "1.0.2", "jsr:@std/jsonc@^1.0.1": "1.0.2", - "jsr:@std/path@1": "1.1.3", - "jsr:@std/path@^1.1.2": "1.1.3", - "jsr:@std/path@^1.1.3": "1.1.3", + "jsr:@std/path@1": "1.1.4", + "jsr:@std/path@^1.1.2": "1.1.4", + "jsr:@std/path@^1.1.4": "1.1.4", "jsr:@std/streams@^1.0.14": "1.0.14", "jsr:@std/text@~1.0.7": "1.0.16", "jsr:@ts-morph/common@0.27": "0.27.0", "jsr:@ts-morph/ts-morph@26": "26.0.0", - "jsr:@zod/zod@^4.1.11": "4.1.13", - "jsr:@zod/zod@^4.1.13": "4.1.13", + "jsr:@zod/zod@^4.1.13": "4.3.6", "npm:@atproto/crypto@*": "0.1.0", "npm:@did-plc/lib@^0.0.4": "0.0.4", "npm:@did-plc/server@^0.0.1": "0.0.1_express@4.21.2", "npm:@ipld/dag-cbor@^9.2.5": "9.2.5", "npm:@opentelemetry/api@^1.9.0": "1.9.0", "npm:@types/node@*": "24.2.0", + "npm:cborg@^4.2.15": "4.2.15", "npm:get-port@^7.1.0": "7.1.0", "npm:key-encoder@^2.0.3": "2.0.3", "npm:multiformats@^13.4.1": "13.4.1", "npm:p-queue@^8.1.1": "8.1.1", - "npm:prettier@^3.6.2": "3.6.2", "npm:rate-limiter-flexible@9": "9.0.0", "npm:ws@^8.18.0": "8.18.3" }, @@ -111,8 +110,8 @@ "@noble/hashes@2.0.1": { "integrity": "e0e908292a0bf91099cf8ba0720a1647cef82ab38b588815b5e9535b4ff4d7bb" }, - "@std/assert@1.0.16": { - "integrity": "6a7272ed1eaa77defe76e5ff63ca705d9c495077e2d5fd0126d2b53fc5bd6532", + "@std/assert@1.0.19": { + "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e", "dependencies": [ "jsr:@std/internal" ] @@ -133,11 +132,11 @@ "@std/fmt@1.0.8": { "integrity": "71e1fc498787e4434d213647a6e43e794af4fd393ef8f52062246e06f7e372b7" }, - "@std/fs@1.0.20": { - "integrity": "e953206aae48d46ee65e8783ded459f23bec7dd1f3879512911c35e5484ea187", + "@std/fs@1.0.23": { + "integrity": "3ecbae4ce4fee03b180fa710caff36bb5adb66631c46a6460aaad49515565a37", "dependencies": [ "jsr:@std/internal", - "jsr:@std/path@^1.1.3" + "jsr:@std/path@^1.1.4" ] }, "@std/internal@1.0.12": { @@ -155,8 +154,8 @@ "jsr:@std/json" ] }, - "@std/path@1.1.3": { - "integrity": "b015962d82a5e6daea980c32b82d2c40142149639968549c649031a230b1afb3", + "@std/path@1.1.4": { + "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", "dependencies": [ "jsr:@std/internal" ] @@ -190,8 +189,8 @@ "@zod/zod@4.1.11": { "integrity": "0d48947455491addca672d8ef766d86bc7bc3add07e78d049b8ffd643bb33a7a" }, - "@zod/zod@4.1.13": { - "integrity": "fef799152d630583b248645fcac03abedd13e39fd2b752d9466b905d73619bfd" + "@zod/zod@4.3.6": { + "integrity": "7144e5e11f8ffc3cf6e2fca624f6597a8762898aac9868cc8938e9398b96ffe4" } }, "npm": { @@ -884,10 +883,6 @@ "xtend" ] }, - "prettier@3.6.2": { - "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", - "bin": true - }, "process-warning@3.0.0": { "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, @@ -1104,7 +1099,7 @@ }, "workspace": { "dependencies": [ - "jsr:@std/assert@^1.0.16", + "jsr:@std/assert@^1.0.19", "npm:@opentelemetry/api@^1.9.0" ], "members": { @@ -1139,6 +1134,12 @@ "npm:get-port@^7.1.0" ] }, + "lex": { + "dependencies": [ + "npm:cborg@^4.2.15", + "npm:multiformats@^13.4.1" + ] + }, "lex-gen": { "dependencies": [ "jsr:@cliffy/ansi@^1.0.0-rc.8", @@ -1146,9 +1147,7 @@ "jsr:@std/fs@^1.0.19", "jsr:@std/jsonc@^1.0.1", "jsr:@std/path@^1.1.2", - "jsr:@ts-morph/ts-morph@26", - "jsr:@zod/zod@^4.1.11", - "npm:prettier@^3.6.2" + "jsr:@ts-morph/ts-morph@26" ] }, "lexicon": { diff --git a/lex-gen/builder/def-builder.ts b/lex-gen/builder/def-builder.ts new file mode 100644 index 0000000..8594880 --- /dev/null +++ b/lex-gen/builder/def-builder.ts @@ -0,0 +1,951 @@ +import assert from "node:assert"; +import { type SourceFile, VariableDeclarationKind } from "ts-morph"; +import type { + LexiconArray, + LexiconArrayItems, + LexiconBlob, + LexiconBoolean, + LexiconBytes, + LexiconCid, + LexiconDocument, + LexiconError, + LexiconIndexer, + LexiconInteger, + LexiconObject, + LexiconParameters, + LexiconPayload, + LexiconProcedure, + LexiconQuery, + LexiconRecord, + LexiconRef, + LexiconRefUnion, + LexiconString, + LexiconSubscription, + LexiconToken, + LexiconUnknown, + MainLexiconDefinition, + NamedLexiconDefinition, +} from "@atp/lex/document"; + +import { + getPublicIdentifiers, + RefResolver, + type ResolvedRef, +} from "./ref-resolver.ts"; +import { isSafeIdentifier } from "./ts-lang.ts"; + +export type LexDefBuilderOptions = { + importExt?: string; + lib?: string; + allowLegacyBlobs?: boolean; + pureAnnotations?: boolean; +}; + +type AnyDef = MainLexiconDefinition | NamedLexiconDefinition; +type DefsMap = Record; + +type MsgSchema = + | LexiconRef + | LexiconRefUnion + | LexiconObject + | undefined; + +export class LexDefBuilder { + private readonly refResolver: RefResolver; + private static readonly BANNER = + '/*\n * THIS FILE WAS GENERATED BY "@atproto/lex". DO NOT EDIT.\n */'; + + constructor( + private readonly options: LexDefBuilderOptions, + private readonly file: SourceFile, + private readonly doc: LexiconDocument, + indexer: LexiconIndexer, + ) { + this.refResolver = new RefResolver(doc, file, indexer, options); + } + + private pure(code: string): string { + return this.options.pureAnnotations ? markPure(code) : code; + } + + async build(): Promise { + this.file.insertText( + 0, + `${LexDefBuilder.BANNER}\n`, + ); + + this.file.addVariableStatement({ + declarationKind: VariableDeclarationKind.Const, + declarations: [ + { + name: "$nsid", + initializer: JSON.stringify(this.doc.id), + }, + ], + }); + + this.file.addExportDeclaration({ + namedExports: [{ name: "$nsid" }], + }); + + const defs = Object.keys(this.doc.defs); + if (defs.length) { + const moduleSpecifier = this.options.lib ?? "@atp/lex"; + this.file + .addImportDeclaration({ moduleSpecifier }) + .addNamedImports([{ name: "l" }]); + + for (const hash of defs) { + await this.addDef(hash); + } + } + + this.normalizeBannerSpacing(); + } + + private normalizeBannerSpacing(): void { + const text = this.file.getFullText(); + const before = `${LexDefBuilder.BANNER}\n`; + const after = `${LexDefBuilder.BANNER}\n\n`; + + if (text.startsWith(before) && !text.startsWith(after)) { + this.file.replaceWithText(`${after}${text.slice(before.length)}`); + } + } + + private addUtils(definitions: Record): void { + const entries = Object.entries(definitions).filter( + (e): e is [(typeof e)[0], string] => e[1] != null, + ); + if (entries.length) { + this.file.addVariableStatement({ + isExported: true, + declarationKind: VariableDeclarationKind.Const, + declarations: entries.map(([name, initializer]) => ({ + name, + initializer, + })), + }); + } + } + + private async addDef(hash: string): Promise { + const defsMap = this.doc.defs as unknown as DefsMap; + const def = Object.hasOwn(this.doc.defs, hash) ? defsMap[hash] : null; + if (def == null) return; + + switch (def.type) { + case "procedure": + return this.addProcedure(hash, def); + case "query": + return this.addQuery(hash, def); + case "subscription": + return this.addSubscription(hash, def); + case "record": + return this.addRecord(hash, def); + case "token": + return this.addToken(hash, def); + case "object": + return this.addObject(hash, def); + case "array": + return this.addArray(hash, def); + default: { + const containedDef = def as LexiconArray | LexiconArrayItems; + await this.addSchema(hash, containedDef, { + type: await this.compileContainedType(containedDef), + schema: await this.compileContainedSchema(containedDef), + validationUtils: true, + }); + } + } + } + + private async addProcedure( + hash: string, + def: LexiconProcedure, + ): Promise { + if (hash !== "main") { + throw new Error(`Definition ${hash} cannot be of type ${def.type}`); + } + + const ref = await this.compileXrpcRef(hash, def, { + schema: async () => + this.pure( + `l.procedure($nsid, ${await this + .compileParamsSchema(def.parameters)}, ${await this.compilePayload( + def.input, + )}, ${await this.compilePayload(def.output)}${await this + .compileErrors(def.errors)})`, + ), + }); + this.addMethodTypeStatements(ref, def); + this.addUtils({ + $lxm: this.pure(`${ref.varName}.nsid`), + $params: this.pure(`${ref.varName}.parameters`), + $input: this.pure(`${ref.varName}.input`), + $output: this.pure(`${ref.varName}.output`), + }); + } + + private async addQuery(hash: string, def: LexiconQuery): Promise { + if (hash !== "main") { + throw new Error(`Definition ${hash} cannot be of type ${def.type}`); + } + + const ref = await this.compileXrpcRef(hash, def, { + schema: async () => + this.pure( + `l.query($nsid, ${await this + .compileParamsSchema(def.parameters)}, ${await this.compilePayload( + def.output, + )}${await this.compileErrors(def.errors)})`, + ), + }); + this.addMethodTypeStatements(ref, def); + this.addUtils({ + $lxm: this.pure(`${ref.varName}.nsid`), + $params: this.pure(`${ref.varName}.parameters`), + $output: this.pure(`${ref.varName}.output`), + }); + } + + private async addSubscription( + hash: string, + def: LexiconSubscription, + ): Promise { + if (hash !== "main") { + throw new Error(`Definition ${hash} cannot be of type ${def.type}`); + } + + const msgSchema = def.message?.schema as MsgSchema; + let messageSchema: string; + + if (!msgSchema) { + messageSchema = "undefined"; + } else if (msgSchema.type === "ref") { + const { varName, typeName } = await this.refResolver.resolve( + msgSchema.ref, + ); + messageSchema = this.pure( + `l.ref<${typeName}>(() => ${varName})`, + ); + } else if (msgSchema.type === "union") { + if (msgSchema.refs.length === 0 && msgSchema.closed) { + messageSchema = this.pure("l.never()"); + } else { + const refs = await Promise.all( + msgSchema.refs.map(async (ref: string) => { + const { varName, typeName } = await this.refResolver.resolve(ref); + return this.pure( + `l.typedRef<${typeName}>(() => ${varName})`, + ); + }), + ); + messageSchema = this.pure( + `l.typedUnion([${refs.join(",")}], ${msgSchema.closed ?? false})`, + ); + } + } else { + messageSchema = await this.compileObjectSchema(msgSchema); + } + + const ref = await this.compileXrpcRef(hash, def, { + schema: async () => + this.pure( + `l.subscription($nsid, ${await this + .compileParamsSchema(def.parameters)}, ${messageSchema}${await this + .compileErrors(def.errors)})`, + ), + }); + this.addMethodTypeStatements(ref, def); + this.addUtils({ + $lxm: this.pure(`${ref.varName}.nsid`), + $params: this.pure(`${ref.varName}.parameters`), + $message: this.pure(`${ref.varName}.message`), + }); + } + + private compileXrpcRef( + hash: string, + def: { description?: string }, + opts: { schema: (ref: ResolvedRef) => Promise }, + ): Promise { + return this.addSchema(hash, def, { + schema: opts.schema, + validationUtils: false, + }); + } + + private addMethodTypeStatements( + ref: ResolvedRef, + def: LexiconProcedure | LexiconQuery | LexiconSubscription, + ): void { + this.file.addTypeAlias({ + isExported: true, + name: "$Params", + type: `l.InferMethodParams`, + }); + + if (def.type === "procedure") { + this.file.addTypeAlias({ + isExported: true, + name: "$Input", + type: `l.InferMethodInput`, + }); + + this.file.addTypeAlias({ + isExported: true, + name: "$InputBody", + type: `l.InferMethodInputBody`, + }); + } + + if (def.type === "procedure" || def.type === "query") { + this.file.addTypeAlias({ + isExported: true, + name: "$Output", + type: `l.InferMethodOutput`, + }); + + this.file.addTypeAlias({ + isExported: true, + name: "$OutputBody", + type: `l.InferMethodOutputBody`, + }); + } + + if (def.type === "subscription") { + this.file.addTypeAlias({ + isExported: true, + name: "$Message", + type: `l.InferSubscriptionMessage`, + }); + } + } + + private async addRecord(hash: string, def: LexiconRecord): Promise { + const key = JSON.stringify(def.key); + const objectSchema = await this.compileObjectSchema(def.record); + const properties = await this.compilePropertiesTypes(def.record); + + await this.addSchema(hash, def, { + type: `{ $type: string; ${properties.join(";")} }`, + schema: this.pure( + `l.record(${key}, $nsid, ${objectSchema})`, + ), + objectUtils: true, + validationUtils: true, + }); + } + + private async addObject(hash: string, def: LexiconObject): Promise { + const objectSchema = await this.compileObjectSchema(def); + const properties = await this.compilePropertiesTypes(def); + const $type = hash === "main" ? this.doc.id : `${this.doc.id}#${hash}`; + properties.unshift(`$type?: ${JSON.stringify($type)}`); + + await this.addSchema(hash, def, { + type: `{ ${properties.join(";")} }`, + schemaType: (ref) => + `l.TypedObjectSchema, l.Validator>>`, + schema: async (ref) => + this.pure( + `l.typedObject<${ref.typeName}>($nsid, ${ + JSON.stringify(hash) + }, ${objectSchema})`, + ), + objectUtils: true, + validationUtils: true, + }); + } + + private async addToken(hash: string, def: LexiconToken): Promise { + await this.addSchema(hash, def, { + type: `l.$Type`, + schema: this.pure(`l.token($nsid, ${JSON.stringify(hash)})`), + validationUtils: true, + }); + } + + private async addArray(hash: string, def: LexiconArray): Promise { + const itemSchema = await this.compileContainedSchema(def.items); + const options = stringifyOptions(def, ["minLength", "maxLength"]); + + await this.addSchema(hash, def, { + type: `(${await this.compileContainedType(def.items)})[]`, + schema: this.pure( + `l.array(${itemSchema}${options ? `, ${options}` : ""})`, + ), + validationUtils: true, + }); + } + + private async addSchema( + hash: string, + def: { description?: string }, + { + type, + schema, + schemaType, + objectUtils, + validationUtils, + }: { + type?: string | ((ref: ResolvedRef) => string); + schema?: string | ((ref: ResolvedRef) => Promise); + schemaType?: string | ((ref: ResolvedRef) => string); + objectUtils?: boolean; + validationUtils?: boolean; + }, + ): Promise { + const ref = await this.refResolver.resolveLocal(hash); + const pub = getPublicIdentifiers(hash); + + assert(isSafeIdentifier(ref.varName), "Expected safe type identifier"); + assert(isSafeIdentifier(ref.typeName), "Expected safe type identifier"); + assert(isSafeIdentifier(pub.typeName), "Expected safe type identifier"); + + if (type) { + const typeStr = typeof type === "function" ? type(ref) : type; + const typeStmt = this.file.addTypeAlias({ + name: ref.typeName, + type: typeStr, + }); + addJsDoc(typeStmt, def); + + this.file.addExportDeclaration({ + isTypeOnly: true, + namedExports: [ + { + name: ref.typeName, + alias: ref.typeName === pub.typeName ? undefined : pub.typeName, + }, + ], + }); + } + + if (schema) { + const schemaStr = typeof schema === "function" + ? await schema(ref) + : schema; + const schemaTypeStr = schemaType + ? typeof schemaType === "function" ? schemaType(ref) : schemaType + : undefined; + + const constStmt = this.file.addVariableStatement({ + declarationKind: VariableDeclarationKind.Const, + declarations: [{ + name: ref.varName, + type: schemaTypeStr, + initializer: schemaStr, + }], + }); + addJsDoc(constStmt, def); + + this.file.addExportDeclaration({ + namedExports: [ + { + name: ref.varName, + alias: ref.varName === pub.varName + ? undefined + : isSafeIdentifier(pub.varName) + ? pub.varName + : JSON.stringify(pub.varName), + }, + ], + }); + } + + if (hash === "main" && objectUtils) { + this.addUtils({ + $isTypeOf: markPure(`${ref.varName}.isTypeOf.bind(${ref.varName})`), + $build: markPure(`${ref.varName}.build.bind(${ref.varName})`), + $type: markPure(`${ref.varName}.$type`), + }); + } + + if (hash === "main" && validationUtils) { + this.addUtils({ + $assert: markPure(`${ref.varName}.assert.bind(${ref.varName})`), + $ifMatches: markPure( + `${ref.varName}.ifMatches.bind(${ref.varName})`, + ), + $matches: markPure(`${ref.varName}.matches.bind(${ref.varName})`), + $parse: markPure(`${ref.varName}.parse.bind(${ref.varName})`), + $safeParse: markPure( + `${ref.varName}.safeParse.bind(${ref.varName})`, + ), + }); + } + + return ref; + } + + private async compilePayload( + def: LexiconPayload | undefined, + ): Promise { + if (!def) return this.pure("l.payload()"); + + const schema = def.schema as + | LexiconRef + | LexiconRefUnion + | LexiconObject + | undefined; + + if (def.encoding === "application/json" && schema?.type === "object") { + const properties = await this.compilePropertiesSchemas(schema); + return this.pure(`l.jsonPayload({${properties.join(",")}})`); + } + + const encodedEncoding = JSON.stringify(def.encoding); + if (schema) { + const bodySchema = await this.compileBodySchema(schema); + return this.pure(`l.payload(${encodedEncoding}, ${bodySchema})`); + } + return this.pure(`l.payload(${encodedEncoding})`); + } + + private compileBodySchema( + def: LexiconRef | LexiconRefUnion | LexiconObject | undefined, + ): Promise { + if (!def) return Promise.resolve("undefined"); + if (def.type === "object") return this.compileObjectSchema(def); + return this.compileContainedSchema(def); + } + + private async compileParamsSchema( + def: LexiconParameters | undefined, + ): Promise { + if (!def) return this.pure("l.params()"); + const properties = await this.compilePropertiesSchemas(def); + return this.pure( + properties.length === 0 + ? "l.params()" + : `l.params({${properties.join(",")}})`, + ); + } + + private compileErrors(defs?: readonly LexiconError[]): Promise { + if (!defs?.length) return Promise.resolve(""); + return Promise.resolve(`, ${JSON.stringify(defs.map((d) => d.name))}`); + } + + private async compileObjectSchema(def: LexiconObject): Promise { + const properties = await this.compilePropertiesSchemas(def); + return this.pure(`l.object({${properties.join(",")}})`); + } + + private compilePropertiesSchemas(options: { + properties: Record; + required?: readonly string[]; + nullable?: readonly string[]; + }): Promise { + for (const opt of ["required", "nullable"] as const) { + if (options[opt]) { + for (const prop of options[opt]!) { + if (!Object.hasOwn(options.properties, prop)) { + throw new Error(`No schema found for ${opt} property "${prop}"`); + } + } + } + } + return Promise.all( + Object.entries(options.properties).map((entry) => + this.compilePropertyEntrySchema(entry, options) + ), + ); + } + + private compilePropertiesTypes(options: { + properties: Record; + required?: readonly string[]; + nullable?: readonly string[]; + }): Promise { + return Promise.all( + Object.entries(options.properties).map((entry) => + this.compilePropertyEntryType(entry, options) + ), + ); + } + + private async compilePropertyEntrySchema( + [key, def]: [string, LexiconArray | LexiconArrayItems], + options: { required?: readonly string[]; nullable?: readonly string[] }, + ): Promise { + const isNullable = options.nullable?.includes(key); + const isRequired = options.required?.includes(key); + + let schema = await this.compileContainedSchema(def); + if (isNullable) schema = this.pure(`l.nullable(${schema})`); + if (!isRequired) schema = this.pure(`l.optional(${schema})`); + + return `${JSON.stringify(key)}:${schema}`; + } + + private async compilePropertyEntryType( + [key, def]: [string, LexiconArray | LexiconArrayItems], + options: { required?: readonly string[]; nullable?: readonly string[] }, + ): Promise { + const isNullable = options.nullable?.includes(key); + const isRequired = options.required?.includes(key); + + const optional = isRequired ? "" : "?"; + const append = isNullable ? " | null" : ""; + const jsDoc = compileLeadingTrivia(def.description) ?? ""; + const name = JSON.stringify(key); + const type = await this.compileContainedType(def); + + return `${jsDoc}${name}${optional}:${type}${append}`; + } + + private compileContainedSchema( + def: LexiconArray | LexiconArrayItems, + ): Promise { + switch (def.type) { + case "unknown": + return Promise.resolve(this.compileUnknownSchema(def)); + case "boolean": + return Promise.resolve(this.compileBooleanSchema(def)); + case "integer": + return Promise.resolve(this.compileIntegerSchema(def)); + case "string": + return Promise.resolve(this.compileStringSchema(def)); + case "bytes": + return Promise.resolve(this.compileBytesSchema(def)); + case "blob": + return Promise.resolve(this.compileBlobSchema(def)); + case "cid-link": + return Promise.resolve(this.compileCidLinkSchema(def)); + case "ref": + return this.compileRefSchema(def); + case "union": + return this.compileRefUnionSchema(def); + case "array": + return this.compileArraySchema(def); + default: + throw new Error( + `Unsupported def type: ${(def as { type: string }).type}`, + ); + } + } + + private compileContainedType( + def: LexiconArray | LexiconArrayItems, + ): Promise { + switch (def.type) { + case "unknown": + return Promise.resolve(this.compileUnknownType(def)); + case "boolean": + return Promise.resolve(this.compileBooleanType(def)); + case "integer": + return Promise.resolve(this.compileIntegerType(def)); + case "string": + return Promise.resolve(this.compileStringType(def)); + case "bytes": + return Promise.resolve(this.compileBytesType(def)); + case "blob": + return Promise.resolve(this.compileBlobType(def)); + case "cid-link": + return Promise.resolve(this.compileCidLinkType(def)); + case "ref": + return this.compileRefType(def); + case "union": + return this.compileRefUnionType(def); + case "array": + return this.compileArrayType(def); + default: + throw new Error( + `Unsupported def type: ${(def as { type: string }).type}`, + ); + } + } + + private compileArraySchema(def: LexiconArray): Promise { + return this.compileContainedSchema(def.items).then((itemSchema) => { + const options = stringifyOptions(def, ["minLength", "maxLength"]); + return this.pure( + `l.array(${itemSchema}${options ? `, ${options}` : ""})`, + ); + }); + } + + private async compileArrayType(def: LexiconArray): Promise { + return `(${await this.compileContainedType(def.items)})[]`; + } + + private compileUnknownSchema(_def: LexiconUnknown): string { + return this.pure("l.unknownObject()"); + } + + private compileUnknownType(_def: LexiconUnknown): string { + return "l.UnknownObject"; + } + + private compileBooleanSchema(def: LexiconBoolean): string { + if (hasConst(def)) return this.compileConstSchema(def); + const options = stringifyOptions(def, ["default"]); + return this.pure(`l.boolean(${options})`); + } + + private compileBooleanType(def: LexiconBoolean): string { + if (hasConst(def)) return this.compileConstType(def); + return "boolean"; + } + + private compileIntegerSchema(def: LexiconInteger): string { + if (hasConst(def)) return this.compileConstSchema(def); + if (hasEnum(def)) return this.compileEnumSchema(def); + const options = stringifyOptions(def, ["default", "maximum", "minimum"]); + return this.pure(`l.integer(${options})`); + } + + private compileIntegerType(def: LexiconInteger): string { + if (hasConst(def)) return this.compileConstType(def); + if (hasEnum(def)) return this.compileEnumType(def); + return "number"; + } + + private compileStringSchema(def: LexiconString): string { + if (hasConst(def)) return this.compileConstSchema(def); + if (hasEnum(def)) return this.compileEnumSchema(def); + const options = stringifyOptions(def, [ + "default", + "format", + "maxGraphemes", + "minGraphemes", + "maxLength", + "minLength", + ]); + return this.pure(`l.string(${options})`); + } + + private compileStringType(def: LexiconString): string { + if (hasConst(def)) return this.compileConstType(def); + if (hasEnum(def)) return this.compileEnumType(def); + + switch (def.format) { + case undefined: + break; + case "datetime": + return "l.DatetimeString"; + case "uri": + return "l.UriString"; + case "at-uri": + return "l.AtUriString"; + case "did": + return "l.DidString"; + case "handle": + return "l.HandleString"; + case "at-identifier": + return "l.AtIdentifierString"; + case "nsid": + return "l.NsidString"; + case "tid": + return "l.TidString"; + case "cid": + return "l.CidString"; + case "language": + return "l.LanguageString"; + case "record-key": + return "l.RecordKeyString"; + default: + throw new Error(`Unknown string format: ${def.format}`); + } + + if (def.knownValues?.length) { + return ( + def.knownValues.map((v) => JSON.stringify(v)).join(" | ") + + " | l.UnknownString" + ); + } + + return "string"; + } + + private compileBytesSchema(def: LexiconBytes): string { + const options = stringifyOptions(def, ["minLength", "maxLength"]); + return this.pure(`l.bytes(${options})`); + } + + private compileBytesType(_def: LexiconBytes): string { + return "Uint8Array"; + } + + private compileBlobSchema(def: LexiconBlob): string { + const opts = { + ...def, + allowLegacy: this.options.allowLegacyBlobs === true, + }; + const options = stringifyOptions(opts, [ + "maxSize", + "accept", + "allowLegacy", + ]); + return this.pure(`l.blob(${options})`); + } + + private compileBlobType(_def: LexiconBlob): string { + return this.options.allowLegacyBlobs + ? "l.BlobRef | l.LegacyBlobRef" + : "l.BlobRef"; + } + + private compileCidLinkSchema(_def: LexiconCid): string { + return this.pure("l.cidLink()"); + } + + private compileCidLinkType(_def: LexiconCid): string { + return "l.Cid"; + } + + private async compileRefSchema(def: LexiconRef): Promise { + const { varName, typeName } = await this.refResolver.resolve(def.ref); + return this.pure(`l.ref<${typeName}>(() => ${varName})`); + } + + private async compileRefType(def: LexiconRef): Promise { + const ref = await this.refResolver.resolve(def.ref); + return ref.typeName; + } + + private async compileRefUnionSchema(def: LexiconRefUnion): Promise { + if (def.refs.length === 0 && def.closed) { + return this.pure("l.never()"); + } + + const refs = await Promise.all( + def.refs.map(async (ref) => { + const { varName, typeName } = await this.refResolver.resolve(ref); + return this.pure( + `l.typedRef<${typeName}>(() => ${varName})`, + ); + }), + ); + + return this.pure( + `l.typedUnion([${refs.join(",")}], ${def.closed ?? false})`, + ); + } + + private async compileRefUnionType(def: LexiconRefUnion): Promise { + const types = await Promise.all( + def.refs.map(async (ref) => { + const { typeName } = await this.refResolver.resolve(ref); + return `l.TypedRef<${typeName}>`; + }), + ); + if (!def.closed) types.push("l.TypedObject"); + return types.join(" | ") || "never"; + } + + private compileConstSchema( + def: { const: T; enum?: readonly T[]; default?: T }, + ): string { + if (hasEnum(def) && !def.enum.includes(def.const)) { + return this.pure("l.never()"); + } + const options = stringifyOptions(def, ["default"]); + return this.pure( + `l.literal(${JSON.stringify(def.const)}${options ? `, ${options}` : ""})`, + ); + } + + private compileConstType( + def: { const: T; enum?: readonly T[] }, + ): string { + if (hasEnum(def) && !def.enum.includes(def.const)) return "never"; + return JSON.stringify(def.const); + } + + private compileEnumSchema(def: { + enum: readonly T[]; + default?: T; + }): string { + if (def.enum.length === 0) return this.pure("l.never()"); + if (def.enum.length === 1 && def.default === undefined) { + return this.pure(`l.literal(${JSON.stringify(def.enum[0])})`); + } + const options = stringifyOptions(def, ["default"]); + return this.pure( + `l.enum(${JSON.stringify(def.enum)}${options ? `, ${options}` : ""})`, + ); + } + + private compileEnumType(def: { + enum: readonly T[]; + }): string { + return def.enum.map((v) => JSON.stringify(v)).join(" | ") || "never"; + } +} + +function parseDescription(description: string): { + description: string; + deprecated: boolean | string; +} { + if (/deprecated/i.test(description)) { + const deprecationMatch = description.match( + /(\s*deprecated\s*(?:--?|:)?\s*([^-]*)(?:-+)?)/i, + ); + if (deprecationMatch) { + const [, match, deprecationNotice] = deprecationMatch; + return { + description: description.replace(match, "").trim(), + deprecated: deprecationNotice?.trim() || true, + }; + } + return { description: description.trim(), deprecated: true }; + } + return { description: description.trim(), deprecated: false }; +} + +function compileLeadingTrivia(description?: string): string | undefined { + if (!description) return undefined; + return `\n\n/**${compileJsDoc(description).replaceAll("\n", "\n * ")}\n */\n`; +} + +function addJsDoc( + declaration: { addJsDoc: (text: string) => void }, + def?: { description?: string }, +): void { + if (def?.description) { + declaration.addJsDoc(compileJsDoc(def.description)); + } +} + +function compileJsDoc(description: string): string { + const parsed = parseDescription(description); + return `\n${parsed.description}${ + !parsed.deprecated ? "" : (parsed.description ? "\n\n" : "") + + (parsed.deprecated === true + ? "@deprecated" + : `@deprecated ${parsed.deprecated}`) + }`; +} + +function stringifyOptions>( + obj: O, + include?: (keyof O)[], +): string { + const filtered = Object.entries(obj).filter( + ([k, v]) => + v !== undefined && + v !== null && + (!include || include.includes(k as keyof O)), + ); + return filtered.length ? JSON.stringify(Object.fromEntries(filtered)) : ""; +} + +function hasConst( + def: T, +): def is T & { const: NonNullable } { + return def.const != null; +} + +function hasEnum( + def: T, +): def is T & { enum: unknown[] } { + return def.enum != null; +} + +function markPure(v: T): `/*#__PURE__*/ ${T}` { + return `/*#__PURE__*/ ${v}`; +} diff --git a/lex-gen/builder/directory-indexer.ts b/lex-gen/builder/directory-indexer.ts new file mode 100644 index 0000000..606066e --- /dev/null +++ b/lex-gen/builder/directory-indexer.ts @@ -0,0 +1,55 @@ +import { join } from "node:path"; +import { + type LexiconDocument, + lexiconDocumentSchema, + LexiconIterableIndexer, +} from "@atp/lex/document"; + +export type LexiconDirectoryIndexerOptions = { + lexicons: string; + ignoreInvalidLexicons?: boolean; +}; + +export class LexiconDirectoryIndexer extends LexiconIterableIndexer { + constructor(options: LexiconDirectoryIndexerOptions) { + super(readLexicons(options)); + } +} + +async function* readLexicons( + options: LexiconDirectoryIndexerOptions, +): AsyncGenerator { + for await (const filePath of listFiles(options.lexicons)) { + if (filePath.endsWith(".json")) { + try { + const data = await Deno.readTextFile(filePath); + yield lexiconDocumentSchema.parse(JSON.parse(data)); + } catch (cause) { + const message = `Error parsing lexicon document ${filePath}`; + if (options.ignoreInvalidLexicons) console.error(`${message}:`, cause); + else throw new Error(message, { cause }); + } + } + } +} + +async function* listFiles(dir: string): AsyncGenerator { + let entries: Deno.DirEntry[]; + try { + entries = []; + for await (const entry of Deno.readDir(dir)) { + entries.push(entry); + } + } catch (err) { + if (err instanceof Deno.errors.NotFound) return; + throw err; + } + for (const entry of entries) { + const res = join(dir, entry.name); + if (entry.isDirectory) { + yield* listFiles(res); + } else if (entry.isFile || entry.isSymlink) { + yield res; + } + } +} diff --git a/lex-gen/builder/filter.ts b/lex-gen/builder/filter.ts new file mode 100644 index 0000000..1f08b39 --- /dev/null +++ b/lex-gen/builder/filter.ts @@ -0,0 +1,39 @@ +export type BuildFilterOptions = { + include?: string | string[]; + exclude?: string | string[]; +}; + +export type Filter = (input: string) => boolean; + +export function buildFilter(options: BuildFilterOptions): Filter { + const include = createMatcher(options.include, () => true); + const exclude = createMatcher(options.exclude, () => false); + return (id) => include(id) && !exclude(id); +} + +function createMatcher( + pattern: undefined | string | string[], + fallback: Filter, +): Filter { + if (!pattern?.length) { + return fallback; + } else if (Array.isArray(pattern)) { + return pattern.map(buildMatcher).reduce(combineFilters); + } else { + return buildMatcher(pattern); + } +} + +function combineFilters(a: Filter, b: Filter): Filter { + return (input: string) => a(input) || b(input); +} + +function buildMatcher(pattern: string): Filter { + if (pattern.includes("*")) { + const regex = new RegExp( + `^${pattern.replaceAll(".", "\\.").replaceAll("*", ".+")}$`, + ); + return (input: string) => regex.test(input); + } + return (input: string) => pattern === input; +} diff --git a/lex-gen/builder/filtered-indexer.ts b/lex-gen/builder/filtered-indexer.ts new file mode 100644 index 0000000..2373bf1 --- /dev/null +++ b/lex-gen/builder/filtered-indexer.ts @@ -0,0 +1,52 @@ +import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document"; +import type { Filter } from "./filter.ts"; + +export class FilteredIndexer implements LexiconIndexer, AsyncDisposable { + protected readonly returned = new Set(); + + constructor( + readonly indexer: LexiconIndexer & AsyncIterable, + readonly filter: Filter, + ) {} + + get(id: string): Promise { + this.returned.add(id); + return this.indexer.get(id); + } + + async *[Symbol.asyncIterator](): AsyncGenerator< + LexiconDocument, + void, + unknown + > { + const returned = new Set(); + + for await (const doc of this.indexer) { + if (returned.has(doc.id)) { + throw new Error(`Duplicate lexicon document id: ${doc.id}`); + } + + if (this.returned.has(doc.id) || this.filter(doc.id)) { + this.returned.add(doc.id); + returned.add(doc.id); + yield doc; + } + } + + let returnedAny: boolean; + do { + returnedAny = false; + for (const id of this.returned) { + if (!returned.has(id)) { + yield await this.indexer.get(id); + returned.add(id); + returnedAny = true; + } + } + } while (returnedAny); + } + + async [Symbol.asyncDispose](): Promise { + await this.indexer[Symbol.asyncDispose]?.(); + } +} diff --git a/lex-gen/builder/lex-builder.ts b/lex-gen/builder/lex-builder.ts new file mode 100644 index 0000000..e7ce5a6 --- /dev/null +++ b/lex-gen/builder/lex-builder.ts @@ -0,0 +1,157 @@ +import { mkdir, rm, stat, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { IndentationText, Project } from "ts-morph"; +import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document"; +import { buildFilter, type BuildFilterOptions } from "./filter.ts"; +import { FilteredIndexer } from "./filtered-indexer.ts"; +import { LexDefBuilder, type LexDefBuilderOptions } from "./def-builder.ts"; +import { + LexiconDirectoryIndexer, + type LexiconDirectoryIndexerOptions, +} from "./directory-indexer.ts"; +import { isSafeIdentifier } from "./ts-lang.ts"; + +export type LexBuilderOptions = LexDefBuilderOptions & { + importExt?: string; + fileExt?: string; +}; + +export type LexBuilderLoadOptions = + & LexiconDirectoryIndexerOptions + & BuildFilterOptions; + +export type LexBuilderSaveOptions = { + out: string; + clear?: boolean; + override?: boolean; +}; + +export class LexBuilder { + readonly #imported = new Set(); + readonly #project = new Project({ + useInMemoryFileSystem: true, + manipulationSettings: { indentationText: IndentationText.TwoSpaces }, + }); + + constructor(private readonly options: LexBuilderOptions = {}) {} + + get fileExt(): string { + return this.options.fileExt ?? ".ts"; + } + + get importExt(): string { + return this.options.importExt ?? ".ts"; + } + + async load(options: LexBuilderLoadOptions): Promise { + await using indexer = new FilteredIndexer( + new LexiconDirectoryIndexer(options), + buildFilter(options), + ); + + for await (const doc of indexer) { + if (!this.#imported.has(doc.id)) { + this.#imported.add(doc.id); + } else { + throw new Error(`Duplicate lexicon document id: ${doc.id}`); + } + + await this.createDefsFile(doc, indexer); + await this.createExportTree(doc); + } + } + + async save(options: LexBuilderSaveOptions): Promise { + const files = this.#project.getSourceFiles(); + const destination = resolve(options.out); + + if (options.clear) { + await rm(destination, { recursive: true, force: true }); + } else if (!options.override) { + await Promise.all( + files.map((f) => + assertNotFileExists(join(destination, f.getFilePath())) + ), + ); + } + + await Promise.all( + Array.from(files, async (file) => { + const filePath = join(destination, file.getFilePath()); + const content = file.getFullText(); + await mkdir(join(filePath, ".."), { recursive: true }); + await rm(filePath, { recursive: true, force: true }); + await writeFile(filePath, content, "utf8"); + }), + ); + } + + private createFile(path: string) { + return this.#project.createSourceFile(path); + } + + private getFile(path: string) { + return this.#project.getSourceFile(path) ?? this.createFile(path); + } + + private createExportTree(doc: LexiconDocument): void { + const namespaces = doc.id.split("."); + + for (let i = 0; i < namespaces.length - 1; i++) { + const currentNs = namespaces[i]; + const childNs = namespaces[i + 1]; + + const path = join("/", ...namespaces.slice(0, i + 1)); + const file = this.getFile(`${path}${this.fileExt}`); + + const childModuleSpecifier = `./${currentNs}/${childNs}${this.importExt}`; + const dec = file.getExportDeclaration(childModuleSpecifier); + if (!dec) { + file.addExportDeclaration({ + moduleSpecifier: childModuleSpecifier, + namespaceExport: isSafeIdentifier(childNs) + ? childNs + : JSON.stringify(childNs), + }); + } + } + + const path = join("/", ...namespaces); + const file = this.getFile(`${path}${this.fileExt}`); + + file.addExportDeclaration({ + moduleSpecifier: `./${namespaces.at(-1)}.defs${this.importExt}`, + }); + + file.addExportDeclaration({ + moduleSpecifier: `./${namespaces.at(-1)}.defs${this.importExt}`, + namespaceExport: "$defs", + }); + } + + private async createDefsFile( + doc: LexiconDocument, + indexer: LexiconIndexer, + ): Promise { + const path = join("/", ...doc.id.split(".")); + const file = this.createFile(`${path}.defs${this.fileExt}`); + + const fileBuilder = new LexDefBuilder( + { ...this.options, importExt: this.importExt }, + file, + doc, + indexer, + ); + await fileBuilder.build(); + } +} + +async function assertNotFileExists(file: string): Promise { + try { + await stat(file); + throw new Error(`File already exists: ${file}`); + } catch (err) { + if (err instanceof Error && "code" in err && err.code === "ENOENT") return; + throw err; + } +} diff --git a/lex-gen/builder/mod.ts b/lex-gen/builder/mod.ts new file mode 100644 index 0000000..0eefecd --- /dev/null +++ b/lex-gen/builder/mod.ts @@ -0,0 +1,22 @@ +export * from "./filter.ts"; +export * from "./directory-indexer.ts"; +export * from "./filtered-indexer.ts"; +export * from "./lex-builder.ts"; + +export type { + LexBuilderLoadOptions, + LexBuilderOptions, + LexBuilderSaveOptions, +} from "./lex-builder.ts"; + +export async function build( + options: + & import("./lex-builder.ts").LexBuilderOptions + & import("./lex-builder.ts").LexBuilderLoadOptions + & import("./lex-builder.ts").LexBuilderSaveOptions, +): Promise { + const { LexBuilder } = await import("./lex-builder.ts"); + const builder = new LexBuilder(options); + await builder.load(options); + await builder.save(options); +} diff --git a/lex-gen/builder/ref-resolver.ts b/lex-gen/builder/ref-resolver.ts new file mode 100644 index 0000000..06afcef --- /dev/null +++ b/lex-gen/builder/ref-resolver.ts @@ -0,0 +1,240 @@ +import assert from "node:assert"; +import { join } from "node:path"; +import type { SourceFile } from "ts-morph"; +import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document"; +import { isReservedWord, isSafeIdentifier } from "./ts-lang.ts"; +import { + asRelativePath, + memoize, + toCamelCase, + toPascalCase, + ucFirst, +} from "./util.ts"; + +export type RefResolverOptions = { + importExt?: string; +}; + +export type ResolvedRef = { + varName: string; + typeName: string; +}; + +export class RefResolver { + constructor( + private doc: LexiconDocument, + private file: SourceFile, + private indexer: LexiconIndexer, + private options: RefResolverOptions, + ) {} + + public readonly resolve = memoize( + (ref: string): Promise => { + const [nsid, hash = "main"] = ref.split("#"); + + if (nsid === "" || nsid === this.doc.id) { + return this.resolveLocal(hash); + } else { + const fullRef = `${nsid}#${hash}`; + return this.resolveExternal(fullRef); + } + }, + ); + + #defCounters = new Map(); + private nextSafeDefinitionIdentifier(safeIdentifier: string): string { + const count = this.#defCounters.get(safeIdentifier) ?? 0; + this.#defCounters.set(safeIdentifier, count + 1); + return `${safeIdentifier}$${count}`; + } + + public readonly resolveLocal = memoize( + (hash: string): Promise => { + const hashes = Object.keys(this.doc.defs); + + if (!hashes.includes(hash)) { + return Promise.reject( + new Error(`Definition ${hash} not found in ${this.doc.id}`), + ); + } + + const pub = getPublicIdentifiers(hash); + for (const otherHash of hashes) { + if (otherHash === hash) continue; + const otherPub = getPublicIdentifiers(otherHash); + if (otherPub.typeName === pub.typeName) { + return Promise.reject( + new Error( + `Conflicting type names for definitions #${hash} and #${otherHash} in ${this.doc.id}`, + ), + ); + } + } + + const safeIdentifier = asSafeDefinitionIdentifier(hash); + + const varName = safeIdentifier + ? !hashes.some((otherHash) => { + if (otherHash === hash) return false; + const otherIdentifier = asSafeDefinitionIdentifier(otherHash); + return otherIdentifier === safeIdentifier; + }) + ? safeIdentifier + : this.nextSafeDefinitionIdentifier(safeIdentifier) + : this.nextSafeDefinitionIdentifier("def"); + + const typeName = ucFirst(varName); + assert( + varName !== typeName, + "Variable and type name should be different", + ); + + return Promise.resolve({ varName, typeName }); + }, + ); + + private readonly resolveExternal = memoize( + (fullRef: string): Promise => { + const [nsid, hash] = fullRef.split("#"); + const moduleSpecifier = `${ + asRelativePath( + this.file.getDirectoryPath(), + join("/", ...nsid.split(".")), + ) + }.defs${this.options.importExt ?? ".ts"}`; + + return this.indexer.get(nsid).then((srcDoc) => { + const srcDefs = srcDoc.defs as unknown as Record; + const srcDef = Object.hasOwn(srcDoc.defs, hash) ? srcDefs[hash] : null; + if (!srcDef) { + throw new Error( + `Missing def "${hash}" in "${nsid}" (referenced from ${this.doc.id})`, + ); + } + + const nsIdentifier = this.getNsIdentifier(nsid, moduleSpecifier); + const publicIds = getPublicIdentifiers(hash); + + return { + varName: isSafeIdentifier(publicIds.varName) + ? `${nsIdentifier}.${publicIds.varName}` + : `${nsIdentifier}[${JSON.stringify(publicIds.varName)}]`, + typeName: `${nsIdentifier}.${publicIds.typeName}`, + }; + }); + }, + ); + + private getNsIdentifier(nsid: string, moduleSpecifier: string): string { + const existing = this.file.getImportDeclaration( + (imp) => + !imp.isTypeOnly() && + imp.getModuleSpecifierValue() === moduleSpecifier && + imp.getNamespaceImport() != null, + ); + + const decl = existing ?? + this.file.addImportDeclaration({ + moduleSpecifier, + namespaceImport: this.computeSafeNamespaceIdentifierFor(nsid), + }); + + return decl.getNamespaceImport()!.getText(); + } + + #nsIdentifiersCounters = new Map(); + private computeSafeNamespaceIdentifierFor(nsid: string): string { + const baseName = nsidToIdentifier(nsid) || "NS"; + + let name = baseName; + while (this.isConflictingIdentifier(name)) { + const count = this.#nsIdentifiersCounters.get(baseName) ?? 0; + this.#nsIdentifiersCounters.set(baseName, count + 1); + name = `${baseName}$$${count}`; + } + + return name; + } + + private isConflictingIdentifier(name: string): boolean { + return ( + this.conflictsWithKeywords(name) || + this.conflictsWithUtils(name) || + this.conflictsWithLocalDefs(name) || + this.conflictsWithLocalDeclarations(name) || + this.conflictsWithImports(name) + ); + } + + private conflictsWithKeywords(name: string): boolean { + return isReservedWord(name); + } + + private conflictsWithUtils(name: string): boolean { + if (name === "Main") return true; + if (name === "Record") return true; + return name.startsWith("$"); + } + + private conflictsWithLocalDefs(name: string): boolean { + return Object.keys(this.doc.defs).some((hash) => { + const identifier = toCamelCase(hash); + if (!identifier) return false; + if (identifier === name || `_${identifier}` === name) return true; + const typeName = ucFirst(identifier); + if (typeName === name || `_${typeName}` === name) return true; + return false; + }); + } + + private conflictsWithLocalDeclarations(name: string): boolean { + return ( + this.file.getVariableDeclarations().some((v) => v.getName() === name) || + this.file + .getVariableStatements() + .some((vs) => vs.getDeclarations().some((d) => d.getName() === name)) || + this.file.getTypeAliases().some((t) => t.getName() === name) || + this.file.getInterfaces().some((i) => i.getName() === name) || + this.file.getClasses().some((c) => c.getName() === name) || + this.file.getFunctions().some((f) => f.getName() === name) || + this.file.getEnums().some((e) => e.getName() === name) + ); + } + + private conflictsWithImports(name: string): boolean { + return this.file.getImportDeclarations().some( + (imp) => + imp.getDefaultImport()?.getText() === name || + imp.getNamespaceImport()?.getText() === name || + imp.getNamedImports().some( + (named) => + (named.getAliasNode()?.getText() ?? named.getName()) === name, + ), + ); + } +} + +function nsidToIdentifier(nsid: string): string | undefined { + const parts = nsid.split("."); + for (let i = 2; i < parts.length; i++) { + const identifier = toPascalCase(parts.slice(-i).join(".")); + if (isSafeIdentifier(identifier)) return identifier; + } + return undefined; +} + +export function getPublicIdentifiers(hash: string): ResolvedRef { + const varName = hash; + const typeName = toPascalCase(hash); + if (!typeName || varName === typeName || !isSafeIdentifier(typeName)) { + return { varName, typeName: `Def${typeName}` }; + } + return { varName, typeName }; +} + +function asSafeDefinitionIdentifier(name: string): string | undefined { + if (isSafeIdentifier(name) && isSafeIdentifier(ucFirst(name))) return name; + const camel = toCamelCase(name); + if (isSafeIdentifier(camel) && isSafeIdentifier(ucFirst(camel))) return camel; + return undefined; +} diff --git a/lex-gen/builder/ts-lang.ts b/lex-gen/builder/ts-lang.ts new file mode 100644 index 0000000..edb2c89 --- /dev/null +++ b/lex-gen/builder/ts-lang.ts @@ -0,0 +1,128 @@ +const RESERVED_WORDS = new Set([ + "abstract", + "arguments", + "as", + "async", + "await", + "boolean", + "break", + "byte", + "case", + "catch", + "char", + "class", + "const", + "continue", + "debugger", + "default", + "delete", + "do", + "double", + "else", + "enum", + "eval", + "export", + "extends", + "false", + "final", + "finally", + "float", + "for", + "from", + "function", + "get", + "goto", + "if", + "implements", + "import", + "in", + "instanceof", + "int", + "interface", + "let", + "long", + "native", + "new", + "null", + "of", + "package", + "private", + "protected", + "public", + "return", + "set", + "short", + "static", + "super", + "switch", + "synchronized", + "this", + "throw", + "throws", + "transient", + "true", + "try", + "typeof", + "undefined", + "using", + "var", + "void", + "volatile", + "while", + "with", + "yield", + "Array", + "Boolean", + "Buffer", + "Date", + "Error", + "Function", + "Infinity", + "JSON", + "Map", + "Math", + "NaN", + "Number", + "Object", + "Set", + "String", + "Symbol", + "console", + "document", + "global", + "globalThis", + "window", + "afterAll", + "afterEach", + "assert", + "beforeAll", + "beforeEach", + "describe", + "expect", + "it", + "test", + "__dirname", + "__filename", + "require", + "module", + "exports", + "Record", + "any", + "declare", + "never", + "number", + "object", + "string", + "symbol", + "unknown", + "constructor", + "meta", +]); + +export function isReservedWord(word: string): boolean { + return RESERVED_WORDS.has(word); +} + +export function isSafeIdentifier(name: string): boolean { + return !isReservedWord(name) && /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(name); +} diff --git a/lex-gen/builder/util.ts b/lex-gen/builder/util.ts new file mode 100644 index 0000000..0ffc628 --- /dev/null +++ b/lex-gen/builder/util.ts @@ -0,0 +1,46 @@ +import { relative } from "node:path"; + +export function memoize unknown>(fn: T): T { + const cache = new Map(); + return ((arg: string) => { + if (cache.has(arg)) return cache.get(arg); + const result = fn(arg); + cache.set(arg, result); + return result; + }) as T; +} + +export function ucFirst(str: string): string { + return str.charAt(0).toUpperCase() + str.slice(1); +} + +export function lcFirst(str: string): string { + return str.charAt(0).toLowerCase() + str.slice(1); +} + +export function toPascalCase(str: string): string { + return extractWords(str).map(toLowerCase).map(ucFirst).join(""); +} + +export function toCamelCase(str: string): string { + return lcFirst(toPascalCase(str)); +} + +function toLowerCase(str: string): string { + return str.toLowerCase(); +} + +function extractWords(str: string): string[] { + const processed = str + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/([A-Z])([A-Z][a-z])/g, "$1 $2") + .replace(/([0-9])([A-Za-z])/g, "$1 $2") + .replace(/[^a-zA-Z0-9]+/g, " ") + .trim(); + return processed ? processed.split(/\s+/) : []; +} + +export function asRelativePath(from: string, to: string): string { + const rel = relative(from, to); + return rel.startsWith("./") || rel.startsWith("../") ? rel : `./${rel}`; +} diff --git a/lex-gen/cmd/build.ts b/lex-gen/cmd/build.ts new file mode 100644 index 0000000..191f4a1 --- /dev/null +++ b/lex-gen/cmd/build.ts @@ -0,0 +1,102 @@ +import { Command } from "@cliffy/command"; +import { build } from "../builder/mod.ts"; + +const command = new Command() + .description( + "Generate TypeScript lexicon schema files from JSON lexicon definitions", + ) + .option( + "-i, --lexicons ", + "directory containing lexicon JSON files", + { default: "./lexicons" }, + ) + .option( + "-o, --out ", + "output directory for generated TS files", + { required: true, default: "./src/lexicons" }, + ) + .option("--clear", "clear output directory before generating files", { + default: false, + }) + .option( + "--override", + "override existing files (no effect when --clear is set)", + { default: false }, + ) + .option("--js", "use .js extension for imports and generated files", { + default: false, + }) + .option( + "--import-ext ", + "file extension for import statements in generated files (overrides --js)", + ) + .option( + "--file-ext ", + "file extension for generated files (overrides --js)", + ) + .option( + "--lib ", + 'package name to import the "l" schema utility from', + { default: "@atp/lex" }, + ) + .option( + "--allow-legacy-blobs", + "generate schemas that accept legacy blob references", + { default: false }, + ) + .option( + "--pure-annotations", + "add /*#__PURE__*/ annotations for tree-shaking tools", + { default: false }, + ) + .option( + "--ignore-invalid-lexicons", + "skip invalid lexicon files instead of exiting with an error", + { default: false }, + ) + .option( + "--include ", + "NSID patterns to include (supports * wildcards)", + ) + .option( + "--exclude ", + "NSID patterns to exclude (supports * wildcards)", + ) + .action(async (opts) => { + const useJs = opts.js ?? false; + const importExt = opts.importExt ?? (useJs ? ".js" : ".ts"); + const fileExt = opts.fileExt ?? (useJs ? ".js" : ".ts"); + + await build({ + lexicons: opts.lexicons, + out: opts.out, + clear: opts.clear, + override: opts.override, + importExt, + fileExt, + lib: opts.lib, + allowLegacyBlobs: opts.allowLegacyBlobs, + pureAnnotations: opts.pureAnnotations, + ignoreInvalidLexicons: opts.ignoreInvalidLexicons, + include: opts.include, + exclude: opts.exclude, + }); + + await denoFmt(opts.out); + console.log("Done."); + }); + +async function denoFmt(dir: string): Promise { + const cmd = new Deno.Command("deno", { + args: ["fmt", dir], + cwd: Deno.cwd(), + stdout: "inherit", + stderr: "inherit", + }); + const { code } = await cmd.output(); + if (code !== 0) { + console.warn(`Warning: deno fmt exited with code ${code}`); + } +} + +export default command; diff --git a/lex-gen/cmd/gen-api.ts b/lex-gen/cmd/gen-api.ts deleted file mode 100644 index 7fed63b..0000000 --- a/lex-gen/cmd/gen-api.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Command } from "@cliffy/command"; -import { - applyFileDiff, - genFileDiff, - printFileDiff, - readAllLexicons, - shouldPullLexicons, -} from "../util.ts"; -import { genClientApi } from "../codegen/client.ts"; -import { formatGeneratedFiles } from "../codegen/util.ts"; -import { loadLexiconConfig } from "../config.ts"; -import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; -import process from "node:process"; - -const command = new Command() - .description("Generate a TS client API") - .option("--js", "use .js extension for imports instead of .ts") - .option("-o, --outdir ", "dir path to write to") - .option("-i, --input ", "paths of lexicon files to include") - .option("--config ", "path to config file") - .action( - async ({ outdir, input, js, config: configPath }) => { - const config = await loadLexiconConfig(configPath); - const finalOutdir = outdir ?? config?.outdir; - const finalInput = input ?? config?.files; - - if (!finalOutdir) { - console.error("outdir is required (provide via -o/--outdir or config)"); - if (typeof Deno !== "undefined") { - Deno.exit(1); - } else { - process.exit(1); - } - } - - if (!finalInput || finalInput.length === 0) { - console.error( - "input is required (provide via -i/--input or config.files)", - ); - if (typeof Deno !== "undefined") { - Deno.exit(1); - } else { - process.exit(1); - } - } - - const filesProvidedViaCli = input !== undefined; - const needsPull = shouldPullLexicons( - config, - filesProvidedViaCli, - finalInput, - ); - if (needsPull && config?.pull) { - await pullLexicons(config.pull); - } - - const useJs = js ?? false; - const importSuffix = config?.modules?.importSuffix; - const mappings = config?.mappings; - const lexicons = readAllLexicons(finalInput); - const api = await genClientApi(lexicons, { - useJsExtension: useJs, - importSuffix: importSuffix, - mappings: mappings, - }); - const diff = genFileDiff(finalOutdir, api); - console.log("This will write the following files:"); - printFileDiff(diff); - applyFileDiff(diff); - if (typeof Deno !== "undefined") { - await formatGeneratedFiles(finalOutdir); - } - console.log("API generated."); - - if (needsPull && config?.pull) { - cleanupPullDirectory(config.pull); - } - }, - ); - -export default command; diff --git a/lex-gen/cmd/gen-md.ts b/lex-gen/cmd/gen-md.ts deleted file mode 100644 index c67a6c9..0000000 --- a/lex-gen/cmd/gen-md.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Command } from "@cliffy/command"; -import { readAllLexicons, shouldPullLexicons } from "../util.ts"; -import * as mdGen from "../mdgen/index.ts"; -import { loadLexiconConfig } from "../config.ts"; -import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; -import process from "node:process"; - -const isDeno = typeof Deno !== "undefined"; - -const command = new Command() - .description("Generate markdown documentation") - .option("-o, --output ", "Output file path") - .option("-i, --input ", "Input file path") - .option("--config ", "path to config file") - .action( - async ({ output, input, config: configPath }) => { - const config = await loadLexiconConfig(configPath); - const finalOutput = output ?? - (config?.outdir ? `${config.outdir}/docs.md` : undefined); - const finalInput = input ?? config?.files?.[0]; - - if (!finalOutput) { - console.error("output is required (provide via -o/--output or config)"); - if (isDeno) { - Deno.exit(1); - } else { - process.exit(1); - } - } - - if (!finalInput) { - console.error( - "input is required (provide via -i/--input or config.files)", - ); - if (isDeno) { - Deno.exit(1); - } else { - process.exit(1); - } - } - - if (!finalOutput.endsWith(".md")) { - console.error( - "Must supply the path to a .md file", - ); - if (isDeno) { - Deno.exit(1); - } else { - process.exit(1); - } - } - - const filesProvidedViaCli = input !== undefined; - const needsPull = shouldPullLexicons( - config, - filesProvidedViaCli, - [finalInput], - ); - if (needsPull && config?.pull) { - await pullLexicons(config.pull); - } - - const lexicons = readAllLexicons(finalInput); - await mdGen.process(finalOutput, lexicons); - - if (needsPull && config?.pull) { - cleanupPullDirectory(config.pull); - } - }, - ); - -export default command; diff --git a/lex-gen/cmd/gen-server.ts b/lex-gen/cmd/gen-server.ts deleted file mode 100644 index 21e8114..0000000 --- a/lex-gen/cmd/gen-server.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Command } from "@cliffy/command"; -import { - applyFileDiff, - genFileDiff, - printFileDiff, - readAllLexicons, - shouldPullLexicons, -} from "../util.ts"; -import { formatGeneratedFiles } from "../codegen/util.ts"; -import { genServerApi } from "../codegen/server.ts"; -import { loadLexiconConfig } from "../config.ts"; -import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; -import process from "node:process"; - -const isDeno = typeof Deno !== "undefined"; - -const command = new Command() - .description("Generate a TS server API") - .option("--js", "use .js extension for imports instead of .ts") - .option("-o, --outdir ", "dir path to write to") - .option("-i, --input ", "paths of lexicon files to include") - .option("--config ", "path to config file") - .action( - async ({ outdir, input, js, config: configPath }) => { - const config = await loadLexiconConfig(configPath); - const finalOutdir = outdir ?? config?.outdir; - const finalInput = input ?? config?.files; - - if (!finalOutdir) { - console.error("outdir is required (provide via -o/--outdir or config)"); - if (isDeno) { - Deno.exit(1); - } else { - process.exit(1); - } - } - - if (!finalInput || finalInput.length === 0) { - console.error( - "input is required (provide via -i/--input or config.files)", - ); - if (isDeno) { - Deno.exit(1); - } else { - process.exit(1); - } - } - - const filesProvidedViaCli = input !== undefined; - const needsPull = shouldPullLexicons( - config, - filesProvidedViaCli, - finalInput, - ); - if (needsPull && config?.pull) { - await pullLexicons(config.pull); - } - - const useJs = js ?? false; - const importSuffix = config?.modules?.importSuffix; - const mappings = config?.mappings; - console.log("Generating API..."); - const lexicons = readAllLexicons(finalInput); - const api = await genServerApi(lexicons, { - useJsExtension: useJs, - importSuffix: importSuffix, - mappings: mappings, - }); - console.log("API generated."); - const diff = genFileDiff(finalOutdir, api); - console.log("This will write the following files:"); - printFileDiff(diff); - applyFileDiff(diff); - if (typeof Deno !== "undefined") { - await formatGeneratedFiles(finalOutdir); - } - console.log("API generated."); - - if (needsPull && config?.pull) { - cleanupPullDirectory(config.pull); - } - }, - ); - -export default command; diff --git a/lex-gen/cmd/gen-ts-obj.ts b/lex-gen/cmd/gen-ts-obj.ts deleted file mode 100644 index 3e89a23..0000000 --- a/lex-gen/cmd/gen-ts-obj.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Command } from "@cliffy/command"; -import { genTsObj, readAllLexicons, shouldPullLexicons } from "../util.ts"; -import { loadLexiconConfig } from "../config.ts"; -import { cleanupPullDirectory, pullLexicons } from "../pull.ts"; -import process from "node:process"; - -const isDeno = typeof Deno !== "undefined"; - -const command = new Command() - .description("Generate a TS file that exports an array of lexicons") - .option("-i, --input ", "paths of the lexicon files to include") - .option("--config ", "path to config file") - .action(async ({ input, config: configPath }) => { - const config = await loadLexiconConfig(configPath); - const finalInput = input ?? config?.files; - - if (!finalInput || finalInput.length === 0) { - console.error( - "input is required (provide via -i/--input or config.files)", - ); - if (isDeno) { - Deno.exit(1); - } else { - process.exit(1); - } - } - - const filesProvidedViaCli = input !== undefined; - const finalInputArray = Array.isArray(finalInput) - ? finalInput - : [finalInput]; - const needsPull = shouldPullLexicons( - config, - filesProvidedViaCli, - finalInputArray, - ); - if (needsPull && config?.pull) { - await pullLexicons(config.pull); - } - - const lexicons = readAllLexicons(finalInput); - console.log(genTsObj(lexicons)); - - if (needsPull && config?.pull) { - cleanupPullDirectory(config.pull); - } - }); - -export default command; diff --git a/lex-gen/cmd/index.ts b/lex-gen/cmd/index.ts index 1335a55..edb68bb 100644 --- a/lex-gen/cmd/index.ts +++ b/lex-gen/cmd/index.ts @@ -1,6 +1,3 @@ -import genMd from "./gen-md.ts"; -import genApi from "./gen-api.ts"; -import genServer from "./gen-server.ts"; -import genTsObj from "./gen-ts-obj.ts"; +import build from "./build.ts"; -export { genApi, genMd, genServer, genTsObj }; +export { build }; diff --git a/lex-gen/codegen/client.ts b/lex-gen/codegen/client.ts deleted file mode 100644 index 0ec6a9d..0000000 --- a/lex-gen/codegen/client.ts +++ /dev/null @@ -1,652 +0,0 @@ -import { - IndentationText, - Project, - type SourceFile, - VariableDeclarationKind, -} from "ts-morph"; -import { type LexiconDoc, Lexicons, type LexRecord } from "@atp/lexicon"; -import { NSID } from "@atp/syntax"; -import type { GeneratedAPI } from "../types.ts"; -import { gen, lexiconsTs, utilTs } from "./common.ts"; -import { - collectExternalImports, - genCommonImports, - genImports, - genRecord, - genUserType, - genXrpcInput, - genXrpcOutput, - genXrpcParams, - resolveExternalImport, -} from "./lex-gen.ts"; -import { - type CodeGenOptions, - type DefTreeNode, - lexiconsToDefTree, - schemasToNsidTokens, - toCamelCase, - toScreamingSnakeCase, - toTitleCase, -} from "./util.ts"; - -const ATP_METHODS = { - list: "com.atproto.repo.listRecords", - get: "com.atproto.repo.getRecord", - create: "com.atproto.repo.createRecord", - put: "com.atproto.repo.putRecord", - delete: "com.atproto.repo.deleteRecord", -}; - -export async function genClientApi( - lexiconDocs: LexiconDoc[], - options?: CodeGenOptions, -): Promise { - const project = new Project({ - useInMemoryFileSystem: true, - manipulationSettings: { indentationText: IndentationText.TwoSpaces }, - }); - const api: GeneratedAPI = { files: [] }; - const lexicons = new Lexicons(lexiconDocs); - const nsidTree = lexiconsToDefTree(lexiconDocs); - const nsidTokens = schemasToNsidTokens(lexiconDocs); - for (const lexiconDoc of lexiconDocs) { - api.files.push(await lexiconTs(project, lexicons, lexiconDoc, options)); - } - api.files.push(await utilTs(project)); - api.files.push(await lexiconsTs(project, lexiconDocs, options)); - api.files.push( - await indexTs(project, lexiconDocs, nsidTree, nsidTokens, options), - ); - return api; -} - -const indexTs = ( - project: Project, - lexiconDocs: LexiconDoc[], - nsidTree: DefTreeNode[], - nsidTokens: Record, - options?: CodeGenOptions, -) => - gen(project, "/index.ts", (file) => { - const importExtension = options?.importSuffix ?? - (options?.useJsExtension ? ".js" : ".ts"); - //= import { XrpcClient, type FetchHandler, type FetchHandlerOptions } from '@atp/xrpc' - file.addImportDeclaration({ - moduleSpecifier: "@atp/xrpc", - namedImports: [ - { name: "XrpcClient" }, - { name: "FetchHandler", isTypeOnly: true }, - { name: "FetchHandlerOptions", isTypeOnly: true }, - ], - }); - //= import {schemas} from './lexicons.ts' - file.addImportDeclaration({ - moduleSpecifier: `./lexicons${importExtension}`, - namedImports: [{ name: "schemas" }], - }); - - //= import { type OmitKey, type Un$Typed } from './util.ts' - file.addImportDeclaration({ - moduleSpecifier: `./util${importExtension}`, - isTypeOnly: true, - namedImports: [ - { name: "OmitKey" }, - { name: "Un$Typed" }, - ], - }); - - // collect and import external lexicon references - const externalImports = collectExternalImports(lexiconDocs, options); - const mappings = options?.mappings; - for (const [nsid, types] of externalImports) { - const mapping = resolveExternalImport(nsid, mappings); - if (mapping) { - if (typeof mapping.imports === "string") { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: mapping.imports, - namedImports: [{ name: toTitleCase(nsid), isTypeOnly: true }], - }); - } else { - const result = mapping.imports(nsid); - if (result.type === "namespace") { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: result.from, - namespaceImport: toTitleCase(nsid), - }); - } else { - const namedImports = Array.from(types).map((typeName) => ({ - name: toTitleCase(typeName), - isTypeOnly: true, - })); - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: result.from, - namedImports, - }); - } - } - } - } - - // generate type imports and re-exports - for (const lexicon of lexiconDocs) { - const moduleSpecifier = `./types/${ - lexicon.id.split(".").join("/") - }${importExtension}`; - - const defs = Object.values(lexicon.defs); - const hasRecord = defs.some((d) => d.type === "record"); - const hasQueryOrProc = defs.some( - (d) => d.type === "query" || d.type === "procedure", - ); - const needsValue = defs.some( - (d) => - (d.type === "query" || d.type === "procedure") && d.errors?.length, - ); - - if (hasRecord || hasQueryOrProc) { - file.addImportDeclaration({ - moduleSpecifier, - isTypeOnly: !needsValue, - namespaceImport: toTitleCase(lexicon.id), - }); - } - - file - .addExportDeclaration({ moduleSpecifier }) - .setNamespaceExport(toTitleCase(lexicon.id)); - } - - // generate token enums - for (const nsidAuthority in nsidTokens) { - // export const {THE_AUTHORITY} = { - // {Name}: "{authority.the.name}" - // } - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: toScreamingSnakeCase(nsidAuthority), - initializer: [ - "{", - ...nsidTokens[nsidAuthority].map( - (nsidName) => - `${toTitleCase(nsidName)}: "${nsidAuthority}.${nsidName}",`, - ), - "}", - ].join("\n"), - }, - ], - }); - } - - //= export class AtpBaseClient {...} - const clientCls = file.addClass({ - name: "AtpBaseClient", - isExported: true, - extends: "XrpcClient", - }); - - for (const ns of nsidTree) { - //= ns: NS - clientCls.addProperty({ - name: ns.propName, - type: ns.className, - }); - } - - //= constructor (options: FetchHandler | FetchHandlerOptions) { - //= super(options, schemas) - //= {namespace declarations} - //= } - clientCls.addConstructor({ - parameters: [ - { name: "options", type: "FetchHandler | FetchHandlerOptions" }, - ], - statements: [ - "super(options, schemas)", - ...nsidTree.map( - (ns) => `this.${ns.propName} = new ${ns.className}(this)`, - ), - ], - }); - - //= /** @deprecated use `this` instead */ - //= get xrpc(): XrpcClient { - //= return this - //= } - clientCls - .addGetAccessor({ - name: "xrpc", - returnType: "XrpcClient", - statements: ["return this"], - }) - .addJsDoc("@deprecated use `this` instead"); - - // generate classes for the schemas - for (const ns of nsidTree) { - genNamespaceCls(file, ns); - } - }); - -function genNamespaceCls(file: SourceFile, ns: DefTreeNode) { - //= export class {ns}NS {...} - const cls = file.addClass({ - name: ns.className, - isExported: true, - }); - //= _client: XrpcClient - cls.addProperty({ - name: "_client", - type: "XrpcClient", - }); - - for (const userType of ns.userTypes) { - if (userType.def.type !== "record") { - continue; - } - //= type: TypeRecord - const name = NSID.parse(userType.nsid).name || ""; - cls.addProperty({ - name: toCamelCase(name), - type: `${toTitleCase(userType.nsid)}Record`, - }); - } - - for (const child of ns.children) { - //= child: ChildNS - cls.addProperty({ - name: child.propName, - type: child.className, - }); - - // recurse - genNamespaceCls(file, child); - } - - //= constructor(public client: XrpcClient) { - //= this._client = client - //= {child namespace prop declarations} - //= {record prop declarations} - //= } - cls.addConstructor({ - parameters: [ - { - name: "client", - type: "XrpcClient", - }, - ], - statements: [ - `this._client = client`, - ...ns.children.map( - (ns) => `this.${ns.propName} = new ${ns.className}(client)`, - ), - ...ns.userTypes - .filter((ut) => ut.def.type === "record") - .map((ut) => { - const name = NSID.parse(ut.nsid).name || ""; - return `this.${toCamelCase(name)} = new ${ - toTitleCase( - ut.nsid, - ) - }Record(client)`; - }), - ], - }); - - // methods - for (const userType of ns.userTypes) { - if (userType.def.type !== "query" && userType.def.type !== "procedure") { - continue; - } - const isGetReq = userType.def.type === "query"; - const moduleName = toTitleCase(userType.nsid); - const name = toCamelCase(NSID.parse(userType.nsid).name || ""); - const method = cls.addMethod({ - name, - returnType: `Promise<${moduleName}.Response>`, - }); - if (isGetReq) { - method.addParameter({ - name: "params?", - type: `${moduleName}.QueryParams`, - }); - } else if (userType.def.type === "procedure") { - method.addParameter({ - name: "data?", - type: `${moduleName}.InputSchema`, - }); - } - method.addParameter({ - name: "opts?", - type: `${moduleName}.CallOptions`, - }); - method.setBodyText( - [ - `return this._client`, - isGetReq - ? `.call('${userType.nsid}', params, undefined, opts)` - : `.call('${userType.nsid}', opts?.qp, data, opts)`, - userType.def.errors?.length - // Only add a catch block if there are custom errors - ? ` .catch((e) => { throw ${moduleName}.toKnownErr(e) })` - : "", - ].join("\n"), - ); - } - - // record api classes - for (const userType of ns.userTypes) { - if (userType.def.type !== "record") { - continue; - } - genRecordCls(file, userType.nsid, userType.def); - } -} - -function genRecordCls(file: SourceFile, nsid: string, lexRecord: LexRecord) { - //= export class {type}Record {...} - const cls = file.addClass({ - name: `${toTitleCase(nsid)}Record`, - isExported: true, - }); - //= _client: XrpcClient - cls.addProperty({ - name: "_client", - type: "XrpcClient", - }); - - //= constructor(client: XrpcClient) { - //= this._client = client - //= } - const cons = cls.addConstructor(); - cons.addParameter({ - name: "client", - type: "XrpcClient", - }); - cons.setBodyText(`this._client = client`); - - // methods - const typeModule = toTitleCase(nsid); - { - //= list() - const method = cls.addMethod({ - isAsync: true, - name: "list", - returnType: - `Promise<{cursor?: string, records: ({uri: string, value: ${typeModule}.Record})[]}>`, - }); - method.addParameter({ - name: "params", - type: `OmitKey<${ - toTitleCase(ATP_METHODS.list) - }.QueryParams, "collection">`, - }); - method.setBodyText( - [ - `const res = await this._client.call('${ATP_METHODS.list}', { collection: '${nsid}', ...params })`, - `return res.data`, - ].join("\n"), - ); - } - { - //= get() - const method = cls.addMethod({ - isAsync: true, - name: "get", - returnType: - `Promise<{uri: string, cid: string, value: ${typeModule}.Record}>`, - }); - method.addParameter({ - name: "params", - type: `OmitKey<${ - toTitleCase(ATP_METHODS.get) - }.QueryParams, "collection">`, - }); - method.setBodyText( - [ - `const res = await this._client.call('${ATP_METHODS.get}', { collection: '${nsid}', ...params })`, - `return res.data`, - ].join("\n"), - ); - } - { - //= create() - const method = cls.addMethod({ - isAsync: true, - name: "create", - returnType: "Promise<{uri: string, cid: string}>", - }); - method.addParameter({ - name: "params", - type: `OmitKey<${ - toTitleCase( - ATP_METHODS.create, - ) - }.InputSchema, "collection" | "record">`, - }); - method.addParameter({ - name: "record", - type: `Un$Typed<${typeModule}.Record>`, - }); - method.addParameter({ - name: "headers?", - type: `Record`, - }); - const maybeRkeyPart = lexRecord.key?.startsWith("literal:") - ? `rkey: '${lexRecord.key.replace("literal:", "")}', ` - : ""; - method.setBodyText( - [ - `const collection = '${nsid}'`, - `const res = await this._client.call('${ATP_METHODS.create}', undefined, { collection, ${maybeRkeyPart}...params, record: { ...record, $type: collection} }, {encoding: 'application/json', headers })`, - `return res.data`, - ].join("\n"), - ); - } - // { - // //= put() - // const method = cls.addMethod({ - // isAsync: true, - // name: 'put', - // returnType: 'Promise<{uri: string, cid: string}>', - // }) - // method.addParameter({ - // name: 'params', - // type: `OmitKey<${toTitleCase(ATP_METHODS.put)}.InputSchema, "collection" | "record">`, - // }) - // method.addParameter({ - // name: 'record', - // type: `${typeModule}.Record`, - // }) - // method.addParameter({ - // name: 'headers?', - // type: `Record`, - // }) - // method.setBodyText( - // [ - // `record.$type = '${userType.nsid}'`, - // `const res = await this._client.call('${ATP_METHODS.put}', undefined, { collection: '${userType.nsid}', record, ...params }, {encoding: 'application/json', headers})`, - // `return res.data`, - // ].join('\n'), - // ) - // } - { - //= delete() - const method = cls.addMethod({ - isAsync: true, - name: "delete", - returnType: "Promise", - }); - method.addParameter({ - name: "params", - type: `OmitKey<${ - toTitleCase( - ATP_METHODS.delete, - ) - }.InputSchema, "collection">`, - }); - method.addParameter({ - name: "headers?", - type: `Record`, - }); - - method.setBodyText( - [ - `await this._client.call('${ATP_METHODS.delete}', undefined, { collection: '${nsid}', ...params }, { headers })`, - ].join("\n"), - ); - } -} - -const lexiconTs = ( - project: Project, - lexicons: Lexicons, - lexiconDoc: LexiconDoc, - options?: CodeGenOptions, -) => - gen( - project, - `/types/${lexiconDoc.id.split(".").join("/")}.ts`, - (file) => { - // Filter out subscriptions as they are not currently generated for client - const filteredDefs = Object.fromEntries( - Object.entries(lexiconDoc.defs).filter(([_, def]) => - def.type !== "subscription" - ), - ); - const filteredDoc = { ...lexiconDoc, defs: filteredDefs }; - - const main = filteredDoc.defs.main; - if ( - main?.type === "query" || - main?.type === "procedure" - ) { - const needsXrpcError = (main.type === "query" || - main.type === "procedure") && main.errors?.length; - - //= import {HeadersMap, XRPCError} from '@atp/xrpc' - file.addImportDeclaration({ - moduleSpecifier: "@atp/xrpc", - isTypeOnly: !needsXrpcError, - namedImports: needsXrpcError - ? [{ name: "HeadersMap", isTypeOnly: true }, { name: "XRPCError" }] - : [{ name: "HeadersMap" }], - }); - } - - genCommonImports(file, lexiconDoc.id, filteredDoc); - - const imports: Map> = new Map(); - for (const defId in filteredDoc.defs) { - const def = filteredDoc.defs[defId]; - const lexUri = `${lexiconDoc.id}#${defId}`; - if (defId === "main") { - if (def.type === "query" || def.type === "procedure") { - genXrpcParams(file, lexicons, lexUri, false); - genXrpcInput(file, imports, lexicons, lexUri, false, options); - genXrpcOutput(file, imports, lexicons, lexUri, false, options); - genClientXrpcCommon(file, lexicons, lexUri); - } else if (def.type === "record") { - genRecord(file, imports, lexicons, lexUri, options); - } else { - genUserType(file, imports, lexicons, lexUri, options); - } - } else { - genUserType(file, imports, lexicons, lexUri, options); - } - } - genImports(file, imports, lexiconDoc.id, options); - return Promise.resolve(); - }, - ); - -function genClientXrpcCommon( - file: SourceFile, - lexicons: Lexicons, - lexUri: string, -) { - const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); - - //= export interface CallOptions {...} - const opts = file.addInterface({ - name: "CallOptions", - isExported: true, - }); - opts.addProperty({ name: "signal?", type: "AbortSignal" }); - opts.addProperty({ name: "headers?", type: "HeadersMap" }); - if (def.type === "procedure") { - opts.addProperty({ name: "qp?", type: "QueryParams" }); - } - if (def.type === "procedure" && def.input) { - let encodingType = "string"; - if (def.input.encoding !== "*/*") { - encodingType = def.input.encoding - .split(",") - .map((v) => `'${v.trim()}'`) - .join(" | "); - } - opts.addProperty({ - name: "encoding?", - type: encodingType, - }); - } - - // export interface Response {...} - const res = file.addInterface({ - name: "Response", - isExported: true, - }); - res.addProperty({ name: "success", type: "boolean" }); - res.addProperty({ name: "headers", type: "HeadersMap" }); - if (def.output?.schema) { - if (def.output.encoding?.includes(",")) { - res.addProperty({ name: "data", type: "OutputSchema | Uint8Array" }); - } else { - res.addProperty({ name: "data", type: "OutputSchema" }); - } - } else if (def.output?.encoding) { - res.addProperty({ name: "data", type: "Uint8Array" }); - } - - // export class {errcode}Error {...} - const customErrors: { name: string; cls: string }[] = []; - for (const error of def.errors || []) { - let name = toTitleCase(error.name); - if (!name.endsWith("Error")) name += "Error"; - const errCls = file.addClass({ - name, - extends: "XRPCError", - isExported: true, - }); - errCls.addConstructor({ - parameters: [{ name: "src", type: "XRPCError" }], - statements: [ - "super(src.status, src.error, src.message, src.headers, { cause: src })", - ], - }); - - customErrors.push({ name: error.name, cls: name }); - } - - // export function toKnownErr(err: any) {...} - file.addFunction({ - name: "toKnownErr", - isExported: true, - parameters: [{ name: "e", type: "unknown" }], - returnType: "unknown", - statements: customErrors.length - ? [ - "if (e instanceof XRPCError) {", - ...customErrors.map( - (err) => `if (e.error === '${err.name}') return new ${err.cls}(e)`, - ), - "}", - "return e", - ] - : ["return e"], - }); -} diff --git a/lex-gen/codegen/common.ts b/lex-gen/codegen/common.ts deleted file mode 100644 index d8c642c..0000000 --- a/lex-gen/codegen/common.ts +++ /dev/null @@ -1,299 +0,0 @@ -import { - type Project, - type SourceFile, - VariableDeclarationKind, -} from "ts-morph"; -import type { LexiconDoc } from "@atp/lexicon"; -import type { GeneratedFile } from "../types.ts"; -import type { CodeGenOptions } from "./util.ts"; -import { format, type Options as PrettierOptions } from "prettier"; - -const PRETTIER_OPTS: PrettierOptions = { - parser: "typescript", - tabWidth: 2, - semi: false, - singleQuote: true, - trailingComma: "all", -}; - -export const utilTs = ( - project: Project, -) => - gen(project, "/util.ts", (file) => { - file.replaceWithText(` -import type { ValidationResult } from '@atp/lexicon' - -export type OmitKey = { - [K2 in keyof T as K2 extends K ? never : K2]: T[K2] -} - -export type $Typed = V & { $type: T } -export type Un$Typed = OmitKey - -export type $Type = Hash extends 'main' - ? Id - : \`\${Id}#\${Hash}\` - -function isObject(v: V): v is V & object { - return v != null && typeof v === 'object' -} - -function is$type( - $type: unknown, - id: Id, - hash: Hash, -): $type is $Type { - return hash === 'main' - ? $type === id - : // $type === \`\${id}#\${hash}\` - typeof $type === 'string' && - $type.length === id.length + 1 + hash.length && - $type.charCodeAt(id.length) === 35 /* '#' */ && - $type.startsWith(id) && - $type.endsWith(hash) -} -${ - /** - * The construct below allows to properly distinguish open unions. Consider - * the following example: - * - * ```ts - * type Foo = { $type?: $Type<'foo', 'main'>; foo: string } - * type Bar = { $type?: $Type<'bar', 'main'>; bar: string } - * type OpenFooBarUnion = $Typed | $Typed | { $type: string } - * ``` - * - * In the context of lexicons, when there is a open union as shown above, the - * if `$type` if either `foo` or `bar`, then the object IS of type `Foo` or - * `Bar`. - * - * ```ts - * declare const obj1: OpenFooBarUnion - * if (is$typed(obj1, 'foo', 'main')) { - * obj1.$type // $Type<'foo', 'main'> - * obj1.foo // string - * } - * ``` - * - * Similarly, if an object is of type `unknown`, then the `is$typed` function - * should only return assurance about the `$type` property, which is what it - * actually checks: - * - * ```ts - * declare const obj2: unknown - * if (is$typed(obj2, 'foo', 'main')) { - * obj2.$type // $Type<'foo', 'main'> - * // @ts-expect-error - * obj2.foo - * } - * ``` - * - * The construct bellow is what makes these two scenarios possible. - */ - ""} -export type $TypedObject = V extends { - $type: $Type -} - ? V - : V extends { $type?: string } - ? V extends { $type?: infer T extends $Type } - ? V & { $type: T } - : never - : V & { $type: $Type } - -export function is$typed( - v: V, - id: Id, - hash: Hash, -): v is $TypedObject { - return isObject(v) && '$type' in v && is$type(v.$type, id, hash) -} - -export function maybe$typed( - v: V, - id: Id, - hash: Hash, -): v is V & object & { $type?: $Type } { - return ( - isObject(v) && - ('$type' in v - ? v.$type === undefined || is$type(v.$type, id, hash) - : true) - ) -} - -export type Validator = (v: unknown) => ValidationResult -export type ValidatorParam = - V extends Validator ? R : never - -/** - * Utility function that allows to convert a "validate*" utility function into a - * type predicate. - */ -export function asPredicate(validate: V) { - return function (v: T): v is T & ValidatorParam { - return validate(v).success - } -} -`); - }); - -export const lexiconsTs = ( - project: Project, - lexiconDocs: LexiconDoc[], - options?: CodeGenOptions, -) => - gen(project, "/lexicons.ts", (file) => { - const importExtension = options?.importSuffix ?? - (options?.useJsExtension ? ".js" : ".ts"); - const nsidToEnum = (nsid: string): string => { - return nsid - .split(".") - .map((word) => word[0].toUpperCase() + word.slice(1)) - .join(""); - }; - - //= import { type LexiconDoc, Lexicons } from '@atp/lexicon' - file - .addImportDeclaration({ - moduleSpecifier: "@atp/lexicon", - }) - .addNamedImports([ - { name: "LexiconDoc", isTypeOnly: true }, - { name: "Lexicons" }, - { name: "ValidationError" }, - { name: "ValidationResult", isTypeOnly: true }, - ]); - - //= import { is$typed, maybe$typed, type $Typed } from "./util${extension}" - file - .addImportDeclaration({ moduleSpecifier: `./util${importExtension}` }) - .addNamedImports([ - { name: "is$typed" }, - { name: "maybe$typed" }, - ]); - - //= export const schemaDict = {...} as const satisfies Record - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: "schemaDict", - initializer: JSON.stringify( - lexiconDocs.reduce( - (acc, cur) => ({ - ...acc, - [nsidToEnum(cur.id)]: cur, - }), - {}, - ), - null, - 2, - ) + " as Record", - }, - ], - }); - - //= export const schemas = Object.values(schemaDict) satisfies LexiconDoc[] - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: "schemas", - initializer: "Object.values(schemaDict) satisfies LexiconDoc[]", - }, - ], - }); - - //= export const lexicons: Lexicons = new Lexicons(schemas) - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: "lexicons", - type: "Lexicons", - initializer: "new Lexicons(schemas)", - }, - ], - }); - - file.addFunction({ - isExported: true, - name: "validate", - overloads: [ - { - typeParameters: ["T extends { $type: string }"], - parameters: [ - { name: "v", type: "unknown" }, - { name: "id", type: "string" }, - { name: "hash", type: "string" }, - { name: "requiredType", type: "true" }, - ], - returnType: "ValidationResult", - }, - { - typeParameters: ["T extends { $type?: string }"], - parameters: [ - { name: "v", type: "unknown" }, - { name: "id", type: "string" }, - { name: "hash", type: "string" }, - { name: "requiredType", type: "false", hasQuestionToken: true }, - ], - returnType: "ValidationResult", - }, - ], - parameters: [ - { name: "v", type: "unknown" }, - { name: "id", type: "string" }, - { name: "hash", type: "string" }, - { name: "requiredType", type: "boolean", hasQuestionToken: true }, - ], - statements: [ - // If $type is present, make sure it is valid before validating the rest of the object - "return (requiredType ? is$typed : maybe$typed)(v, id, hash) ? lexicons.validate(`${id}#${hash}`, v) : { success: false, error: new ValidationError(`Must be an object with \"${hash === 'main' ? id : `${id}#${hash}`}\" $type property`) }", - ], - returnType: "ValidationResult", - }); - - //= export const ids = {...} - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: "ids", - initializer: `{${ - lexiconDocs - .map( - (lex) => - `\n ${nsidToEnum(lex.id)}: ${JSON.stringify(lex.id)},`, - ) - .join("") - }\n} as const`, - }, - ], - }); - }); - -export async function gen( - project: Project, - path: string, - gen: (file: SourceFile) => void | Promise, -): Promise { - const file = project.createSourceFile(path); - gen(file); - await file.save(); // Save in the "in memory" file system - let content = `${banner()}${file.getFullText()}`; - if (!(typeof Deno !== "undefined")) { - content = await format(content, PRETTIER_OPTS); - } - - return { path, content }; -} - -function banner() { - return `/**\n * GENERATED CODE - DO NOT MODIFY\n */\n`; -} diff --git a/lex-gen/codegen/lex-gen.ts b/lex-gen/codegen/lex-gen.ts deleted file mode 100644 index efb98a4..0000000 --- a/lex-gen/codegen/lex-gen.ts +++ /dev/null @@ -1,1060 +0,0 @@ -import { relative as getRelativePath } from "@std/path"; -import { type JSDoc, type SourceFile, VariableDeclarationKind } from "ts-morph"; -import type { - LexArray, - LexBlob, - LexBytes, - LexCidLink, - Lexicons, - LexIpldType, - LexObject, - LexPrimitive, - LexToken, -} from "@atp/lexicon"; -import { - type CodeGenOptions, - toCamelCase, - toScreamingSnakeCase, - toTitleCase, -} from "./util.ts"; -import type { LexiconDoc, LexUserType } from "@atp/lexicon"; -import type { ImportMapping } from "../types.ts"; - -interface Commentable { - addJsDoc: ({ description }: { description: string }) => JSDoc; -} -export function genComment( - commentable: T, - def: { description?: string }, -): T { - if (def.description) { - commentable.addJsDoc({ description: def.description }); - } - return commentable; -} - -export function genCommonImports( - file: SourceFile, - baseNsid: string, - lexiconDoc: LexiconDoc, - options?: CodeGenOptions, -) { - const importExtension = options?.importSuffix ?? - (options?.useJsExtension ? ".js" : ".ts"); - const needsBlobRef = Object.values(lexiconDoc.defs).some((def: LexUserType) => - def.type === "blob" || - (def.type === "object" && - Object.values((def as LexObject).properties || {}).some((prop) => - "type" in prop && (prop.type === "blob" || - (prop.type === "array" && "items" in prop && - prop.items.type === "blob")) - )) || - (def.type === "array" && def.items.type === "blob") || - // Check record schema for blobs - (def.type === "record" && - Object.values(def.record.properties || {}).some((prop) => - "type" in prop && (prop.type === "blob" || - (prop.type === "array" && "items" in prop && - prop.items.type === "blob")) - )) || - // Check output schema for blobs - (def.type === "query" || def.type === "procedure") && - def.output?.schema?.type === "object" && - Object.values(def.output.schema.properties || {}).some((prop) => - "type" in prop && (prop.type === "blob" || - (prop.type === "array" && "items" in prop && - prop.items.type === "blob")) - ) - ); - - const needsCID = Object.values(lexiconDoc.defs).some((def: LexUserType) => - def.type === "cid-link" || - (def.type === "object" && - Object.values((def as LexObject).properties || {}).some((prop) => - "type" in prop && prop.type === "cid-link" - )) || - (def.type === "array" && def.items.type === "cid-link") || - // Check record schema for cid-links - (def.type === "record" && - Object.values(def.record.properties || {}).some((prop) => - "type" in prop && (prop.type === "cid-link" || - (prop.type === "array" && "items" in prop && - prop.items.type === "cid-link")) - )) || - // Check output schema for cid-links - (def.type === "query" || def.type === "procedure") && - def.output?.schema?.type === "object" && - Object.values(def.output.schema.properties || {}).some((prop) => - "type" in prop && (prop.type === "cid-link" || - (prop.type === "array" && "items" in prop && - prop.items.type === "cid-link")) - ) - ); - - const needsTypedValidation = Object.values(lexiconDoc.defs).some(( - def: LexUserType, - ) => def.type === "record" || def.type === "object"); - - const needsId = Object.values(lexiconDoc.defs).some(( - def: LexUserType, - ) => def.type === "token") || needsTypedValidation; - - const needsUnionType = Object.values(lexiconDoc.defs).some( - (def: LexUserType) => { - // Check direct array unions - if (def.type === "array" && def.items.type === "union") return true; - - // Check object property unions - if (def.type === "object") { - return Object.values((def as LexObject).properties || {}).some((prop) => - prop.type === "union" || - (prop.type === "array" && prop.items?.type === "union") - ); - } - - // Check record property unions - if (def.type === "record") { - return Object.values(def.record.properties || {}).some((prop) => - "type" in prop && ( - prop.type === "union" || - (prop.type === "array" && "items" in prop && - prop.items.type === "union") - ) - ); - } - - // Check procedure input/output schemas - if (def.type === "procedure") { - // Check input schema - if (def.input?.schema?.type === "union") return true; - if (def.input?.schema?.type === "object") { - return Object.values(def.input.schema.properties || {}).some((prop) => - "type" in prop && ( - prop.type === "union" || - (prop.type === "array" && "items" in prop && - prop.items.type === "union") - ) - ); - } - // Check output schema - if (def.output?.schema?.type === "union") return true; - if (def.output?.schema?.type === "object") { - return Object.values(def.output.schema.properties || {}).some(( - prop, - ) => - "type" in prop && ( - prop.type === "union" || - (prop.type === "array" && "items" in prop && - prop.items.type === "union") - ) - ); - } - } - - // Check query output schemas - if (def.type === "query") { - if (def.output?.schema?.type === "union") return true; - if (def.output?.schema?.type === "object") { - return Object.values(def.output.schema.properties || {}).some(( - prop, - ) => - "type" in prop && ( - prop.type === "union" || - (prop.type === "array" && "items" in prop && - prop.items.type === "union") - ) - ); - } - } - - // Check subscription message schemas - if (def.type === "subscription") { - if (def.message?.schema?.type === "union") return true; - if (def.message?.schema?.type === "object") { - return Object.values(def.message.schema.properties || {}).some(( - prop, - ) => - "type" in prop && ( - prop.type === "union" || - (prop.type === "array" && "items" in prop && - prop.items.type === "union") - ) - ); - } - } - - return false; - }, - ); - - //= import {BlobRef} from '@atp/lexicon' - if (needsBlobRef) { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: "@atp/lexicon", - namedImports: [{ name: "BlobRef" }], - }); - } - - //= import {CID} from 'multiformats/cid' - if (needsCID) { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: "multiformats/cid", - namedImports: [{ name: "CID" }], - }); - } - - const utilPath = `${ - baseNsid - .split(".") - .map((_str) => "..") - .join("/") - }/util${importExtension}`; - - if (needsTypedValidation) { - //= import { validate as _validate } from '../../lexicons.ts' - file - .addImportDeclaration({ - moduleSpecifier: `${ - baseNsid - .split(".") - .map((_str) => "..") - .join("/") - }/lexicons${importExtension}`, - }) - .addNamedImports([{ name: "validate", alias: "_validate" }]); - - //= import type { ValidationResult } from '@atp/lexicon' - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: "@atp/lexicon", - namedImports: [{ name: "ValidationResult" }], - }); - - // tsc adds protection against circular imports, which hurts bundle size. - // Since we know that lexicon.ts and util.ts do not depend on the file being - // generated, we can safely bypass this protection. - // Note that we are not using `import * as util from '../../util'` because - // typescript will emit is own helpers for the import, which we want to avoid. - file.addVariableStatement({ - isExported: false, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { name: "is$typed", initializer: "_is$typed" }, - { name: "validate", initializer: "_validate" }, - ], - }); - } - - const utilImports: Array< - { name: string; alias?: string; isTypeOnly?: boolean } - > = []; - if (needsTypedValidation) { - utilImports.push({ name: "is$typed", alias: "_is$typed" }); - } - if (needsUnionType) { - utilImports.push({ name: "$Typed", isTypeOnly: true }); - } - - if (utilImports.length > 0) { - const allTypeOnly = utilImports.every((imp) => imp.isTypeOnly); - if (allTypeOnly) { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: utilPath, - namedImports: utilImports.map((imp) => ({ - name: imp.name, - alias: imp.alias, - })), - }); - } else { - file - .addImportDeclaration({ - moduleSpecifier: utilPath, - }) - .addNamedImports(utilImports); - } - } - - if (needsId) { - //= const id = "{baseNsid}" - file.addVariableStatement({ - isExported: false, // Do not export to allow tree-shaking - declarationKind: VariableDeclarationKind.Const, - declarations: [{ name: "id", initializer: JSON.stringify(baseNsid) }], - }); - } -} - -export function collectExternalImports( - lexiconDocs: LexiconDoc[], - options?: CodeGenOptions, -): Map> { - const imports: Map> = new Map(); - const mappings = options?.mappings; - - // Check if any records exist (which use ATP_METHODS) - const hasRecords = lexiconDocs.some((lexiconDoc) => - Object.values(lexiconDoc.defs).some((def) => def.type === "record") - ); - - // Record classes use ATP_METHODS which may need external imports - // Note: put is commented out in genRecordCls, so we don't import it - if (hasRecords) { - const atpMethods = [ - "com.atproto.repo.listRecords", - "com.atproto.repo.getRecord", - "com.atproto.repo.createRecord", - "com.atproto.repo.deleteRecord", - ]; - for (const methodNsid of atpMethods) { - const mapping = resolveExternalImport(methodNsid, mappings); - if (mapping) { - if (!imports.has(methodNsid)) { - imports.set(methodNsid, new Set()); - } - // These methods use QueryParams, InputSchema, etc. - imports.get(methodNsid)!.add("main"); - } - } - } - return imports; -} - -export function genImports( - file: SourceFile, - imports: Map>, - baseNsid: string, - options?: CodeGenOptions, -) { - const startPath = "/" + baseNsid.split(".").slice(0, -1).join("/"); - const importExtension = options?.importSuffix ?? - (options?.useJsExtension ? ".js" : ".ts"); - const mappings = options?.mappings; - - for (const [nsid, types] of imports) { - const mapping = resolveExternalImport(nsid, mappings); - if (mapping) { - if (typeof mapping.imports === "string") { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: mapping.imports, - namedImports: [{ name: toTitleCase(nsid), isTypeOnly: true }], - }); - } else { - const result = mapping.imports(nsid); - if (result.type === "namespace") { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: result.from, - namespaceImport: toTitleCase(nsid), - }); - } else { - const namedImports = Array.from(types).map((typeName) => ({ - name: toTitleCase(typeName), - isTypeOnly: true, - })); - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: result.from, - namedImports, - }); - } - } - } else { - const targetPath = "/" + nsid.split(".").join("/") + importExtension; - let resolvedPath = getRelativePath(startPath, targetPath); - if (!resolvedPath.startsWith(".")) { - resolvedPath = `./${resolvedPath}`; - } - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: resolvedPath, - namespaceImport: toTitleCase(nsid), - }); - } - } -} - -export function genUserType( - file: SourceFile, - imports: Map>, - lexicons: Lexicons, - lexUri: string, - options?: CodeGenOptions, -) { - const def = lexicons.getDefOrThrow(lexUri); - switch (def.type) { - case "array": - genArray(file, imports, lexUri, def, options); - break; - case "token": - genToken(file, lexUri, def); - break; - case "object": { - const ifaceName: string = toTitleCase(getHash(lexUri)); - genObject(file, imports, lexUri, def, ifaceName, { - typeProperty: true, - }, options); - genObjHelpers(file, lexUri, ifaceName, { - requireTypeProperty: false, - }); - break; - } - - case "blob": - case "bytes": - case "cid-link": - case "boolean": - case "integer": - case "string": - case "unknown": - genPrimitiveOrBlob(file, lexUri, def); - break; - - default: - throw new Error( - `genLexUserType() called with wrong definition type (${def.type}) in ${lexUri}`, - ); - } -} - -function genObject( - file: SourceFile, - imports: Map>, - lexUri: string, - def: LexObject, - ifaceName: string, - { - defaultsArePresent = true, - allowUnknownProperties = false, - typeProperty = false, - }: { - defaultsArePresent?: boolean; - allowUnknownProperties?: boolean; - typeProperty?: boolean | "required"; - } = {}, - options?: CodeGenOptions, -) { - const iface = file.addInterface({ - name: ifaceName, - isExported: true, - }); - genComment(iface, def); - - if (typeProperty) { - const hash = getHash(lexUri); - const baseNsid = stripScheme(stripHash(lexUri)); - - //= $type?: - iface.addProperty({ - name: typeProperty === "required" ? `$type` : `$type?`, - type: - // Not using $Type here because it is less readable than a plain string - // `$Type<${JSON.stringify(baseNsid)}, ${JSON.stringify(hash)}>` - hash === "main" - ? JSON.stringify(`${baseNsid}`) - : JSON.stringify(`${baseNsid}#${hash}`), - }); - } - - const nullableProps = new Set(def.nullable); - if (def.properties) { - for (const propKey in def.properties) { - const propDef = def.properties[propKey]; - const propNullable = nullableProps.has(propKey); - const req = def.required?.includes(propKey) || - (defaultsArePresent && - "default" in propDef && - propDef.default !== undefined); - if (propDef.type === "ref" || propDef.type === "union") { - //= propName: External|External - const types = propDef.type === "union" - ? propDef.refs.map((ref) => - refToUnionType(ref, lexUri, imports, options?.mappings) - ) - : [ - refToType( - propDef.ref, - stripScheme(stripHash(lexUri)), - imports, - options?.mappings, - ), - ]; - if (propDef.type === "union" && !propDef.closed) { - types.push("{ $type: string }"); - } - iface.addProperty({ - name: `${propKey}${req ? "" : "?"}`, - type: makeType(types, { nullable: propNullable }), - }); - continue; - } else { - if (propDef.type === "array") { - //= propName: type[] - let propAst; - if (propDef.items.type === "ref") { - propAst = iface.addProperty({ - name: `${propKey}${req ? "" : "?"}`, - type: makeType( - refToType( - propDef.items.ref, - stripScheme(stripHash(lexUri)), - imports, - options?.mappings, - ), - { - nullable: propNullable, - array: true, - }, - ), - }); - } else if (propDef.items.type === "union") { - const types = propDef.items.refs.map((ref) => - refToUnionType(ref, lexUri, imports, options?.mappings) - ); - if (!propDef.items.closed) { - types.push("{ $type: string }"); - } - propAst = iface.addProperty({ - name: `${propKey}${req ? "" : "?"}`, - type: makeType(types, { - nullable: propNullable, - array: true, - }), - }); - } else { - propAst = iface.addProperty({ - name: `${propKey}${req ? "" : "?"}`, - type: makeType(primitiveOrBlobToType(propDef.items), { - nullable: propNullable, - array: true, - }), - }); - } - genComment(propAst, propDef); - } else { - //= propName: type - genComment( - iface.addProperty({ - name: `${propKey}${req ? "" : "?"}`, - type: makeType(primitiveOrBlobToType(propDef), { - nullable: propNullable, - }), - }), - propDef, - ); - } - } - } - - if (allowUnknownProperties) { - //= [k: string]: unknown - iface.addIndexSignature({ - keyName: "k", - keyType: "string", - returnType: "unknown", - }); - } - } -} - -export function genToken(file: SourceFile, lexUri: string, def: LexToken) { - //= /** */ - //= export const = `${id}#` - genComment( - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: toScreamingSnakeCase(getHash(lexUri)), - type: "string", - initializer: `\`\${id}#${getHash(lexUri)}\``, - }, - ], - }), - def, - ); -} - -export function genArray( - file: SourceFile, - imports: Map>, - lexUri: string, - def: LexArray, - options?: CodeGenOptions, -) { - if (def.items.type === "ref") { - file.addTypeAlias({ - name: toTitleCase(getHash(lexUri)), - type: `${ - refToType( - def.items.ref, - stripScheme(stripHash(lexUri)), - imports, - options?.mappings, - ) - }[]`, - isExported: true, - }); - } else if (def.items.type === "union") { - const types = def.items.refs.map((ref) => - refToUnionType(ref, lexUri, imports, options?.mappings) - ); - if (!def.items.closed) { - types.push("{ $type: string }"); - } - file.addTypeAlias({ - name: toTitleCase(getHash(lexUri)), - type: `(${types.join("|")})[]`, - isExported: true, - }); - } else { - genComment( - file.addTypeAlias({ - name: toTitleCase(getHash(lexUri)), - type: `${primitiveOrBlobToType(def.items)}[]`, - isExported: true, - }), - def, - ); - } -} - -export function genPrimitiveOrBlob( - file: SourceFile, - lexUri: string, - def: LexPrimitive | LexBlob | LexIpldType, -) { - genComment( - file.addTypeAlias({ - name: toTitleCase(getHash(lexUri)), - type: primitiveOrBlobToType(def), - isExported: true, - }), - def, - ); -} - -export function genXrpcParams( - file: SourceFile, - lexicons: Lexicons, - lexUri: string, - defaultsArePresent = true, -) { - const def = lexicons.getDefOrThrow(lexUri, [ - "query", - "subscription", - "procedure", - ]); - - // @NOTE We need to use a `type` here instead of an `interface` because we - // need the generated type to be used as generic type parameter like this: - // - // type QueryParams = {} // Generated by this function - // - // type MyUtil

= (...) - // type NsType = MyUtil // ERROR if `NS.QueryParams` is an `interface` - // - // Second line will fail if `NS.QueryParams` is an `interface` that does - // not explicitly extend `xrpcServer.QueryParam`, or have a string index - // signature that encompasses `xrpcServer.QueryParam`. - - //= export type QueryParams = {...} - if ( - def.parameters && def.parameters.properties && - Object.keys(def.parameters.properties).length > 0 - ) { - genComment( - file.addTypeAlias({ - name: "QueryParams", - isExported: true, - type: `{ - ${ - Object.entries(def.parameters.properties) - .map(([paramKey, paramDef]) => { - const req = def.parameters!.required?.includes(paramKey) || - (defaultsArePresent && - "default" in paramDef && - paramDef.default !== undefined); - const jsDoc = paramDef.description - ? `/** ${paramDef.description} */\n` - : ""; - return `${jsDoc}${paramKey}${req ? "" : "?"}: ${ - paramDef.type === "array" - ? primitiveToType(paramDef.items) + "[]" - : primitiveToType(paramDef) - }`; - }) - .join("\n") - } - }`, - }), - def.parameters, - ); - } else { - file.addTypeAlias({ - name: "QueryParams", - isExported: true, - type: "globalThis.Record", - }); - } -} - -export function genXrpcInput( - file: SourceFile, - imports: Map>, - lexicons: Lexicons, - lexUri: string, - defaultsArePresent = true, - options?: CodeGenOptions, -) { - const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); - - if (def.type === "procedure" && def.input?.schema) { - if (def.input.schema.type === "ref" || def.input.schema.type === "union") { - //= export type InputSchema = ... - - const types = def.input.schema.type === "union" - ? def.input.schema.refs.map((ref) => - refToUnionType(ref, lexUri, imports, options?.mappings) - ) - : [ - refToType( - def.input.schema.ref, - stripScheme(stripHash(lexUri)), - imports, - options?.mappings, - ), - ]; - - if (def.input.schema.type === "union" && !def.input.schema.closed) { - types.push("{ $type: string }"); - } - file.addTypeAlias({ - name: "InputSchema", - type: types.join("|"), - isExported: true, - }); - } else { - //= export interface InputSchema {...} - genObject(file, imports, lexUri, def.input.schema, `InputSchema`, { - defaultsArePresent, - }, options); - } - } else if (def.type === "procedure" && def.input?.encoding) { - //= export type InputSchema = string | Uint8Array | Blob - file.addTypeAlias({ - isExported: true, - name: "InputSchema", - type: "string | Uint8Array | Blob", - }); - } else { - //= export type InputSchema = undefined - file.addTypeAlias({ - isExported: true, - name: "InputSchema", - type: "undefined", - }); - } -} - -export function genXrpcOutput( - file: SourceFile, - imports: Map>, - lexicons: Lexicons, - lexUri: string, - defaultsArePresent = true, - options?: CodeGenOptions, -) { - const def = lexicons.getDefOrThrow(lexUri, [ - "query", - "subscription", - "procedure", - ]); - - const schema = def.type === "subscription" - ? def.message?.schema - : def.output?.schema; - if (schema) { - if (schema.type === "ref" || schema.type === "union") { - //= export type OutputSchema = ... - const types = schema.type === "union" - ? schema.refs.map((ref) => - refToUnionType(ref, lexUri, imports, options?.mappings) - ) - : [ - refToType( - schema.ref, - stripScheme(stripHash(lexUri)), - imports, - options?.mappings, - ), - ]; - if (schema.type === "union" && !schema.closed) { - types.push("{ $type: string }"); - } - file.addTypeAlias({ - name: "OutputSchema", - type: types.join("|"), - isExported: true, - }); - } else { - // Check if schema is empty (no properties) - const isEmpty = !schema.properties || - Object.keys(schema.properties).length === 0; - if (isEmpty) { - //= export type OutputSchema = Record - file.addTypeAlias({ - name: "OutputSchema", - type: "globalThis.Record", - isExported: true, - }); - } else { - //= export interface OutputSchema {...} - genObject(file, imports, lexUri, schema, `OutputSchema`, { - defaultsArePresent, - }, options); - } - } - } -} - -export function genRecord( - file: SourceFile, - imports: Map>, - lexicons: Lexicons, - lexUri: string, - options?: CodeGenOptions, -) { - const def = lexicons.getDefOrThrow(lexUri, ["record"]); - - //= export interface Record {...} - genObject(file, imports, lexUri, def.record, "Record", { - defaultsArePresent: true, - allowUnknownProperties: true, - typeProperty: "required", - }, options); - - //= export function isRecord(v: unknown): v is Record {...} - genObjHelpers(file, lexUri, "Record", { - requireTypeProperty: true, - }); - - const hash = getHash(lexUri); - if (hash === "main") { - //= export type Main = Record - file.addTypeAlias({ - name: "Main", - type: "Record", - isExported: true, - }); - } -} - -function genObjHelpers( - file: SourceFile, - lexUri: string, - ifaceName: string, - { - requireTypeProperty, - }: { - requireTypeProperty: boolean; - }, -) { - const hash = getHash(lexUri); - - const hashVar = `hash${ifaceName}`; - - file.addVariableStatement({ - isExported: false, - declarationKind: VariableDeclarationKind.Const, - declarations: [{ name: hashVar, initializer: JSON.stringify(hash) }], - }); - - const isX = toCamelCase(`is-${ifaceName}`); - - //= export function is{X}(v: V): v is {ifaceName} & V {...} - file - .addFunction({ - name: isX, - typeParameters: [{ name: `V` }], - parameters: [{ name: `v`, type: `V` }], - returnType: `v is ${ifaceName} & V`, - isExported: true, - }) - .setBodyText(`return is$typed(v, id, ${hashVar})`); - - const validateX = toCamelCase(`validate-${ifaceName}`); - - //= export function validate{X}(v: V): ValidationResult<{ifaceName} & V> {...} - file - .addFunction({ - name: validateX, - typeParameters: [{ name: `V` }], - parameters: [{ name: `v`, type: `V` }], - returnType: `ValidationResult<${ifaceName} & V>`, - isExported: true, - }) - .setBodyText( - `return validate<${ifaceName} & V>(v, id, ${hashVar}${ - requireTypeProperty ? ", true" : "" - })`, - ); -} - -export function stripScheme(uri: string): string { - if (uri.startsWith("lex:")) return uri.slice(4); - return uri; -} - -export function stripHash(uri: string): string { - return uri.split("#")[0] || ""; -} - -export function getHash(uri: string): string { - return uri.split("#").pop() || ""; -} - -export function ipldToType(def: LexCidLink | LexBytes) { - if (def.type === "bytes") { - return "Uint8Array"; - } - return "CID"; -} - -function refToUnionType( - ref: string, - lexUri: string, - imports: Map>, - mappings?: ImportMapping[], -): string { - const baseNsid = stripScheme(stripHash(lexUri)); - return `$Typed<${refToType(ref, baseNsid, imports, mappings)}>`; -} - -export function resolveExternalImport( - nsid: string, - mappings?: ImportMapping[], -): ImportMapping | undefined { - if (!mappings) return undefined; - return mappings.find((mapping) => { - return mapping.nsid.some((pattern) => { - if (pattern.endsWith(".*")) { - return nsid.startsWith(pattern.slice(0, -1)); - } - return nsid === pattern; - }); - }); -} - -function refToType( - ref: string, - baseNsid: string, - imports: Map>, - mappings?: ImportMapping[], -): string { - let [refBase, refHash] = ref.split("#"); - refBase = stripScheme(refBase); - if (!refHash) refHash = "main"; - - // internal - if (!refBase || baseNsid === refBase) { - return toTitleCase(refHash); - } - - // external - check if there's a mapping - const mapping = resolveExternalImport(refBase, mappings); - if (mapping) { - if (!imports.has(refBase)) { - imports.set(refBase, new Set()); - } - const types = imports.get(refBase)!; - types.add(refHash); - - if (typeof mapping.imports === "string") { - // String mapping means namespace import - return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; - } else { - const result = mapping.imports(refBase); - if (result.type === "namespace") { - return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; - } else { - // Named import - return just the type name - return toTitleCase(refHash); - } - } - } - - // external - no mapping, use relative import - if (!imports.has(refBase)) { - imports.set(refBase, new Set()); - } - return `${toTitleCase(refBase)}.${toTitleCase(refHash)}`; -} - -export function primitiveOrBlobToType( - def: LexBlob | LexPrimitive | LexIpldType, -): string { - switch (def.type) { - case "blob": - return "BlobRef"; - case "bytes": - return "Uint8Array"; - case "cid-link": - return "CID"; - default: - return primitiveToType(def); - } -} - -export function primitiveToType(def: LexPrimitive): string { - switch (def.type) { - case "string": - if (def.knownValues?.length) { - return `${ - def.knownValues - .map((v) => JSON.stringify(v)) - .join(" | ") - } | (string & globalThis.Record)`; - } else if (def.enum) { - return def.enum.map((v) => JSON.stringify(v)).join(" | "); - } else if (def.const) { - return JSON.stringify(def.const); - } - return "string"; - case "integer": - if (def.enum) { - return def.enum.map((v) => JSON.stringify(v)).join(" | "); - } else if (def.const) { - return JSON.stringify(def.const); - } - return "number"; - case "boolean": - if (def.const) { - return JSON.stringify(def.const); - } - return "boolean"; - case "unknown": - // @TODO Should we use "object" here ? - // the "Record" identifier from typescript get overwritten by the Record - // interface created by lex-cli. - return "{ [_ in string]: unknown }"; // Record - default: - throw new Error(`Unexpected primitive type: ${JSON.stringify(def)}`); - } -} - -function makeType( - _types: string | string[], - opts?: { array?: boolean; nullable?: boolean }, -) { - const types = ([] as string[]).concat(_types); - if (opts?.nullable) types.push("null"); - const arr = opts?.array ? "[]" : ""; - if (types.length === 1) return `(${types[0]})${arr}`; - if (arr) return `(${types.join(" | ")})${arr}`; - return types.join(" | "); -} diff --git a/lex-gen/codegen/server.ts b/lex-gen/codegen/server.ts deleted file mode 100644 index 1ac8ac2..0000000 --- a/lex-gen/codegen/server.ts +++ /dev/null @@ -1,503 +0,0 @@ -import { - IndentationText, - Project, - type SourceFile, - VariableDeclarationKind, -} from "ts-morph"; -import { type LexiconDoc, Lexicons } from "@atp/lexicon"; -import { NSID } from "@atp/syntax"; -import type { GeneratedAPI } from "../types.ts"; -import { gen, lexiconsTs, utilTs } from "./common.ts"; -import { - collectExternalImports, - genCommonImports, - genImports, - genRecord, - genUserType, - genXrpcInput, - genXrpcOutput, - genXrpcParams, - resolveExternalImport, -} from "./lex-gen.ts"; -import { - type CodeGenOptions, - type DefTreeNode, - lexiconsToDefTree, - schemasToNsidTokens, - toCamelCase, - toScreamingSnakeCase, - toTitleCase, -} from "./util.ts"; - -export async function genServerApi( - lexiconDocs: LexiconDoc[], - options?: CodeGenOptions, -): Promise { - const project = new Project({ - useInMemoryFileSystem: true, - manipulationSettings: { indentationText: IndentationText.TwoSpaces }, - }); - const api: GeneratedAPI = { files: [] }; - const lexicons = new Lexicons(lexiconDocs); - const nsidTree = lexiconsToDefTree(lexiconDocs); - const nsidTokens = schemasToNsidTokens(lexiconDocs); - for (const lexiconDoc of lexiconDocs) { - api.files.push(await lexiconTs(project, lexicons, lexiconDoc, options)); - } - api.files.push(await utilTs(project)); - api.files.push(await lexiconsTs(project, lexiconDocs)); - api.files.push( - await indexTs(project, lexiconDocs, nsidTree, nsidTokens, options), - ); - return api; -} - -const indexTs = ( - project: Project, - lexiconDocs: LexiconDoc[], - nsidTree: DefTreeNode[], - nsidTokens: Record, - options?: CodeGenOptions, -) => - gen(project, "/index.ts", (file) => { - const importExtension = options?.importSuffix ?? - (options?.useJsExtension ? ".js" : ".ts"); - - // Check if there are any subscription types - const hasSubscriptions = lexiconDocs.some((doc) => - doc.defs.main?.type === "subscription" - ); - - //= import {createServer as createXrpcServer, Server as XrpcServer} from '@atp/xrpc-server' - const namedImports = [ - { name: "Auth", isTypeOnly: true }, - { name: "Options", alias: "XrpcOptions", isTypeOnly: true }, - { name: "Server", alias: "XrpcServer", isTypeOnly: true }, - { name: "MethodConfigOrHandler", isTypeOnly: true }, - { name: "createServer", alias: "createXrpcServer" }, - ]; - - if (hasSubscriptions) { - namedImports.splice(3, 0, { - name: "StreamConfigOrHandler", - isTypeOnly: true, - }); - } - - file.addImportDeclaration({ - moduleSpecifier: "@atp/xrpc-server", - namedImports, - }); - //= import {schemas} from './lexicons.ts' - file - .addImportDeclaration({ - moduleSpecifier: `./lexicons${importExtension}`, - }) - .addNamedImport({ - name: "schemas", - }); - - // collect and import external lexicon references - const externalImports = collectExternalImports(lexiconDocs, options); - const mappings = options?.mappings; - for (const [nsid, types] of externalImports) { - const mapping = resolveExternalImport(nsid, mappings); - if (mapping) { - if (typeof mapping.imports === "string") { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: mapping.imports, - namedImports: [{ name: toTitleCase(nsid), isTypeOnly: true }], - }); - } else { - const result = mapping.imports(nsid); - if (result.type === "namespace") { - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: result.from, - namespaceImport: toTitleCase(nsid), - }); - } else { - const namedImports = Array.from(types).map((typeName) => ({ - name: toTitleCase(typeName), - isTypeOnly: true, - })); - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: result.from, - namedImports, - }); - } - } - } - } - - // generate type imports - for (const lexiconDoc of lexiconDocs) { - if ( - lexiconDoc.defs.main?.type !== "query" && - lexiconDoc.defs.main?.type !== "subscription" && - lexiconDoc.defs.main?.type !== "procedure" - ) { - continue; - } - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: `./types/${ - lexiconDoc.id.split(".").join("/") - }${importExtension}`, - namespaceImport: toTitleCase(lexiconDoc.id), - }); - } - - // generate token enums - for (const nsidAuthority in nsidTokens) { - // export const {THE_AUTHORITY} = { - // {Name}: "{authority.the.name}" - // } - file.addVariableStatement({ - isExported: true, - declarationKind: VariableDeclarationKind.Const, - declarations: [ - { - name: toScreamingSnakeCase(nsidAuthority), - initializer: [ - "{", - ...nsidTokens[nsidAuthority].map( - (nsidName) => - `${toTitleCase(nsidName)}: "${nsidAuthority}.${nsidName}",`, - ), - "}", - ].join("\n"), - }, - ], - }); - } - - //= export function createServer(options?: XrpcOptions) { ... } - const createServerFn = file.addFunction({ - name: "createServer", - returnType: "Server", - parameters: [ - { name: "options", type: "XrpcOptions", hasQuestionToken: true }, - ], - isExported: true, - }); - createServerFn.setBodyText(`return new Server(options)`); - - //= export class Server {...} - const serverCls = file.addClass({ - name: "Server", - isExported: true, - }); - //= xrpc: XrpcServer = createXrpcServer(methodSchemas) - serverCls.addProperty({ - name: "xrpc", - type: "XrpcServer", - }); - - // generate classes for the schemas - for (const ns of nsidTree) { - //= ns: NS - serverCls.addProperty({ - name: ns.propName, - type: ns.className, - }); - - // class... - genNamespaceCls(file, ns); - } - - //= constructor (options?: XrpcOptions) { - //= this.xrpc = createXrpcServer(schemas, options) - //= {namespace declarations} - //= } - serverCls - .addConstructor({ - parameters: [ - { name: "options", type: "XrpcOptions", hasQuestionToken: true }, - ], - }) - .setBodyText( - [ - "this.xrpc = createXrpcServer(schemas, options)", - ...nsidTree.map( - (ns) => `this.${ns.propName} = new ${ns.className}(this)`, - ), - ].join("\n"), - ); - }); - -function genNamespaceCls(file: SourceFile, ns: DefTreeNode) { - //= export class {ns}NS {...} - const cls = file.addClass({ - name: ns.className, - isExported: true, - }); - //= _server: Server - cls.addProperty({ - name: "_server", - type: "Server", - }); - - for (const child of ns.children) { - //= child: ChildNS - cls.addProperty({ - name: child.propName, - type: child.className, - }); - - // recurse - genNamespaceCls(file, child); - } - - //= constructor(server: Server) { - //= this._server = server - //= {child namespace declarations} - //= } - const cons = cls.addConstructor(); - cons.addParameter({ - name: "server", - type: "Server", - }); - cons.setBodyText( - [ - `this._server = server`, - ...ns.children.map( - (ns) => `this.${ns.propName} = new ${ns.className}(server)`, - ), - ].join("\n"), - ); - - // methods - for (const userType of ns.userTypes) { - if ( - userType.def.type !== "query" && - userType.def.type !== "subscription" && - userType.def.type !== "procedure" - ) { - continue; - } - const moduleName = toTitleCase(userType.nsid); - const name = toCamelCase(NSID.parse(userType.nsid).name || ""); - const isSubscription = userType.def.type === "subscription"; - const method = cls.addMethod({ - name, - typeParameters: [ - { - name: "A", - constraint: "Auth", - default: "void", - }, - ], - }); - method.addParameter({ - name: "cfg", - type: isSubscription - ? `StreamConfigOrHandler< - A, - ${moduleName}.QueryParams, - ${moduleName}.HandlerOutput, - >` - : `MethodConfigOrHandler< - A, - ${moduleName}.QueryParams, - ${moduleName}.HandlerInput, - ${moduleName}.HandlerOutput, - >`, - }); - const methodType = isSubscription ? "streamMethod" : "method"; - method.setBodyText( - [ - `const nsid = '${userType.nsid}' // @ts-ignore - dynamically generated`, - `return this._server.xrpc.${methodType}(nsid, cfg)`, - ].join("\n"), - ); - } -} - -const lexiconTs = ( - project: Project, - lexicons: Lexicons, - lexiconDoc: LexiconDoc, - options?: CodeGenOptions, -) => - gen( - project, - `/types/${lexiconDoc.id.split(".").join("/")}.ts`, - (file) => { - const main = lexiconDoc.defs.main; - if (main?.type === "query" || main?.type === "procedure") { - const streamingInput = main?.type === "procedure" && - main.input?.encoding && - !main.input.schema; - const streamingOutput = main.output?.encoding && !main.output.schema; - if (streamingInput || streamingOutput) { - //= ReadableStream is a web standard API - // No import needed for ReadableStream - } - } - - genCommonImports(file, lexiconDoc.id, lexiconDoc); - - const imports: Map> = new Map(); - for (const defId in lexiconDoc.defs) { - const def = lexiconDoc.defs[defId]; - const lexUri = `${lexiconDoc.id}#${defId}`; - if (defId === "main") { - if (def.type === "query" || def.type === "procedure") { - genXrpcParams(file, lexicons, lexUri); - genXrpcInput(file, imports, lexicons, lexUri, false, options); - genXrpcOutput(file, imports, lexicons, lexUri, false, options); - genServerXrpcMethod(file, lexicons, lexUri); - } else if (def.type === "subscription") { - genXrpcParams(file, lexicons, lexUri); - genXrpcOutput(file, imports, lexicons, lexUri, false, options); - genServerXrpcStreaming(file, lexicons, lexUri); - } else if (def.type === "record") { - genRecord(file, imports, lexicons, lexUri, options); - } else { - genUserType(file, imports, lexicons, lexUri, options); - } - } else { - genUserType(file, imports, lexicons, lexUri, options); - } - } - genImports(file, imports, lexiconDoc.id, options); - }, - ); - -function genServerXrpcMethod( - file: SourceFile, - lexicons: Lexicons, - lexUri: string, -) { - const def = lexicons.getDefOrThrow(lexUri, ["query", "procedure"]); - - //= export interface HandlerInput {...} - if (def.type === "procedure" && def.input?.encoding) { - const handlerInput = file.addInterface({ - name: "HandlerInput", - isExported: true, - }); - - handlerInput.addProperty({ - name: "encoding", - type: def.input.encoding - .split(",") - .map((v) => `'${v.trim()}'`) - .join(" | "), - }); - handlerInput.addProperty({ - name: "body", - type: def.input.schema - ? def.input.encoding.includes(",") - ? "InputSchema | ReadableStream" - : "InputSchema" - : "ReadableStream", - }); - } else { - file.addTypeAlias({ - isExported: true, - name: "HandlerInput", - type: "void", - }); - } - - // export interface HandlerSuccess {...} - let hasHandlerSuccess = false; - if (def.output?.schema || def.output?.encoding) { - hasHandlerSuccess = true; - const handlerSuccess = file.addInterface({ - name: "HandlerSuccess", - isExported: true, - }); - - if (def.output.encoding) { - handlerSuccess.addProperty({ - name: "encoding", - type: def.output.encoding - .split(",") - .map((v) => `'${v.trim()}'`) - .join(" | "), - }); - } - if (def.output?.schema) { - if (def.output.encoding.includes(",")) { - handlerSuccess.addProperty({ - name: "body", - type: "OutputSchema | Uint8Array | ReadableStream", - }); - } else { - handlerSuccess.addProperty({ name: "body", type: "OutputSchema" }); - } - } else if (def.output?.encoding) { - handlerSuccess.addProperty({ - name: "body", - type: "Uint8Array | ReadableStream", - }); - } - handlerSuccess.addProperty({ - name: "headers?", - type: "{ [key: string]: string }", - }); - } - - // export interface HandlerError {...} - const handlerError = file.addInterface({ - name: "HandlerError", - isExported: true, - }); - handlerError.addProperties([ - { name: "status", type: "number" }, - { name: "message?", type: "string" }, - ]); - if (def.errors?.length) { - handlerError.addProperty({ - name: "error?", - type: def.errors.map((err) => `'${err.name}'`).join(" | "), - }); - } - - // export type HandlerOutput = ... - file.addTypeAlias({ - isExported: true, - name: "HandlerOutput", - type: `HandlerError | ${hasHandlerSuccess ? "HandlerSuccess" : "void"}`, - }); -} - -function genServerXrpcStreaming( - file: SourceFile, - lexicons: Lexicons, - lexUri: string, -) { - const def = lexicons.getDefOrThrow(lexUri, ["subscription"]); - - file.addImportDeclaration({ - isTypeOnly: true, - moduleSpecifier: "@atp/xrpc-server", - namedImports: [{ name: "ErrorFrame" }], - }); - - // export type HandlerError = ... - file.addTypeAlias({ - name: "HandlerError", - isExported: true, - type: `ErrorFrame<${arrayToUnion(def.errors?.map((e) => e.name))}>`, - }); - - // export type HandlerOutput = ... - file.addTypeAlias({ - isExported: true, - name: "HandlerOutput", - type: `HandlerError | ${def.message?.schema ? "OutputSchema" : "void"}`, - }); -} - -function arrayToUnion(arr?: string[]) { - if (!arr?.length) { - return "never"; - } - return arr.map((item) => `'${item}'`).join(" | "); -} diff --git a/lex-gen/codegen/util.ts b/lex-gen/codegen/util.ts deleted file mode 100644 index 170c768..0000000 --- a/lex-gen/codegen/util.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { LexiconDoc, LexUserType } from "@atp/lexicon"; -import { NSID } from "@atp/syntax"; -import type { ImportMapping } from "../types.ts"; - -export interface CodeGenOptions { - useJsExtension?: boolean; - importSuffix?: string; - mappings?: ImportMapping[]; -} - -export interface DefTreeNodeUserType { - nsid: string; - def: LexUserType; -} - -export interface DefTreeNode { - name: string; - className: string; - propName: string; - children: DefTreeNode[]; - userTypes: DefTreeNodeUserType[]; -} - -export function lexiconsToDefTree(lexicons: LexiconDoc[]): DefTreeNode[] { - const tree: DefTreeNode[] = []; - for (const lexicon of lexicons) { - if (!lexicon.defs.main) { - continue; - } - const node = getOrCreateNode(tree, lexicon.id.split(".").slice(0, -1)); - node.userTypes.push({ nsid: lexicon.id, def: lexicon.defs.main }); - } - return tree; -} - -function getOrCreateNode(tree: DefTreeNode[], path: string[]): DefTreeNode { - let node: DefTreeNode | undefined; - for (let i = 0; i < path.length; i++) { - const segment = path[i]; - node = tree.find((v) => v.name === segment); - if (!node) { - node = { - name: segment, - className: `${toTitleCase(path.slice(0, i + 1).join("-"))}NS`, - propName: toCamelCase(segment), - children: [], - userTypes: [], - } as DefTreeNode; - tree.push(node); - } - tree = node.children; - } - if (!node) throw new Error(`Invalid schema path: ${path.join(".")}`); - return node; -} - -export function schemasToNsidTokens( - lexiconDocs: LexiconDoc[], -): Record { - const nsidTokens: Record = {}; - for (const lexiconDoc of lexiconDocs) { - const nsidp = NSID.parse(lexiconDoc.id); - if (!nsidp.name) continue; - for (const defId in lexiconDoc.defs) { - const def = lexiconDoc.defs[defId]; - if (def.type !== "token") continue; - const authority = nsidp.segments.slice(0, -1).join("."); - nsidTokens[authority] ??= []; - nsidTokens[authority].push( - nsidp.name + (defId === "main" ? "" : `#${defId}`), - ); - } - } - return nsidTokens; -} - -export function toTitleCase(v: string): string { - v = v.replace(/^([a-z])/gi, (_, g) => g.toUpperCase()); // upper-case first letter - v = v.replace(/[.#-]([a-z])/gi, (_, g) => g.toUpperCase()); // uppercase any dash, dot, or hash segments - return v.replace(/[.-]/g, ""); // remove lefover dashes or dots -} - -export function toCamelCase(v: string): string { - v = v.replace(/[.#-]([a-z])/gi, (_, g) => g.toUpperCase()); // uppercase any dash, dot, or hash segments - return v.replace(/[.-]/g, ""); // remove lefover dashes or dots -} - -export function toScreamingSnakeCase(v: string): string { - v = v.replace(/[.#-]+/gi, "_"); // convert dashes, dots, and hashes into underscores - return v.toUpperCase(); // and scream! -} - -export async function formatGeneratedFiles(outDir: string) { - console.log("Formatting generated files..."); - const cmd = new Deno.Command("deno", { - args: ["fmt", outDir], - cwd: Deno.cwd(), - }); - - const { code, stderr } = await cmd.output(); - - if (code !== 0) { - const errorMsg = new TextDecoder().decode(stderr); - console.warn(`Warning: deno fmt failed: ${errorMsg}`); - } else { - console.log("Files formatted successfully."); - } -} diff --git a/lex-gen/config.ts b/lex-gen/config.ts deleted file mode 100644 index dfb684e..0000000 --- a/lex-gen/config.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { NSID } from "@atp/syntax"; -import { parse } from "@std/jsonc"; -import type { LexiconConfig } from "./types.ts"; - -function isValidLexiconPattern(pattern: string): boolean { - if (pattern.endsWith(".*")) { - try { - NSID.parse(`${pattern.slice(0, -2)}.x`); - return true; - } catch { - return false; - } - } - return NSID.isValid(pattern); -} - -function validateConfig(config: LexiconConfig): void { - if (!config.outdir || config.outdir.length === 0) { - throw new Error("outdir must not be empty"); - } - - if (!config.files || config.files.length === 0) { - throw new Error("files must include at least one glob pattern"); - } - - for (const file of config.files) { - if (!file || file.length === 0) { - throw new Error("files must not contain empty strings"); - } - } - - if (config.mappings) { - for (const mapping of config.mappings) { - if (!mapping.nsid || mapping.nsid.length === 0) { - throw new Error("mappings.nsid requires at least one pattern"); - } - - for (const pattern of mapping.nsid) { - if (!isValidLexiconPattern(pattern)) { - throw new Error( - `invalid NSID pattern: ${pattern} (must be valid NSID or end with .*)`, - ); - } - } - - if (typeof mapping.imports === "string") { - if (mapping.imports.length === 0) { - throw new Error("mappings.imports must not be empty"); - } - } else if (typeof mapping.imports !== "function") { - throw new Error("mappings.imports must be a string or function"); - } - } - } - - if (config.modules?.importSuffix !== undefined) { - if (config.modules.importSuffix.length === 0) { - throw new Error("modules.importSuffix must not be empty"); - } - } - - if (config.pull) { - if (!config.pull.outdir || config.pull.outdir.length === 0) { - throw new Error("pull.outdir must not be empty"); - } - - if (!config.pull.sources || config.pull.sources.length === 0) { - throw new Error("pull.sources must include at least one source"); - } - - for (const source of config.pull.sources) { - if (source.type === "git") { - if (!source.remote || source.remote.length === 0) { - throw new Error("pull.sources[].remote must not be empty"); - } - - if (source.ref !== undefined && source.ref.length === 0) { - throw new Error("pull.sources[].ref must not be empty"); - } - - if (!source.pattern || source.pattern.length === 0) { - throw new Error( - "pull.sources[].pattern must include at least one glob pattern", - ); - } - - for (const pattern of source.pattern) { - if (!pattern || pattern.length === 0) { - throw new Error( - "pull.sources[].pattern must not contain empty strings", - ); - } - } - } - } - } -} - -export function defineLexiconConfig(config: LexiconConfig): LexiconConfig { - validateConfig(config); - return config; -} - -export async function loadLexiconConfig( - configPath?: string, -): Promise { - if (!configPath) { - const possiblePaths = [ - "./lexicon.config.json", - "./lexicon.config.jsonc", - ]; - for (const path of possiblePaths) { - try { - if (typeof Deno !== "undefined") { - const stat = Deno.statSync(path); - if (stat.isFile) { - configPath = path; - break; - } - } - } catch { - continue; - } - } - } - - if (!configPath) { - return null; - } - - try { - const content = typeof Deno !== "undefined" - ? Deno.readTextFileSync(configPath) - : (await import("node:fs")).readFileSync(configPath, "utf-8"); - - const parsed = parse(content) as unknown as LexiconConfig; - return defineLexiconConfig(parsed); - } catch (error) { - console.warn(`Failed to load config from ${configPath}:`, error); - return null; - } -} diff --git a/lex-gen/deno.json b/lex-gen/deno.json index 864d88c..4748353 100644 --- a/lex-gen/deno.json +++ b/lex-gen/deno.json @@ -9,8 +9,6 @@ "@std/fs": "jsr:@std/fs@^1.0.19", "@std/jsonc": "jsr:@std/jsonc@^1.0.1", "@std/path": "jsr:@std/path@^1.1.2", - "prettier": "npm:prettier@^3.6.2", - "ts-morph": "jsr:@ts-morph/ts-morph@^26.0.0", - "zod": "jsr:@zod/zod@^4.1.11" + "ts-morph": "jsr:@ts-morph/ts-morph@^26.0.0" } } diff --git a/lex-gen/mdgen/index.ts b/lex-gen/mdgen/index.ts deleted file mode 100644 index 312f96e..0000000 --- a/lex-gen/mdgen/index.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { readFileSync } from "@std/fs/unstable-read-file"; -import { writeFileSync } from "@std/fs/unstable-write-file"; -import type { LexiconDoc } from "@atp/lexicon"; - -const INSERT_START = [ - "", - "", -]; -const INSERT_END = [ - "", -]; - -export async function process(outFilePath: string, lexicons: LexiconDoc[]) { - let existingContent = ""; - try { - existingContent = new TextDecoder().decode(readFileSync(outFilePath)); - } catch { - // ignore - no existing content - } - const fileLines: StringTree = existingContent.split("\n"); - - // find previously generated content - let startIndex = fileLines.findIndex((line) => matchesStart(line as string)); - let endIndex = fileLines.findIndex((line) => matchesEnd(line as string)); - if (startIndex === -1) { - startIndex = fileLines.length; - } - if (endIndex === -1) { - endIndex = fileLines.length; - } - - // generate & insert content - fileLines.splice(startIndex, endIndex - startIndex + 1, [ - INSERT_START, - await genMdLines(lexicons), - INSERT_END, - ]); - - writeFileSync(outFilePath, new TextEncoder().encode(merge(fileLines))); -} - -function genMdLines(lexicons: LexiconDoc[]): StringTree { - const doc: StringTree = []; - for (const lexicon of lexicons) { - console.log(lexicon.id); - const desc: StringTree = []; - if (lexicon.description) { - desc.push(lexicon.description, ``); - } - doc.push([ - `---`, - ``, - `## ${lexicon.id}`, - "", - desc, - "```json", - JSON.stringify(lexicon, null, 2), - "```", - ]); - } - return doc; -} - -type StringTree = (StringTree | string | undefined)[]; -function merge(arr: StringTree): string { - return arr - .flat(10) - .filter((v) => typeof v === "string") - .join("\n"); -} - -function matchesStart(line: string) { - return /