From b2acf2fe5c9bac3a0107fcf7933f7a6fcbc99f75 Mon Sep 17 00:00:00 2001 From: Roscoe Rubin-Rottenberg Date: Thu, 16 Apr 2026 08:54:24 -0400 Subject: [PATCH] feat: lex resolver and installer --- deno.lock | 2 + lex-gen/builder/mod.ts | 23 +- lex-gen/cmd/index.ts | 3 +- lex-gen/mod.ts | 21 +- {lex-gen/builder => lex/build}/def-builder.ts | 8 +- .../build}/directory-indexer.ts | 2 +- {lex-gen/builder => lex/build}/filter.ts | 0 .../builder => lex/build}/filtered-indexer.ts | 4 +- {lex-gen/builder => lex/build}/lex-builder.ts | 2 +- lex/build/mod.ts | 22 + .../builder => lex/build}/ref-resolver.ts | 2 +- {lex-gen/builder => lex/build}/ts-lang.ts | 0 {lex-gen/builder => lex/build}/util.ts | 0 lex/cli.ts | 15 + {lex-gen/cmd => lex/cli}/build.ts | 2 +- lex/cli/install.ts | 43 ++ lex/deno.json | 18 +- lex/document/lexicon.ts | 8 +- lex/installer/fs.ts | 24 ++ lex/installer/install.ts | 45 ++ lex/installer/lex-installer-error.ts | 10 + lex/installer/lex-installer.ts | 369 +++++++++++++++++ lex/installer/lexicons-manifest.ts | 57 +++ lex/installer/mod.ts | 7 + lex/installer/nsid-map.ts | 75 ++++ lex/installer/nsid-set.ts | 71 ++++ lex/mod.ts | 5 + lex/resolver/lex-resolver-error.ts | 20 + lex/resolver/lex-resolver.ts | 386 ++++++++++++++++++ lex/resolver/mod.ts | 2 + {lex-gen => lex}/tests/lex-builder_test.ts | 2 +- lex/tests/lex-installer_test.ts | 308 ++++++++++++++ lex/tests/lex-resolver_test.ts | 321 +++++++++++++++ .../tests/method-generation_test.ts | 2 +- 34 files changed, 1833 insertions(+), 46 deletions(-) rename {lex-gen/builder => lex/build}/def-builder.ts (99%) rename {lex-gen/builder => lex/build}/directory-indexer.ts (98%) rename {lex-gen/builder => lex/build}/filter.ts (100%) rename {lex-gen/builder => lex/build}/filtered-indexer.ts (89%) rename {lex-gen/builder => lex/build}/lex-builder.ts (98%) create mode 100644 lex/build/mod.ts rename {lex-gen/builder => lex/build}/ref-resolver.ts (99%) rename {lex-gen/builder => lex/build}/ts-lang.ts (100%) rename {lex-gen/builder => lex/build}/util.ts (100%) create mode 100644 lex/cli.ts rename {lex-gen/cmd => lex/cli}/build.ts (98%) create mode 100644 lex/cli/install.ts create mode 100644 lex/installer/fs.ts create mode 100644 lex/installer/install.ts create mode 100644 lex/installer/lex-installer-error.ts create mode 100644 lex/installer/lex-installer.ts create mode 100644 lex/installer/lexicons-manifest.ts create mode 100644 lex/installer/mod.ts create mode 100644 lex/installer/nsid-map.ts create mode 100644 lex/installer/nsid-set.ts create mode 100644 lex/resolver/lex-resolver-error.ts create mode 100644 lex/resolver/lex-resolver.ts create mode 100644 lex/resolver/mod.ts rename {lex-gen => lex}/tests/lex-builder_test.ts (96%) create mode 100644 lex/tests/lex-installer_test.ts create mode 100644 lex/tests/lex-resolver_test.ts rename {lex-gen => lex}/tests/method-generation_test.ts (98%) diff --git a/deno.lock b/deno.lock index 4607e5e..8cb271e 100644 --- a/deno.lock +++ b/deno.lock @@ -1142,6 +1142,8 @@ }, "lex": { "dependencies": [ + "jsr:@cliffy/command@^1.0.0-rc.8", + "jsr:@ts-morph/ts-morph@26", "npm:cborg@^4.2.15", "npm:multiformats@^13.4.1" ] diff --git a/lex-gen/builder/mod.ts b/lex-gen/builder/mod.ts index 0eefecd..74eb23a 100644 --- a/lex-gen/builder/mod.ts +++ b/lex-gen/builder/mod.ts @@ -1,22 +1 @@ -export * from "./filter.ts"; -export * from "./directory-indexer.ts"; -export * from "./filtered-indexer.ts"; -export * from "./lex-builder.ts"; - -export type { - LexBuilderLoadOptions, - LexBuilderOptions, - LexBuilderSaveOptions, -} from "./lex-builder.ts"; - -export async function build( - options: - & import("./lex-builder.ts").LexBuilderOptions - & import("./lex-builder.ts").LexBuilderLoadOptions - & import("./lex-builder.ts").LexBuilderSaveOptions, -): Promise { - const { LexBuilder } = await import("./lex-builder.ts"); - const builder = new LexBuilder(options); - await builder.load(options); - await builder.save(options); -} +export * from "@atp/lex/build"; diff --git a/lex-gen/cmd/index.ts b/lex-gen/cmd/index.ts index b0eb559..1335a55 100644 --- a/lex-gen/cmd/index.ts +++ b/lex-gen/cmd/index.ts @@ -1,7 +1,6 @@ -import build from "./build.ts"; import genMd from "./gen-md.ts"; import genApi from "./gen-api.ts"; import genServer from "./gen-server.ts"; import genTsObj from "./gen-ts-obj.ts"; -export { build, genApi, genMd, genServer, genTsObj }; +export { genApi, genMd, genServer, genTsObj }; diff --git a/lex-gen/mod.ts b/lex-gen/mod.ts index cdff5bc..1a2f831 100644 --- a/lex-gen/mod.ts +++ b/lex-gen/mod.ts @@ -11,18 +11,17 @@ * * @example * ```bash - * lex-gen build -i ./lexicons -o ./lex + * lex-gen api -i ./lexicons -o ./api * ``` * * @module */ import { Command } from "@cliffy/command"; -import { build, genApi, genMd, genServer, genTsObj } from "./cmd/index.ts"; import { defineLexiconConfig, loadLexiconConfig } from "./config.ts"; import process from "node:process"; export { defineLexiconConfig, loadLexiconConfig }; -export { build as buildCommand } from "./builder/mod.ts"; +export { build } from "./builder/mod.ts"; export type { LexBuilderLoadOptions, LexBuilderOptions, @@ -38,6 +37,19 @@ export type { } from "./types.ts"; const isDeno = typeof Deno !== "undefined"; +const args = isDeno ? Deno.args : process.argv.slice(2); + +const [ + { default: genApi }, + { default: genMd }, + { default: genServer }, + { default: genTsObj }, +] = await Promise.all([ + import("./cmd/gen-api.ts"), + import("./cmd/gen-md.ts"), + import("./cmd/gen-server.ts"), + import("./cmd/gen-ts-obj.ts"), +]); await new Command() .name("lex-gen") @@ -46,5 +58,4 @@ await new Command() .command("md", genMd) .command("server", genServer) .command("ts-obj", genTsObj) - .command("build", build) - .parse(isDeno ? Deno.args : process.argv.slice(2)); + .parse(args); diff --git a/lex-gen/builder/def-builder.ts b/lex/build/def-builder.ts similarity index 99% rename from lex-gen/builder/def-builder.ts rename to lex/build/def-builder.ts index 3287b6d..6fdbc1f 100644 --- a/lex-gen/builder/def-builder.ts +++ b/lex/build/def-builder.ts @@ -25,7 +25,7 @@ import type { LexiconUnknown, MainLexiconDefinition, NamedLexiconDefinition, -} from "@atp/lex/document"; +} from "../document/mod.ts"; import { getPublicIdentifiers, @@ -753,7 +753,7 @@ export class LexDefBuilder { if (def.knownValues?.length) { return ( - def.knownValues.map((v) => JSON.stringify(v)).join(" | ") + + def.knownValues.map((v: string) => JSON.stringify(v)).join(" | ") + " | l.UnknownString" ); } @@ -813,7 +813,7 @@ export class LexDefBuilder { } const refs = await Promise.all( - def.refs.map(async (ref) => { + def.refs.map(async (ref: string) => { const { varName, typeName } = await this.refResolver.resolve(ref); return this.pure( `l.typedRef<${typeName}>(() => ${varName})`, @@ -828,7 +828,7 @@ export class LexDefBuilder { private async compileRefUnionType(def: LexiconRefUnion): Promise { const types = await Promise.all( - def.refs.map(async (ref) => { + def.refs.map(async (ref: string) => { const { typeName } = await this.refResolver.resolve(ref); return `l.TypedRef<${typeName}>`; }), diff --git a/lex-gen/builder/directory-indexer.ts b/lex/build/directory-indexer.ts similarity index 98% rename from lex-gen/builder/directory-indexer.ts rename to lex/build/directory-indexer.ts index 606066e..2a46dde 100644 --- a/lex-gen/builder/directory-indexer.ts +++ b/lex/build/directory-indexer.ts @@ -3,7 +3,7 @@ import { type LexiconDocument, lexiconDocumentSchema, LexiconIterableIndexer, -} from "@atp/lex/document"; +} from "../document/mod.ts"; export type LexiconDirectoryIndexerOptions = { lexicons: string; diff --git a/lex-gen/builder/filter.ts b/lex/build/filter.ts similarity index 100% rename from lex-gen/builder/filter.ts rename to lex/build/filter.ts diff --git a/lex-gen/builder/filtered-indexer.ts b/lex/build/filtered-indexer.ts similarity index 89% rename from lex-gen/builder/filtered-indexer.ts rename to lex/build/filtered-indexer.ts index 6112aaf..3db733d 100644 --- a/lex-gen/builder/filtered-indexer.ts +++ b/lex/build/filtered-indexer.ts @@ -1,8 +1,8 @@ -import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document"; +import type { LexiconDocument, LexiconIndexer } from "../document/mod.ts"; import type { Filter } from "./filter.ts"; export class FilteredIndexer implements LexiconIndexer, AsyncDisposable { - protected readonly returned = new Set(); + protected readonly returned: Set = new Set(); constructor( readonly indexer: LexiconIndexer & AsyncIterable, diff --git a/lex-gen/builder/lex-builder.ts b/lex/build/lex-builder.ts similarity index 98% rename from lex-gen/builder/lex-builder.ts rename to lex/build/lex-builder.ts index 0cccf67..d8885fb 100644 --- a/lex-gen/builder/lex-builder.ts +++ b/lex/build/lex-builder.ts @@ -1,7 +1,7 @@ import { mkdir, rm, stat, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { IndentationText, Project } from "ts-morph"; -import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document"; +import type { LexiconDocument, LexiconIndexer } from "../document/mod.ts"; import { buildFilter, type BuildFilterOptions } from "./filter.ts"; import { FilteredIndexer } from "./filtered-indexer.ts"; import { LexDefBuilder, type LexDefBuilderOptions } from "./def-builder.ts"; diff --git a/lex/build/mod.ts b/lex/build/mod.ts new file mode 100644 index 0000000..0eefecd --- /dev/null +++ b/lex/build/mod.ts @@ -0,0 +1,22 @@ +export * from "./filter.ts"; +export * from "./directory-indexer.ts"; +export * from "./filtered-indexer.ts"; +export * from "./lex-builder.ts"; + +export type { + LexBuilderLoadOptions, + LexBuilderOptions, + LexBuilderSaveOptions, +} from "./lex-builder.ts"; + +export async function build( + options: + & import("./lex-builder.ts").LexBuilderOptions + & import("./lex-builder.ts").LexBuilderLoadOptions + & import("./lex-builder.ts").LexBuilderSaveOptions, +): Promise { + const { LexBuilder } = await import("./lex-builder.ts"); + const builder = new LexBuilder(options); + await builder.load(options); + await builder.save(options); +} diff --git a/lex-gen/builder/ref-resolver.ts b/lex/build/ref-resolver.ts similarity index 99% rename from lex-gen/builder/ref-resolver.ts rename to lex/build/ref-resolver.ts index 8d0f07f..87ae2b5 100644 --- a/lex-gen/builder/ref-resolver.ts +++ b/lex/build/ref-resolver.ts @@ -1,7 +1,7 @@ import assert from "node:assert"; import { join } from "node:path"; import type { SourceFile } from "ts-morph"; -import type { LexiconDocument, LexiconIndexer } from "@atp/lex/document"; +import type { LexiconDocument, LexiconIndexer } from "../document/mod.ts"; import { isReservedWord, isSafeIdentifier } from "./ts-lang.ts"; import { asRelativePath, diff --git a/lex-gen/builder/ts-lang.ts b/lex/build/ts-lang.ts similarity index 100% rename from lex-gen/builder/ts-lang.ts rename to lex/build/ts-lang.ts diff --git a/lex-gen/builder/util.ts b/lex/build/util.ts similarity index 100% rename from lex-gen/builder/util.ts rename to lex/build/util.ts diff --git a/lex/cli.ts b/lex/cli.ts new file mode 100644 index 0000000..742f0f6 --- /dev/null +++ b/lex/cli.ts @@ -0,0 +1,15 @@ +import { Command } from "@cliffy/command"; +import buildCommand from "./cli/build.ts"; +import installCommand from "./cli/install.ts"; + +const command = new Command() + .name("lex") + .description("AT Protocol Lex tools") + .command("build", buildCommand) + .command("install", installCommand); + +if (import.meta.main) { + await command.parse(Deno.args); +} + +export default command; diff --git a/lex-gen/cmd/build.ts b/lex/cli/build.ts similarity index 98% rename from lex-gen/cmd/build.ts rename to lex/cli/build.ts index 0d74867..4e7560f 100644 --- a/lex-gen/cmd/build.ts +++ b/lex/cli/build.ts @@ -1,5 +1,5 @@ import { Command } from "@cliffy/command"; -import { build } from "../builder/mod.ts"; +import { build } from "../build/mod.ts"; const command = new Command() .description( diff --git a/lex/cli/install.ts b/lex/cli/install.ts new file mode 100644 index 0000000..700d1b9 --- /dev/null +++ b/lex/cli/install.ts @@ -0,0 +1,43 @@ +import { Command } from "@cliffy/command"; + +const command = new Command() + .description("Fetch and install lexicon documents") + .arguments("[additions...:string]") + .option( + "-i, --lexicons ", + "directory containing lexicon JSON files", + { default: "./lexicons" }, + ) + .option( + "-m, --manifest ", + "path to lexicons manifest file", + { default: "./lexicons.json" }, + ) + .option( + "--save [save:boolean]", + "write the updated lexicons manifest to disk", + { default: true }, + ) + .option( + "--update", + "re-resolve and re-install existing lexicons instead of reusing local files", + { default: false }, + ) + .option( + "--ci", + "error if the current install would change the manifest", + { default: false }, + ) + .action(async (opts, ...additions: string[]) => { + const { install } = await import("../installer/mod.ts"); + await install({ + add: additions, + lexicons: opts.lexicons, + manifest: opts.manifest, + save: opts.save, + update: opts.update, + ci: opts.ci, + }); + }); + +export default command; diff --git a/lex/deno.json b/lex/deno.json index f78ccc4..cba55fb 100644 --- a/lex/deno.json +++ b/lex/deno.json @@ -4,18 +4,30 @@ "exports": { ".": "./mod.ts", "./cbor": "./cbor/mod.ts", - "./document": "./document/mod.ts" + "./document": "./document/mod.ts", + "./build": "./build/mod.ts", + "./installer": "./installer/mod.ts", + "./resolver": "./resolver/mod.ts" }, "license": "MIT", "imports": { + "@cliffy/command": "jsr:@cliffy/command@^1.0.0-rc.8", "cborg": "npm:cborg@^4.2.15", "multiformats/cid": "npm:multiformats@^13.4.1/cid", "multiformats/hashes/digest": "npm:multiformats@^13.4.1/hashes/digest", - "multiformats/hashes/sha2": "npm:multiformats@^13.4.1/hashes/sha2" + "multiformats/hashes/sha2": "npm:multiformats@^13.4.1/hashes/sha2", + "ts-morph": "jsr:@ts-morph/ts-morph@^26.0.0" + }, + "test": { + "permissions": { + "env": true, + "read": true, + "write": true + } }, "lint": { "rules": { - "exclude": ["no-explicit-any", "no-slow-types", "require-await"] + "exclude": ["no-explicit-any"] } } } diff --git a/lex/document/lexicon.ts b/lex/document/lexicon.ts index 1525c69..622d9ff 100644 --- a/lex/document/lexicon.ts +++ b/lex/document/lexicon.ts @@ -1,4 +1,4 @@ -import { l } from "../mod.ts"; +import * as l from "../external.ts"; const bool: l.BooleanSchema = l.boolean(); const int: l.IntegerSchema = l.integer(); @@ -183,7 +183,11 @@ const requirePropertiesRefinement: l.RefinementCheck<{ required?: string[]; properties: Record; }> = { - check: (v) => !v.required || v.required.every((k) => k in v.properties), + check: ( + value: { required?: string[]; properties: Record }, + ) => + !value.required || + value.required.every((key: string) => key in value.properties), message: "All required parameters must be defined in properties", path: "required", }; diff --git a/lex/installer/fs.ts b/lex/installer/fs.ts new file mode 100644 index 0000000..24f3593 --- /dev/null +++ b/lex/installer/fs.ts @@ -0,0 +1,24 @@ +import { dirname } from "node:path"; + +export async function readJsonFile(path: string): Promise { + const contents = await Deno.readTextFile(path); + return JSON.parse(contents); +} + +export async function writeJsonFile( + path: string, + data: unknown, +): Promise { + await Deno.mkdir(dirname(path), { recursive: true }); + await Deno.writeTextFile(path, JSON.stringify(data, null, 2)); +} + +export function isEnoentError(err: unknown): boolean { + return err instanceof Deno.errors.NotFound || + ( + err instanceof Error && + "code" in err && + typeof err.code === "string" && + err.code === "ENOENT" + ); +} diff --git a/lex/installer/install.ts b/lex/installer/install.ts new file mode 100644 index 0000000..82d3a5d --- /dev/null +++ b/lex/installer/install.ts @@ -0,0 +1,45 @@ +import { LexInstaller } from "./lex-installer.ts"; +import { isEnoentError, readJsonFile } from "./fs.ts"; +import { LexInstallerError } from "./lex-installer-error.ts"; +import { + type LexiconsManifest, + lexiconsManifestSchema, +} from "./lexicons-manifest.ts"; +import type { LexInstallerOptions } from "./lex-installer.ts"; + +export interface LexInstallOptions extends LexInstallerOptions { + add?: string[]; + save?: boolean; + ci?: boolean; +} + +export async function install(options: LexInstallOptions): Promise { + const manifest = await readJsonFile(options.manifest).then( + (json) => lexiconsManifestSchema.parse(json) as LexiconsManifest, + (cause: unknown) => { + if (isEnoentError(cause)) return undefined; + throw new LexInstallerError("Failed to read lexicons manifest", { + cause, + }); + }, + ); + + const installer = new LexInstaller(options); + try { + await installer.install({ + additions: new Set(options.add ?? []), + manifest, + write: !options.ci, + }); + + if (options.ci) { + if (!manifest || !installer.equals(manifest)) { + throw new LexInstallerError("Lexicons manifest is out of date"); + } + } else if (options.save) { + await installer.save(); + } + } finally { + await installer[Symbol.asyncDispose](); + } +} diff --git a/lex/installer/lex-installer-error.ts b/lex/installer/lex-installer-error.ts new file mode 100644 index 0000000..e9916e1 --- /dev/null +++ b/lex/installer/lex-installer-error.ts @@ -0,0 +1,10 @@ +export class LexInstallerError extends Error { + override name = "LexInstallerError"; + + constructor( + public readonly description = "Could not install Lexicons", + options?: ErrorOptions, + ) { + super(description, options); + } +} diff --git a/lex/installer/lex-installer.ts b/lex/installer/lex-installer.ts new file mode 100644 index 0000000..d095732 --- /dev/null +++ b/lex/installer/lex-installer.ts @@ -0,0 +1,369 @@ +import { join } from "node:path"; +import { type Cid, cidForLex } from "../cbor/mod.ts"; +import { parseCid } from "../data/cid.ts"; +import type { LexValue } from "../data/lex.ts"; +import type { + LexiconDocument, + LexiconParameters, + LexiconPermission, + LexiconRef, + LexiconRefUnion, + LexiconUnknown, + MainLexiconDefinition, + NamedLexiconDefinition, +} from "../document/mod.ts"; +import { LexiconDirectoryIndexer } from "../build/mod.ts"; +import { + LexResolver, + type LexResolverFetchOptions, + type LexResolverOptions, + type LexResolverResult, +} from "../resolver/mod.ts"; +import { AtUri, ensureValidDid, NSID } from "@atp/syntax"; +import { isEnoentError, writeJsonFile } from "./fs.ts"; +import { LexInstallerError } from "./lex-installer-error.ts"; +import { + type LexiconsManifest, + normalizeLexiconsManifest, +} from "./lexicons-manifest.ts"; +import { NsidMap } from "./nsid-map.ts"; +import { NsidSet } from "./nsid-set.ts"; + +const LEXICON_COLLECTION = "com.atproto.lexicon.schema"; + +export interface LexInstallerResolver { + resolve(nsidStr: NSID | string): Promise; + fetch( + uriStr: AtUri | string, + options?: LexResolverFetchOptions, + ): Promise; +} + +export interface LexInstallerOptions extends LexResolverOptions { + lexicons: string; + manifest: string; + update?: boolean; + resolver?: LexInstallerResolver; +} + +export interface LexInstallerFetchResult { + lexicon: LexiconDocument; + cid: Cid; +} + +export class LexInstaller implements AsyncDisposable { + protected readonly lexiconResolver: LexInstallerResolver; + protected readonly indexer: LexiconDirectoryIndexer; + protected readonly documents: NsidMap = new NsidMap< + LexiconDocument + >(); + protected readonly manifest: LexiconsManifest = { + version: 1, + lexicons: [], + resolutions: {}, + }; + + constructor(protected readonly options: LexInstallerOptions) { + this.lexiconResolver = options.resolver ?? new LexResolver(options); + this.indexer = new LexiconDirectoryIndexer({ + lexicons: options.lexicons, + }); + } + + async [Symbol.asyncDispose](): Promise { + await this.indexer[Symbol.asyncDispose](); + } + + equals(manifest: LexiconsManifest): boolean { + return JSON.stringify(normalizeLexiconsManifest(manifest)) === + JSON.stringify(normalizeLexiconsManifest(this.manifest)); + } + + async install( + { + additions, + manifest, + write = true, + }: { + additions?: Iterable; + manifest?: LexiconsManifest; + write?: boolean; + } = {}, + ): Promise { + const roots = new NsidMap(); + + for (const addition of new Set(additions ?? [])) { + const [nsid, uri] = addition.startsWith("at://") + ? ((parsedUri) => [NSID.from(parsedUri.rkey), parsedUri] as const)( + new AtUri(addition), + ) + : [NSID.from(addition), null] as const; + + if (roots.has(nsid)) { + throw new LexInstallerError( + `Duplicate lexicon addition: ${nsid} (${ + roots.get(nsid) ?? addition + })`, + ); + } + + roots.set(nsid, uri); + } + + if (manifest) { + for (const lexiconId of manifest.lexicons) { + const nsid = NSID.from(lexiconId); + if (roots.has(nsid)) continue; + + const resolution = manifest.resolutions[lexiconId]; + roots.set(nsid, resolution ? new AtUri(resolution.uri) : null); + } + } + + await Promise.all( + Array.from(roots, async ([nsid, uri]) => { + const { lexicon } = uri + ? await this.installFromUri(uri, { write }) + : await this.installFromNsid(nsid, { write }); + this.manifest.lexicons.push(lexicon.id); + }), + ); + + let installedCount = 0; + do { + const missing = Array.from(this.getMissingIds()); + installedCount = missing.length; + + await Promise.all( + missing.map(async (nsid) => { + const resolution = manifest?.resolutions[nsid.toString()]; + if (resolution?.uri) { + await this.installFromUri(new AtUri(resolution.uri), { write }); + } else { + await this.installFromNsid(nsid, { write }); + } + }), + ); + } while (installedCount > 0); + } + + async save(): Promise { + await writeJsonFile( + this.options.manifest, + normalizeLexiconsManifest(this.manifest), + ); + } + + async fetch( + uri: AtUri, + { write = true }: { write?: boolean } = {}, + ): Promise { + const { lexicon, cid } = await this.lexiconResolver.fetch(uri, { + noCache: this.options.update, + }); + const normalizedCid = parseCid(cid.toString()); + if (write) { + const filePath = join(this.options.lexicons, ...lexicon.id.split(".")) + + ".json"; + await writeJsonFile(filePath, lexicon); + } + return { lexicon, cid: normalizedCid }; + } + + protected getMissingIds(): NsidSet { + const missing = new NsidSet(); + + for (const document of this.documents.values()) { + for (const nsid of listDocumentNsidRefs(document)) { + if (!this.documents.has(nsid)) { + missing.add(nsid); + } + } + } + + return missing; + } + + protected async installFromNsid( + nsid: NSID, + options?: { write?: boolean }, + ): Promise<{ lexicon: LexiconDocument; uri: AtUri }> { + const uri = await this.lexiconResolver.resolve(nsid); + return this.installFromUri(uri, options); + } + + protected async installFromUri( + uri: AtUri, + { write = true }: { write?: boolean } = {}, + ): Promise<{ lexicon: LexiconDocument; uri: AtUri }> { + assertLexiconUri(uri); + + const { lexicon, cid } = this.options.update + ? await this.fetch(uri, { write }) + : await this.indexer.get(uri.rkey).then( + async (existingLexicon) => ({ + lexicon: existingLexicon, + cid: await cidForLexicon(existingLexicon), + }), + async (cause) => { + if (isEnoentError(cause)) return await this.fetch(uri, { write }); + throw cause; + }, + ); + + this.documents.set(NSID.from(lexicon.id), lexicon); + this.manifest.resolutions[lexicon.id] = { + cid: cid.toString(), + uri: uri.toString(), + }; + + return { lexicon, uri }; + } +} + +function assertLexiconUri(uri: AtUri): void { + if (uri.collection !== LEXICON_COLLECTION) { + throw new LexInstallerError( + `Invalid lexicon URI collection for ${uri}: expected ${LEXICON_COLLECTION}`, + ); + } + + try { + ensureValidDid(uri.host); + } catch (cause) { + throw new LexInstallerError( + `Invalid lexicon URI authority for ${uri}: expected DID authority`, + { cause }, + ); + } +} + +function cidForLexicon(lexicon: LexiconDocument): Promise { + return cidForLex(lexicon as unknown as LexValue); +} + +function* listDocumentNsidRefs(doc: LexiconDocument): Iterable { + try { + for (const def of Object.values(doc.defs)) { + if (!def) continue; + for (const ref of defRefs(def)) { + const [nsid] = ref.split("#", 1); + if (nsid) { + yield NSID.from(nsid); + } + } + } + } catch (cause) { + throw new LexInstallerError( + `Failed to extract refs from lexicon ${doc.id}`, + { cause }, + ); + } +} + +function* defRefs( + def: + | MainLexiconDefinition + | NamedLexiconDefinition + | LexiconPermission + | LexiconParameters + | LexiconRef + | LexiconRefUnion + | LexiconUnknown, +): Iterable { + switch (def.type) { + case "string": + for (const value of def.knownValues ?? []) { + const [nsid, hash, extra] = value.split("#"); + if (!nsid || !hash || extra) continue; + try { + NSID.from(nsid); + yield value; + } catch { + continue; + } + } + return; + case "array": + yield* defRefs(def.items); + return; + case "params": + case "object": + for (const property of Object.values(def.properties)) { + yield* defRefs(property); + } + return; + case "union": + yield* def.refs; + return; + case "ref": + yield def.ref; + return; + case "record": + yield* defRefs(def.record); + return; + case "procedure": + if (def.input?.schema) { + yield* defRefs(def.input.schema); + } + if (def.output?.schema) { + yield* defRefs(def.output.schema); + } + if (def.parameters) { + yield* defRefs(def.parameters); + } + return; + case "query": + if (def.output?.schema) { + yield* defRefs(def.output.schema); + } + if (def.parameters) { + yield* defRefs(def.parameters); + } + return; + case "subscription": + if (def.parameters) { + yield* defRefs(def.parameters); + } + if (def.message?.schema) { + yield* defRefs(def.message.schema); + } + return; + case "permission-set": + for (const permission of def.permissions) { + yield* defRefs(permission); + } + return; + case "permission": + if (def.resource === "rpc" && Array.isArray(def.lxm)) { + for (const lxm of def.lxm) { + if (typeof lxm === "string") { + yield lxm; + } + } + } + if (def.resource === "repo" && Array.isArray(def.collection)) { + for (const collection of def.collection) { + if (typeof collection === "string") { + yield collection; + } + } + } + return; + case "boolean": + case "blob": + case "bytes": + case "cid-link": + case "integer": + case "token": + case "unknown": + return; + default: { + throw new LexInstallerError( + `Unknown lexicon def type: ${ + (def as { type?: string }).type ?? "unknown" + }`, + ); + } + } +} diff --git a/lex/installer/lexicons-manifest.ts b/lex/installer/lexicons-manifest.ts new file mode 100644 index 0000000..1e50941 --- /dev/null +++ b/lex/installer/lexicons-manifest.ts @@ -0,0 +1,57 @@ +import * as l from "../external.ts"; + +export const lexiconsManifestSchema: l.ObjectSchema<{ + version: l.LiteralSchema<1>; + lexicons: l.ArraySchema>; + resolutions: l.DictSchema< + l.StringSchema<{ format: "nsid" }>, + l.ObjectSchema<{ + uri: l.StringSchema<{ format: "at-uri" }>; + cid: l.StringSchema<{ format: "cid" }>; + }> + >; +}> = l.object({ + version: l.literal(1), + lexicons: l.array(l.string({ format: "nsid" })), + resolutions: l.dict( + l.string({ format: "nsid" }), + l.object({ + uri: l.string({ format: "at-uri" }), + cid: l.string({ format: "cid" }), + }), + ), +}); + +export interface LexiconsManifestResolution { + uri: string; + cid: string; +} + +export interface LexiconsManifest { + version: 1; + lexicons: string[]; + resolutions: Record; +} + +export function normalizeLexiconsManifest( + manifest: LexiconsManifest, +): LexiconsManifest { + return lexiconsManifestSchema.parse({ + version: manifest.version, + lexicons: [...manifest.lexicons].sort(), + resolutions: Object.fromEntries( + Object.entries(manifest.resolutions) + .sort(compareObjectEntries) + .map(([key, value]) => [key, { uri: value.uri, cid: value.cid }]), + ), + }) as LexiconsManifest; +} + +function compareObjectEntries( + a: [string, unknown], + b: [string, unknown], +): number { + if (a[0] > b[0]) return 1; + if (a[0] < b[0]) return -1; + return 0; +} diff --git a/lex/installer/mod.ts b/lex/installer/mod.ts new file mode 100644 index 0000000..805c999 --- /dev/null +++ b/lex/installer/mod.ts @@ -0,0 +1,7 @@ +export * from "./fs.ts"; +export * from "./install.ts"; +export * from "./lex-installer-error.ts"; +export * from "./lex-installer.ts"; +export * from "./lexicons-manifest.ts"; +export * from "./nsid-map.ts"; +export * from "./nsid-set.ts"; diff --git a/lex/installer/nsid-map.ts b/lex/installer/nsid-map.ts new file mode 100644 index 0000000..83de4fe --- /dev/null +++ b/lex/installer/nsid-map.ts @@ -0,0 +1,75 @@ +import { NSID } from "@atp/syntax"; + +class MappedMap { + readonly #map = new Map(); + + constructor( + private readonly encodeKey: (key: K) => I, + private readonly decodeKey: (enc: I) => K, + ) {} + + get size(): number { + return this.#map.size; + } + + clear(): void { + this.#map.clear(); + } + + set(key: K, value: V): this { + this.#map.set(this.encodeKey(key), value); + return this; + } + + get(key: K): V | undefined { + return this.#map.get(this.encodeKey(key)); + } + + has(key: K): boolean { + return this.#map.has(this.encodeKey(key)); + } + + delete(key: K): boolean { + return this.#map.delete(this.encodeKey(key)); + } + + values(): IterableIterator { + return this.#map.values(); + } + + *keys(): IterableIterator { + for (const key of this.#map.keys()) { + yield this.decodeKey(key); + } + } + + *entries(): IterableIterator<[K, V]> { + for (const [key, value] of this.#map.entries()) { + yield [this.decodeKey(key), value]; + } + } + + forEach( + callbackfn: (value: V, key: K, map: MappedMap) => void, + thisArg?: unknown, + ): void { + for (const [key, value] of this) { + callbackfn.call(thisArg, value, key, this); + } + } + + [Symbol.iterator](): IterableIterator<[K, V]> { + return this.entries(); + } + + readonly [Symbol.toStringTag] = "MappedMap"; +} + +export class NsidMap extends MappedMap { + constructor() { + super( + (key) => key.toString(), + (enc) => NSID.from(enc), + ); + } +} diff --git a/lex/installer/nsid-set.ts b/lex/installer/nsid-set.ts new file mode 100644 index 0000000..4155b3f --- /dev/null +++ b/lex/installer/nsid-set.ts @@ -0,0 +1,71 @@ +import { NSID } from "@atp/syntax"; + +class MappedSet { + readonly #set = new Set(); + + constructor( + private readonly encodeValue: (val: K) => I, + private readonly decodeValue: (enc: I) => K, + ) {} + + get size(): number { + return this.#set.size; + } + + clear(): void { + this.#set.clear(); + } + + add(val: K): this { + this.#set.add(this.encodeValue(val)); + return this; + } + + has(val: K): boolean { + return this.#set.has(this.encodeValue(val)); + } + + delete(val: K): boolean { + return this.#set.delete(this.encodeValue(val)); + } + + *values(): IterableIterator { + for (const val of this.#set.values()) { + yield this.decodeValue(val); + } + } + + keys(): IterableIterator { + return this.values(); + } + + *entries(): IterableIterator<[K, K]> { + for (const value of this) { + yield [value, value]; + } + } + + forEach( + callbackfn: (value: K, value2: K, set: MappedSet) => void, + thisArg?: unknown, + ): void { + for (const value of this) { + callbackfn.call(thisArg, value, value, this); + } + } + + [Symbol.iterator](): IterableIterator { + return this.values(); + } + + readonly [Symbol.toStringTag] = "MappedSet"; +} + +export class NsidSet extends MappedSet { + constructor() { + super( + (val) => val.toString(), + (enc) => NSID.from(enc), + ); + } +} diff --git a/lex/mod.ts b/lex/mod.ts index 55542f7..1e9ce0b 100644 --- a/lex/mod.ts +++ b/lex/mod.ts @@ -2,3 +2,8 @@ import * as l from "./external.ts"; export { l }; export * from "./external.ts"; + +if (import.meta.main) { + const { default: command } = await import("./cli.ts"); + void command.parse(Deno.args); +} diff --git a/lex/resolver/lex-resolver-error.ts b/lex/resolver/lex-resolver-error.ts new file mode 100644 index 0000000..3b00e8b --- /dev/null +++ b/lex/resolver/lex-resolver-error.ts @@ -0,0 +1,20 @@ +import { NSID } from "@atp/syntax"; + +export class LexResolverError extends Error { + override name = "LexResolverError"; + + constructor( + public readonly nsid: NSID, + public readonly description = "Could not resolve Lexicon for NSID", + options?: ErrorOptions, + ) { + super(`${description} (${nsid})`, options); + } + + static from(nsid: NSID | string, description?: string): LexResolverError { + return new LexResolverError( + typeof nsid === "string" ? NSID.from(nsid) : nsid, + description, + ); + } +} diff --git a/lex/resolver/lex-resolver.ts b/lex/resolver/lex-resolver.ts new file mode 100644 index 0000000..fc134e1 --- /dev/null +++ b/lex/resolver/lex-resolver.ts @@ -0,0 +1,386 @@ +import type { CID } from "multiformats/cid"; +import { resolveTxt as resolveTxtWithNode } from "node:dns/promises"; +import { type AtprotoData, type DidCache, DidResolver } from "@atp/identity"; +import { + assertDid, + assertRecordKey, + type DidString, + type RecordKeyString, +} from "../external.ts"; +import * as l from "../external.ts"; +import { + type LexiconDocument, + lexiconDocumentSchema, +} from "../document/mod.ts"; +import { + def as repoDef, + MemoryBlockstore, + MST, + readCarWithRoot, + verifyCommitSig, +} from "@atp/repo"; +import { AtUri, ensureValidDid, NSID } from "@atp/syntax"; +import { XrpcClient } from "@atp/xrpc"; +import { LexResolverError } from "./lex-resolver-error.ts"; + +const LEXICON_COLLECTION = "com.atproto.lexicon.schema"; + +const getRecordQuery = l.query( + "com.atproto.sync.getRecord", + l.params({ + did: l.string({ format: "did" }), + collection: l.string({ format: "nsid" }), + rkey: l.string({ format: "record-key" }), + }), + l.payload("application/vnd.ipld.car"), +); + +type MaybePromise = Promise | T; + +export type LexResolverResult = { + uri: AtUri; + cid: CID; + lexicon: LexiconDocument; +}; + +export type LexResolverFetchResult = { + cid: CID; + lexicon: LexiconDocument; +}; + +export type LexResolverHooks = { + onResolveAuthority?(data: { nsid: NSID }): MaybePromise; + onResolveAuthorityResult?( + data: { nsid: NSID; did: string }, + ): MaybePromise; + onResolveAuthorityError?( + data: { nsid: NSID; err: unknown }, + ): MaybePromise; + onFetch?(data: { uri: AtUri }): MaybePromise; + onFetchResult?(data: { + uri: AtUri; + cid: CID; + lexicon: LexiconDocument; + }): MaybePromise; + onFetchError?(data: { uri: AtUri; err: unknown }): MaybePromise; +}; + +export type TxtResolver = (domain: string) => Promise; + +type DenoResolveTxt = ( + domain: string, + recordType: "TXT", +) => Promise; + +export type LexResolverDidResolver = { + resolveAtprotoData( + did: string, + forceRefresh?: boolean, + ): Promise; +}; + +export type LexResolverOptions = { + timeout?: number; + plcUrl?: string; + didCache?: DidCache; + fetch?: typeof globalThis.fetch; + hooks?: LexResolverHooks; + didResolver?: LexResolverDidResolver; + resolveTxt?: TxtResolver; +}; + +export type LexResolverFetchOptions = { + signal?: AbortSignal; + forceRefresh?: boolean; + noCache?: boolean; +}; + +export type DefaultTxtResolverOptions = { + denoResolveDns?: DenoResolveTxt | null; + nodeResolveTxt?: TxtResolver; +}; + +export { AtUri, NSID }; +export type { CID, LexiconDocument }; + +export class LexResolver { + protected readonly didResolver: LexResolverDidResolver; + protected readonly resolveTxt: TxtResolver; + + constructor(protected readonly options: LexResolverOptions = {}) { + const { timeout = 3000, plcUrl, didCache } = options; + this.didResolver = options.didResolver ?? + new DidResolver({ timeout, plcUrl, didCache }); + this.resolveTxt = options.resolveTxt ?? defaultResolveTxt; + } + + async get( + nsidStr: NSID | string, + options?: LexResolverFetchOptions, + ): Promise { + const uri = await this.resolve(nsidStr); + return this.fetch(uri, options); + } + + async resolve(nsidStr: NSID | string): Promise { + const nsid = NSID.from(nsidStr); + + const hookedDid = await this.options.hooks?.onResolveAuthority?.({ nsid }); + if (hookedDid !== undefined) { + ensureValidDid(hookedDid); + return AtUri.make(hookedDid, LEXICON_COLLECTION, nsid.toString()); + } + + const did = await this.resolveLexiconAuthority(nsid).then( + async (resolvedDid) => { + await this.options.hooks?.onResolveAuthorityResult?.({ + nsid, + did: resolvedDid, + }); + return resolvedDid; + }, + async (err) => { + await this.options.hooks?.onResolveAuthorityError?.({ nsid, err }); + throw err; + }, + ); + + return AtUri.make(did, LEXICON_COLLECTION, nsid.toString()); + } + + async fetch( + uriStr: AtUri | string, + options?: LexResolverFetchOptions, + ): Promise { + const uri = typeof uriStr === "string" ? new AtUri(uriStr) : uriStr; + + const hookedResult = await this.options.hooks?.onFetch?.({ uri }); + if (hookedResult !== undefined) { + return { uri, ...validateLexiconResult(uri, hookedResult) }; + } + + const fetched = await this.fetchLexiconUri(uri, options).then( + async (result) => { + const validated = validateLexiconResult(uri, result); + await this.options.hooks?.onFetchResult?.({ uri, ...validated }); + return validated; + }, + async (err) => { + await this.options.hooks?.onFetchError?.({ uri, err }); + throw err; + }, + ); + + return { uri, ...fetched }; + } + + protected async resolveLexiconAuthority(nsid: NSID): Promise { + try { + const did = parseDomainTxtDid( + await this.resolveTxt(`_lexicon.${nsid.authority}`), + ); + ensureValidDid(did); + return did; + } catch (cause) { + throw new LexResolverError( + nsid, + `Failed to resolve lexicon DID authority for ${nsid}`, + { cause }, + ); + } + } + + protected async fetchLexiconUri( + uri: AtUri, + options?: LexResolverFetchOptions, + ): Promise { + const { did, nsid } = parseLexiconUri(uri); + + const atprotoData = await this.didResolver.resolveAtprotoData( + did, + options?.forceRefresh, + ).catch((cause) => { + throw new LexResolverError( + nsid, + `Failed to resolve DID document for ${did}`, + { cause }, + ); + }); + + if (!atprotoData.signingKey || !atprotoData.pds) { + throw new LexResolverError( + nsid, + `No atproto PDS service endpoint or signing key found in ${did} DID document`, + ); + } + + const client = new XrpcClient({ + service: atprotoData.pds, + fetch: this.options.fetch, + }); + const didParam = did; + const rkey = nsid.toString(); + assertDid(didParam); + assertRecordKey(rkey); + + const response = await client.call(getRecordQuery, { + params: { + did: didParam as DidString, + collection: LEXICON_COLLECTION, + rkey: rkey as RecordKeyString, + }, + headers: options?.noCache ? { "cache-control": "no-cache" } : undefined, + signal: options?.signal, + validateRequest: true, + validateResponse: false, + }).catch((cause) => { + throw new LexResolverError(nsid, `Failed to fetch Record ${uri}`, { + cause, + }); + }); + + if (!(response.data instanceof Uint8Array)) { + throw new LexResolverError( + nsid, + `Invalid record response at ${uri}`, + { cause: new TypeError("Expected CAR bytes") }, + ); + } + + return verifyRecordProof( + response.data, + did, + atprotoData.signingKey, + LEXICON_COLLECTION, + nsid.toString(), + ).catch((cause) => { + throw new LexResolverError( + nsid, + `Failed to verify Lexicon record proof at ${uri}`, + { cause }, + ); + }); + } +} + +export function createDefaultResolveTxt( + options: DefaultTxtResolverOptions = {}, +): TxtResolver { + const denoResolveDns = options.denoResolveDns === undefined + ? getDenoResolveDns() + : options.denoResolveDns; + + if (denoResolveDns) { + return (domain) => denoResolveDns(domain, "TXT"); + } + + const nodeResolveTxt = options.nodeResolveTxt ?? resolveTxtWithNode; + return (domain) => nodeResolveTxt(domain); +} + +const defaultResolveTxt = createDefaultResolveTxt(); + +function getDenoResolveDns(): DenoResolveTxt | undefined { + if (typeof Deno === "undefined") { + return undefined; + } + + return (domain, recordType) => Deno.resolveDns(domain, recordType); +} + +function parseDomainTxtDid(records: string[][]): string { + const didLines = records + .map((chunks) => chunks.join("")) + .filter((value) => value.startsWith("did=")); + + if (didLines.length === 1) { + return didLines[0].slice(4); + } + + throw didLines.length > 1 + ? new Error("Multiple DIDs found in DNS TXT records") + : new Error("No DID found in DNS TXT records"); +} + +function parseLexiconUri(uri: AtUri): { did: string; nsid: NSID } { + const nsid = NSID.from(uri.rkey); + + if (uri.collection !== LEXICON_COLLECTION) { + throw new LexResolverError( + nsid, + `URI collection is not ${LEXICON_COLLECTION}: ${uri}`, + ); + } + + try { + ensureValidDid(uri.host); + return { did: uri.host, nsid }; + } catch (cause) { + throw new LexResolverError(nsid, `URI host is not a DID ${uri}`, { cause }); + } +} + +function validateLexiconResult( + uri: AtUri, + result: LexResolverFetchResult, +): LexResolverFetchResult { + const nsid = NSID.from(uri.rkey); + const validation = lexiconDocumentSchema.safeParse(result.lexicon); + + if (!validation.success) { + throw new LexResolverError(nsid, `Invalid Lexicon document at ${uri}`, { + cause: validation.error, + }); + } + + if (validation.value.id !== uri.rkey) { + throw new LexResolverError( + nsid, + `Invalid document id "${validation.value.id}" at ${uri}`, + ); + } + + return { + cid: result.cid, + lexicon: validation.value, + }; +} + +async function verifyRecordProof( + car: Uint8Array, + did: string, + signingKey: string, + collection: string, + rkey: string, +): Promise { + const { root, blocks } = await readCarWithRoot(car); + const blockstore = new MemoryBlockstore(blocks); + + const commit = blockstore.readObj(root, repoDef.commit); + if (commit.did !== did) { + throw new Error(`Invalid repo did: ${commit.did}`); + } + + const validSig = verifyCommitSig(commit, signingKey); + if (!validSig) { + throw new Error(`Invalid signature on commit: ${root.toString()}`); + } + + const mst = MST.load(blockstore, (commit as { data: CID }).data); + const cid = await mst.get(`${collection}/${rkey}`); + if (!cid) { + throw new Error("Record not found in proof"); + } + + const record = blockstore.readRecord(cid); + if (record.$type !== collection) { + throw new Error( + `Invalid record type: expected ${collection}, got ${record.$type}`, + ); + } + + return { + cid, + lexicon: record as LexiconDocument, + }; +} diff --git a/lex/resolver/mod.ts b/lex/resolver/mod.ts new file mode 100644 index 0000000..472c577 --- /dev/null +++ b/lex/resolver/mod.ts @@ -0,0 +1,2 @@ +export * from "./lex-resolver.ts"; +export * from "./lex-resolver-error.ts"; diff --git a/lex-gen/tests/lex-builder_test.ts b/lex/tests/lex-builder_test.ts similarity index 96% rename from lex-gen/tests/lex-builder_test.ts rename to lex/tests/lex-builder_test.ts index c562bc0..0fe002f 100644 --- a/lex-gen/tests/lex-builder_test.ts +++ b/lex/tests/lex-builder_test.ts @@ -1,6 +1,6 @@ import { assertRejects, assertStringIncludes } from "@std/assert"; import { join } from "node:path"; -import { LexBuilder } from "../builder/lex-builder.ts"; +import { LexBuilder } from "../build/lex-builder.ts"; Deno.test({ name: "save writes files under output directory and rejects existing files", diff --git a/lex/tests/lex-installer_test.ts b/lex/tests/lex-installer_test.ts new file mode 100644 index 0000000..12b8079 --- /dev/null +++ b/lex/tests/lex-installer_test.ts @@ -0,0 +1,308 @@ +import { CID } from "multiformats/cid"; +import { assertEquals, assertRejects } from "@std/assert"; +import { join } from "node:path"; +import { AtUri, NSID } from "@atp/syntax"; +import { cidForLex } from "../cbor/mod.ts"; +import { + install, + type LexiconsManifest, + LexInstaller, + LexInstallerError, + type LexInstallerResolver, + normalizeLexiconsManifest, +} from "../installer/mod.ts"; +import { + type LexiconDocument, + lexiconDocumentSchema, +} from "../document/mod.ts"; +import type { LexValue } from "../data/lex.ts"; + +const COLLECTION = "com.atproto.lexicon.schema"; + +class StubResolver implements LexInstallerResolver { + readonly resolved: string[] = []; + readonly fetched: string[] = []; + + constructor( + private readonly documents: Record, + private readonly authority = "did:plc:test", + ) {} + + resolve(nsidStr: NSID | string): Promise { + const nsid = NSID.from(nsidStr).toString(); + this.resolved.push(nsid); + return Promise.resolve(AtUri.make(this.authority, COLLECTION, nsid)); + } + + async fetch(uriStr: AtUri | string) { + const uri = typeof uriStr === "string" ? new AtUri(uriStr) : uriStr; + const lexicon = this.documents[uri.rkey]; + if (!lexicon) { + throw new Error(`Unknown lexicon ${uri.rkey}`); + } + this.fetched.push(uri.toString()); + const cid = CID.parse((await cidForDocument(lexicon)).toString()); + return { uri, cid, lexicon }; + } +} + +function createLexicon( + id: string, + dependency?: string, +): LexiconDocument { + return lexiconDocumentSchema.parse({ + lexicon: 1, + id, + defs: { + main: dependency + ? { + type: "query", + output: { + encoding: "application/json", + schema: { + type: "ref" as const, + ref: dependency, + }, + }, + } + : { + type: "object" as const, + properties: { + ok: { type: "boolean" }, + }, + }, + }, + }); +} + +function cidForDocument(lexicon: LexiconDocument) { + return cidForLex(lexicon as unknown as LexValue); +} + +Deno.test("installer reuses local lexicons and fetches missing dependencies", async () => { + const root = await Deno.makeTempDir({ prefix: "lex-installer-" }); + + try { + const lexicons = join(root, "lexicons"); + const manifestPath = join(root, "lexicons.json"); + const rootLexicon = createLexicon("com.example.root", "com.example.dep"); + const depLexicon = createLexicon("com.example.dep"); + const resolver = new StubResolver({ + [rootLexicon.id]: rootLexicon, + [depLexicon.id]: depLexicon, + }); + + await Deno.mkdir(join(lexicons, "com", "example"), { recursive: true }); + await Deno.writeTextFile( + join(lexicons, "com", "example", "root.json"), + JSON.stringify(rootLexicon), + ); + + const installer = new LexInstaller({ + lexicons, + manifest: manifestPath, + resolver, + }); + + try { + await installer.install({ + additions: [rootLexicon.id], + }); + await installer.save(); + } finally { + await installer[Symbol.asyncDispose](); + } + + assertEquals(resolver.resolved, [rootLexicon.id, depLexicon.id]); + assertEquals(resolver.fetched, [ + `at://did:plc:test/${COLLECTION}/${depLexicon.id}`, + ]); + + const savedDep = JSON.parse( + await Deno.readTextFile( + join(lexicons, "com", "example", "dep.json"), + ), + ); + assertEquals(savedDep.id, depLexicon.id); + + const manifest = JSON.parse( + await Deno.readTextFile(manifestPath), + ) as LexiconsManifest; + assertEquals( + manifest, + normalizeLexiconsManifest({ + version: 1, + lexicons: [rootLexicon.id], + resolutions: { + [rootLexicon.id]: { + uri: `at://did:plc:test/${COLLECTION}/${rootLexicon.id}`, + cid: (await cidForDocument(rootLexicon)).toString(), + }, + [depLexicon.id]: { + uri: `at://did:plc:test/${COLLECTION}/${depLexicon.id}`, + cid: (await cidForDocument(depLexicon)).toString(), + }, + }, + }), + ); + } finally { + await Deno.remove(root, { recursive: true }); + } +}); + +Deno.test("install supports explicit at:// additions without resolving", async () => { + const root = await Deno.makeTempDir({ prefix: "lex-installer-uri-" }); + + try { + const lexicons = join(root, "lexicons"); + const manifestPath = join(root, "lexicons.json"); + const lexicon = createLexicon("com.example.uri"); + const resolver = new StubResolver({ + [lexicon.id]: lexicon, + }, "did:plc:uri"); + + await install({ + lexicons, + manifest: manifestPath, + resolver, + add: [`at://did:plc:uri/${COLLECTION}/${lexicon.id}`], + save: true, + }); + + assertEquals(resolver.resolved, []); + assertEquals(resolver.fetched, [ + `at://did:plc:uri/${COLLECTION}/${lexicon.id}`, + ]); + } finally { + await Deno.remove(root, { recursive: true }); + } +}); + +Deno.test("install fails in ci mode when manifest is stale", async () => { + const root = await Deno.makeTempDir({ prefix: "lex-installer-ci-" }); + + try { + const lexicons = join(root, "lexicons"); + const manifestPath = join(root, "lexicons.json"); + const lexicon = createLexicon("com.example.ci"); + const resolver = new StubResolver({ + [lexicon.id]: lexicon, + }); + const staleManifest = { + version: 1 as const, + lexicons: [], + resolutions: {}, + }; + + await Deno.writeTextFile( + manifestPath, + JSON.stringify(staleManifest), + ); + + const error = await assertRejects( + () => + install({ + lexicons, + manifest: manifestPath, + resolver, + add: [lexicon.id], + ci: true, + save: false, + }), + LexInstallerError, + "Lexicons manifest is out of date", + ); + + assertEquals(error.name, "LexInstallerError"); + await assertRejects( + () => Deno.readTextFile(join(lexicons, "com", "example", "ci.json")), + Deno.errors.NotFound, + ); + assertEquals( + JSON.parse(await Deno.readTextFile(manifestPath)), + staleManifest, + ); + } finally { + await Deno.remove(root, { recursive: true }); + } +}); + +Deno.test("install rejects explicit at:// additions outside the lexicon collection", async () => { + const root = await Deno.makeTempDir({ prefix: "lex-installer-invalid-uri-" }); + + try { + const lexicons = join(root, "lexicons"); + const manifestPath = join(root, "lexicons.json"); + const lexicon = createLexicon("com.example.root"); + const resolver = new StubResolver({ + [lexicon.id]: lexicon, + }, "did:plc:uri"); + + await Deno.mkdir(join(lexicons, "com", "example"), { recursive: true }); + await Deno.writeTextFile( + join(lexicons, "com", "example", "root.json"), + JSON.stringify(lexicon), + ); + + const error = await assertRejects( + () => + install({ + lexicons, + manifest: manifestPath, + resolver, + add: [`at://did:plc:uri/app.bsky.feed.post/${lexicon.id}`], + save: true, + }), + LexInstallerError, + "Invalid lexicon URI collection", + ); + + assertEquals(error.name, "LexInstallerError"); + assertEquals(resolver.resolved, []); + assertEquals(resolver.fetched, []); + await assertRejects(() => Deno.stat(manifestPath), Deno.errors.NotFound); + } finally { + await Deno.remove(root, { recursive: true }); + } +}); + +Deno.test("install rejects explicit at:// additions with handle authorities", async () => { + const root = await Deno.makeTempDir({ + prefix: "lex-installer-handle-uri-", + }); + + try { + const lexicons = join(root, "lexicons"); + const manifestPath = join(root, "lexicons.json"); + const lexicon = createLexicon("com.example.root"); + const resolver = new StubResolver({ + [lexicon.id]: lexicon, + }, "did:plc:uri"); + + await Deno.mkdir(join(lexicons, "com", "example"), { recursive: true }); + await Deno.writeTextFile( + join(lexicons, "com", "example", "root.json"), + JSON.stringify(lexicon), + ); + + const error = await assertRejects( + () => + install({ + lexicons, + manifest: manifestPath, + resolver, + add: [`at://example.com/${COLLECTION}/${lexicon.id}`], + save: true, + }), + LexInstallerError, + "Invalid lexicon URI authority", + ); + + assertEquals(error.name, "LexInstallerError"); + assertEquals(resolver.resolved, []); + assertEquals(resolver.fetched, []); + await assertRejects(() => Deno.stat(manifestPath), Deno.errors.NotFound); + } finally { + await Deno.remove(root, { recursive: true }); + } +}); diff --git a/lex/tests/lex-resolver_test.ts b/lex/tests/lex-resolver_test.ts new file mode 100644 index 0000000..c6668a6 --- /dev/null +++ b/lex/tests/lex-resolver_test.ts @@ -0,0 +1,321 @@ +import { cidForCbor, streamToBuffer } from "@atp/common"; +import * as crypto from "@atp/crypto"; +import { getRecords, MemoryBlockstore, Repo, WriteOpAction } from "@atp/repo"; +import { AtUri } from "@atp/syntax"; +import { assertEquals, assertInstanceOf, assertRejects } from "@std/assert"; +import { LexResolver, LexResolverError } from "../resolver/mod.ts"; +import { createDefaultResolveTxt } from "../resolver/lex-resolver.ts"; + +const collection = "com.atproto.lexicon.schema"; +const nsid = "app.bsky.feed.post"; + +type ProofFixture = { + car: Uint8Array; + cid: string; + did: string; + pds: string; + signingKey: string; + lexicon: Record; +}; + +const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer => { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +}; + +const createLexiconRecord = ( + id: string, +): Record => ({ + $type: collection, + lexicon: 1, + id, + defs: { + main: { + type: "query", + }, + }, +}); + +const createProofFixture = async ( + record = createLexiconRecord(nsid), +): Promise => { + const storage = new MemoryBlockstore(); + const keypair = crypto.Secp256k1Keypair.create(); + const did = "did:plc:resolvertest"; + const repo = await Repo.create(storage, did, keypair, [{ + action: WriteOpAction.Create, + collection, + rkey: nsid, + record, + }]); + const car = await streamToBuffer(getRecords(storage, repo.cid, [{ + collection, + rkey: nsid, + }])); + const fetched = await repo.data.get(`${collection}/${nsid}`); + if (!fetched) { + throw new Error("expected cid"); + } + return { + car, + cid: fetched.toString(), + did, + pds: "https://pds.test", + signingKey: keypair.did(), + lexicon: record, + }; +}; + +Deno.test("get resolves and fetches lexicons through xrpc", async () => { + const fixture = await createProofFixture(); + const calls: { forceRefresh?: boolean; url?: string } = {}; + const events: string[] = []; + + const resolver = new LexResolver({ + didResolver: { + resolveAtprotoData(did, forceRefresh) { + assertEquals(did, fixture.did); + calls.forceRefresh = forceRefresh; + return Promise.resolve({ + did, + handle: "resolver.test", + pds: fixture.pds, + signingKey: fixture.signingKey, + }); + }, + }, + resolveTxt(domain) { + assertEquals(domain, "_lexicon.feed.bsky.app"); + return Promise.resolve([[`did=${fixture.did}`]]); + }, + fetch: ((input, init) => { + const url = input instanceof URL ? input : new URL(String(input)); + const headers = new Headers(init?.headers); + calls.url = url.toString(); + assertEquals(init?.method, "get"); + assertEquals(headers.get("accept"), "application/vnd.ipld.car"); + return Promise.resolve( + new Response(toArrayBuffer(fixture.car), { + headers: { "content-type": "application/vnd.ipld.car" }, + }), + ); + }) as typeof fetch, + hooks: { + onResolveAuthorityResult() { + events.push("resolve"); + }, + onFetchResult() { + events.push("fetch"); + }, + }, + }); + + const result = await resolver.get(nsid, { + forceRefresh: true, + noCache: true, + }); + + assertEquals( + calls.url, + "https://pds.test/xrpc/com.atproto.sync.getRecord?" + + "did=did%3Aplc%3Aresolvertest&collection=com.atproto.lexicon.schema" + + "&rkey=app.bsky.feed.post", + ); + assertEquals(calls.forceRefresh, true); + assertEquals( + result.uri.toString(), + `at://${fixture.did}/${collection}/${nsid}`, + ); + assertEquals(result.cid.toString(), fixture.cid); + assertEquals(result.lexicon.id, nsid); + assertEquals(events, ["resolve", "fetch"]); +}); + +Deno.test("resolve and fetch hooks can short-circuit the network path", async () => { + const uri = AtUri.make("did:plc:hooked", collection, nsid); + const cid = await cidForCbor({ ok: true }); + const lexicon = createLexiconRecord(nsid); + + const resolver = new LexResolver({ + didResolver: { + resolveAtprotoData() { + throw new Error("did resolver should not be called"); + }, + }, + resolveTxt() { + throw new Error("dns should not be called"); + }, + hooks: { + onResolveAuthority({ nsid }) { + assertEquals(nsid.toString(), "app.bsky.feed.post"); + return "did:plc:hooked"; + }, + onFetch({ uri }) { + assertEquals( + uri.toString(), + `at://did:plc:hooked/${collection}/${nsid}`, + ); + return { + cid, + lexicon: lexicon as never, + }; + }, + }, + }); + + const resolved = await resolver.resolve(nsid); + const fetched = await resolver.fetch(uri); + + assertEquals(resolved.toString(), uri.toString()); + assertEquals(fetched.cid.toString(), cid.toString()); + assertEquals(fetched.lexicon.id, nsid); +}); + +Deno.test("resolve wraps dns failures in LexResolverError", async () => { + const seen: unknown[] = []; + + const resolver = new LexResolver({ + resolveTxt() { + return Promise.resolve([["v=spf1 -all"]]); + }, + hooks: { + onResolveAuthorityError({ err }) { + seen.push(err); + }, + }, + }); + + const error = await assertRejects( + () => resolver.resolve(nsid), + LexResolverError, + ); + + assertEquals(error.nsid.toString(), nsid); + assertEquals(seen.length, 1); +}); + +Deno.test("createDefaultResolveTxt prefers Deno DNS when available", async () => { + const calls: Array<[string, "TXT"]> = []; + const resolveTxt = createDefaultResolveTxt({ + denoResolveDns(domain, recordType) { + calls.push([domain, recordType]); + return Promise.resolve([["did=did:plc:deno"]]); + }, + nodeResolveTxt() { + throw new Error("node dns should not be called"); + }, + }); + + const records = await resolveTxt("_lexicon.feed.bsky.app"); + + assertEquals(calls, [["_lexicon.feed.bsky.app", "TXT"]]); + assertEquals(records, [["did=did:plc:deno"]]); +}); + +Deno.test("createDefaultResolveTxt falls back to Node DNS when Deno DNS is unavailable", async () => { + const calls: string[] = []; + const resolveTxt = createDefaultResolveTxt({ + denoResolveDns: null, + nodeResolveTxt(domain) { + calls.push(domain); + return Promise.resolve([["did=did:plc:node"]]); + }, + }); + + const records = await resolveTxt("_lexicon.feed.bsky.app"); + + assertEquals(calls, ["_lexicon.feed.bsky.app"]); + assertEquals(records, [["did=did:plc:node"]]); +}); + +Deno.test("fetch wraps proof verification failures in LexResolverError", async () => { + const fixture = await createProofFixture(); + const seen: unknown[] = []; + + const resolver = new LexResolver({ + didResolver: { + resolveAtprotoData(did) { + return Promise.resolve({ + did, + handle: "resolver.test", + pds: fixture.pds, + signingKey: "did:key:zWrongKey", + }); + }, + }, + fetch: (() => { + return Promise.resolve( + new Response(toArrayBuffer(fixture.car), { + headers: { "content-type": "application/vnd.ipld.car" }, + }), + ); + }) as typeof fetch, + hooks: { + onFetchError({ err }) { + seen.push(err); + }, + }, + }); + + const error = await assertRejects( + () => resolver.fetch(`at://${fixture.did}/${collection}/${nsid}`), + LexResolverError, + ); + + assertEquals(error.nsid.toString(), nsid); + assertEquals(seen.length, 1); +}); + +Deno.test("fetch rejects lexicons with mismatched ids", async () => { + const fixture = await createProofFixture( + createLexiconRecord("app.bsky.feed.like"), + ); + + const resolver = new LexResolver({ + didResolver: { + resolveAtprotoData(did) { + return Promise.resolve({ + did, + handle: "resolver.test", + pds: fixture.pds, + signingKey: fixture.signingKey, + }); + }, + }, + fetch: (() => { + return Promise.resolve( + new Response(toArrayBuffer(fixture.car), { + headers: { "content-type": "application/vnd.ipld.car" }, + }), + ); + }) as typeof fetch, + }); + + const error = await assertRejects( + () => resolver.fetch(`at://${fixture.did}/${collection}/${nsid}`), + LexResolverError, + ); + + assertEquals(error.nsid.toString(), nsid); +}); + +Deno.test("fetch rejects non-lexicon collections", async () => { + const resolver = new LexResolver({ + didResolver: { + resolveAtprotoData() { + throw new Error("did resolver should not be called"); + }, + }, + }); + + const error = await assertRejects( + () => + resolver.fetch( + "at://did:plc:resolvertest/app.bsky.feed.post/app.bsky.feed.like", + ), + LexResolverError, + ); + + assertInstanceOf(error, LexResolverError); +}); diff --git a/lex-gen/tests/method-generation_test.ts b/lex/tests/method-generation_test.ts similarity index 98% rename from lex-gen/tests/method-generation_test.ts rename to lex/tests/method-generation_test.ts index fb78a60..99692ee 100644 --- a/lex-gen/tests/method-generation_test.ts +++ b/lex/tests/method-generation_test.ts @@ -5,7 +5,7 @@ import { lexiconDocumentSchema, type LexiconIndexer, } from "@atp/lex/document"; -import { LexDefBuilder } from "../builder/def-builder.ts"; +import { LexDefBuilder } from "../build/def-builder.ts"; class DummyIndexer implements LexiconIndexer, AsyncIterable { readonly #docs: Map; -- 2.51.2