From 73e1681cf80700e001a5561da08bf44b856ec623 Mon Sep 17 00:00:00 2001 From: Jacob Zweifel Date: Wed, 12 Aug 2026 18:16:53 -0400 Subject: [PATCH] Add lexicon publication and live-record-check tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packages/lexicons gains two node scripts. publish-lexicons turns each document under lexicons/ into a com.atproto.lexicon.schema record (rkey = NSID, the record being the document plus $type) and puts it in the authority repo — createSession with app-password env vars, putRecord per document, so republishing after an add-optional change is one command; --dry-run prints the records without touching the network. check-live-records resolves a handle or DID to its PDS, lists every page.mooring.* record in that repo, and validates each against the drafts — run before publication or a draft edit, since resolving PDSes enforce published lexicons on write. The label-defs lexicon both record types reference is vendored (same convention as apps/web), and the first live record's shape is pinned as a test fixture so a draft edit that would strand it fails in CI. @types/node was declared in tsconfig but never installed; now it is. Co-Authored-By: Claude Fable 5 --- package-lock.json | 18 +++ packages/lexicons/package.json | 5 +- .../lexicons/scripts/check-live-records.ts | 99 ++++++++++++ packages/lexicons/scripts/publish-lexicons.ts | 66 ++++++++ packages/lexicons/scripts/xrpc.ts | 39 +++++ packages/lexicons/src/index.ts | 14 ++ packages/lexicons/src/schema-records.ts | 25 +++ .../lexicons/tests/schema-records.test.ts | 61 +++++++ packages/lexicons/tsconfig.json | 3 +- packages/lexicons/vendor/README.md | 12 ++ .../vendor/com.atproto.label.defs.json | 153 ++++++++++++++++++ 11 files changed, 493 insertions(+), 2 deletions(-) create mode 100644 packages/lexicons/scripts/check-live-records.ts create mode 100644 packages/lexicons/scripts/publish-lexicons.ts create mode 100644 packages/lexicons/scripts/xrpc.ts create mode 100644 packages/lexicons/src/schema-records.ts create mode 100644 packages/lexicons/tests/schema-records.test.ts create mode 100644 packages/lexicons/vendor/README.md create mode 100644 packages/lexicons/vendor/com.atproto.label.defs.json diff --git a/package-lock.json b/package-lock.json index f9deea1..35a1af2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2003,6 +2003,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "26.2.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", + "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -3325,6 +3335,13 @@ "node": ">=22.19.0" } }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + }, "node_modules/unenv": { "version": "2.0.0-rc.24", "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", @@ -3705,6 +3722,7 @@ "@atproto/lexicon": "^0.7.10" }, "devDependencies": { + "@types/node": "^26.2.0", "typescript": "^6.0.3", "vitest": "^4.1.10" } diff --git a/packages/lexicons/package.json b/packages/lexicons/package.json index 36a0b4c..0972b3f 100644 --- a/packages/lexicons/package.json +++ b/packages/lexicons/package.json @@ -6,12 +6,15 @@ "description": "Loader and convention tests for the page.mooring.* lexicons (source of truth: /lexicons at the repo root).", "main": "src/index.ts", "scripts": { - "test": "vitest run" + "test": "vitest run", + "check-live-records": "node scripts/check-live-records.ts", + "publish-lexicons": "node scripts/publish-lexicons.ts" }, "dependencies": { "@atproto/lexicon": "^0.7.10" }, "devDependencies": { + "@types/node": "^26.2.0", "typescript": "^6.0.3", "vitest": "^4.1.10" } diff --git a/packages/lexicons/scripts/check-live-records.ts b/packages/lexicons/scripts/check-live-records.ts new file mode 100644 index 0000000..8a2bae6 --- /dev/null +++ b/packages/lexicons/scripts/check-live-records.ts @@ -0,0 +1,99 @@ +/** + * Validate the page.mooring.* records that exist in a live repo against the + * draft lexicon documents in this tree. Once the lexicons are published + * on-network, PDSes that resolve them start enforcing them — this check + * catches a live record the drafts would reject, before publication (or a + * draft edit) makes that record invalid. + * + * Usage: + * npm run check-live-records -- + * + * Exits 1 when any record fails validation. + */ + +import { Lexicons } from '@atproto/lexicon'; +import { loadLexicons, loadVendoredLexicons } from '../src/index.ts'; +import { xrpc } from './xrpc.ts'; + +const actor = process.argv[2]; +if (!actor) { + console.error('Usage: npm run check-live-records -- '); + process.exit(1); +} + +async function resolveDid(actor: string): Promise { + if (actor.startsWith('did:')) return actor; + const result = await xrpc( + 'https://public.api.bsky.app', + 'com.atproto.identity.resolveHandle', + { params: { handle: actor } } + ); + return result.did as string; +} + +async function pdsEndpoint(did: string): Promise { + let docUrl: string; + if (did.startsWith('did:plc:')) { + docUrl = `https://plc.directory/${did}`; + } else if (did.startsWith('did:web:')) { + const parts = did.slice('did:web:'.length).split(':').map(decodeURIComponent); + const host = parts[0]; + const path = parts.length > 1 ? `/${parts.slice(1).join('/')}/did.json` : '/.well-known/did.json'; + docUrl = `https://${host}${path}`; + } else { + throw new Error(`unsupported DID method: ${did}`); + } + const response = await fetch(docUrl); + if (!response.ok) throw new Error(`failed to resolve ${did}: HTTP ${response.status}`); + const doc = (await response.json()) as { + service?: { id: string; type: string; serviceEndpoint: string }[]; + }; + const pds = doc.service?.find( + (s) => s.type === 'AtprotoPersonalDataServer' || s.id.endsWith('#atproto_pds') + ); + if (!pds) throw new Error(`no PDS service in the DID document for ${did}`); + return pds.serviceEndpoint; +} + +const loaded = loadLexicons(); +const lexicons = new Lexicons([...loaded.map((l) => l.doc), ...loadVendoredLexicons()]); +const recordNsids = loaded + .filter(({ doc }) => doc.defs.main?.type === 'record') + .map(({ doc }) => doc.id); + +const did = await resolveDid(actor); +const pds = await pdsEndpoint(did); +console.log(`repo ${did} on ${pds}`); + +let checked = 0; +let failed = 0; + +for (const nsid of recordNsids) { + let cursor: string | undefined; + do { + const page = await xrpc(pds, 'com.atproto.repo.listRecords', { + params: { + repo: did, + collection: nsid, + limit: '100', + ...(cursor ? { cursor } : {}) + } + }); + const records = page.records as { uri: string; value: unknown }[]; + for (const record of records) { + checked += 1; + const result = lexicons.validate(nsid, record.value); + if (result.success) { + console.log(` ok ${record.uri}`); + } else { + failed += 1; + console.log(` INVALID ${record.uri}`); + console.log(` ${result.error.message}`); + } + } + cursor = typeof page.cursor === 'string' && records.length > 0 ? page.cursor : undefined; + } while (cursor); +} + +console.log(`${checked} record(s) checked, ${failed} invalid`); +process.exit(failed > 0 ? 1 : 0); diff --git a/packages/lexicons/scripts/publish-lexicons.ts b/packages/lexicons/scripts/publish-lexicons.ts new file mode 100644 index 0000000..a9b0129 --- /dev/null +++ b/packages/lexicons/scripts/publish-lexicons.ts @@ -0,0 +1,66 @@ +/** + * Publish every lexicon document under the repo-root lexicons/ directory as a + * com.atproto.lexicon.schema record (rkey = NSID) in the authority account's + * repo. putRecord overwrites, so re-running the script republishes after a + * schema change. + * + * Usage: + * npm run publish-lexicons -- [--dry-run] + * + * Environment (not needed with --dry-run): + * LEXICON_AUTHORITY_IDENTIFIER handle of the authority account + * LEXICON_AUTHORITY_APP_PASSWORD an app password for that account + * LEXICON_AUTHORITY_PDS PDS origin, default https://bsky.social + * + * Consumers resolve the records through the _lexicon.mooring.page DNS TXT + * record, which must name the authority account's DID: "did=did:...". + */ + +import { loadLexicons } from '../src/index.ts'; +import { toSchemaWrite } from '../src/schema-records.ts'; +import { xrpc } from './xrpc.ts'; + +const dryRun = process.argv.includes('--dry-run'); +const writes = loadLexicons().map(({ doc }) => toSchemaWrite(doc)); + +if (dryRun) { + for (const write of writes) { + console.log(`--- ${write.collection}/${write.rkey}`); + console.log(JSON.stringify(write.record, null, 2)); + } + process.exit(0); +} + +const identifier = process.env.LEXICON_AUTHORITY_IDENTIFIER; +const appPassword = process.env.LEXICON_AUTHORITY_APP_PASSWORD; +const pds = process.env.LEXICON_AUTHORITY_PDS ?? 'https://bsky.social'; + +if (!identifier || !appPassword) { + console.error( + 'LEXICON_AUTHORITY_IDENTIFIER and LEXICON_AUTHORITY_APP_PASSWORD must be set (or pass --dry-run).' + ); + process.exit(1); +} + +const session = await xrpc(pds, 'com.atproto.server.createSession', { + body: { identifier, password: appPassword } +}); +const did = session.did as string; +const accessJwt = session.accessJwt as string; + +for (const write of writes) { + const result = await xrpc(pds, 'com.atproto.repo.putRecord', { + accessJwt, + body: { + repo: did, + collection: write.collection, + rkey: write.rkey, + record: write.record + } + }); + console.log(`published ${result.uri} (${result.cid})`); +} + +console.log( + `\nDone. Lexicon resolution needs the DNS record:\n _lexicon.mooring.page TXT "did=${did}"` +); diff --git a/packages/lexicons/scripts/xrpc.ts b/packages/lexicons/scripts/xrpc.ts new file mode 100644 index 0000000..98ab732 --- /dev/null +++ b/packages/lexicons/scripts/xrpc.ts @@ -0,0 +1,39 @@ +/** Minimal XRPC-over-fetch helpers shared by the scripts in this directory. */ + +export class XrpcError extends Error { + readonly status: number; + + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +/** GET or POST a JSON XRPC endpoint; throws XrpcError on a non-2xx response. */ +export async function xrpc( + service: string, + nsid: string, + options: { + params?: Record; + body?: unknown; + accessJwt?: string; + } = {} +): Promise> { + const url = new URL(`/xrpc/${nsid}`, service); + for (const [key, value] of Object.entries(options.params ?? {})) { + url.searchParams.set(key, value); + } + const headers: Record = {}; + if (options.accessJwt) headers.authorization = `Bearer ${options.accessJwt}`; + if (options.body !== undefined) headers['content-type'] = 'application/json'; + const response = await fetch(url, { + method: options.body === undefined ? 'GET' : 'POST', + headers, + body: options.body === undefined ? undefined : JSON.stringify(options.body) + }); + const text = await response.text(); + if (!response.ok) { + throw new XrpcError(response.status, `${nsid} failed (${response.status}): ${text}`); + } + return text ? (JSON.parse(text) as Record) : {}; +} diff --git a/packages/lexicons/src/index.ts b/packages/lexicons/src/index.ts index 22e7299..d8aa33f 100644 --- a/packages/lexicons/src/index.ts +++ b/packages/lexicons/src/index.ts @@ -6,6 +6,9 @@ import { parseLexiconDoc, type LexiconDoc } from '@atproto/lexicon'; /** Absolute path of the repo-root lexicons/ directory (the source of truth). */ export const lexiconsDir = fileURLToPath(new URL('../../../lexicons', import.meta.url)); +/** Absolute path of the vendored third-party lexicons (see vendor/README.md). */ +export const vendorDir = fileURLToPath(new URL('../vendor', import.meta.url)); + export interface LoadedLexicon { /** Path relative to lexicons/, e.g. page/mooring/site.json */ relPath: string; @@ -33,3 +36,14 @@ export function loadLexicons(): LoadedLexicon[] { return { relPath, pathNsid, doc }; }); } + +/** + * Load and parse the vendored third-party lexicon documents. Our documents + * reference these (e.g. com.atproto.label.defs#selfLabels), so record + * validation needs them in the Lexicons instance alongside the drafts. + */ +export function loadVendoredLexicons(): LexiconDoc[] { + return walk(vendorDir) + .sort() + .map((full) => parseLexiconDoc(JSON.parse(readFileSync(full, 'utf8')))); +} diff --git a/packages/lexicons/src/schema-records.ts b/packages/lexicons/src/schema-records.ts new file mode 100644 index 0000000..7b106a1 --- /dev/null +++ b/packages/lexicons/src/schema-records.ts @@ -0,0 +1,25 @@ +/** + * Pure builders for publishing lexicon documents on-network as + * com.atproto.lexicon.schema records (rkey = the document's NSID). + */ + +import type { LexiconDoc } from '@atproto/lexicon'; + +export const LEXICON_SCHEMA_COLLECTION = 'com.atproto.lexicon.schema'; + +export interface LexiconSchemaWrite { + collection: typeof LEXICON_SCHEMA_COLLECTION; + /** The record key: the NSID of the lexicon document. */ + rkey: string; + /** The record value: the lexicon document itself, plus $type. */ + record: Record; +} + +/** Build the putRecord write that publishes a lexicon document. */ +export function toSchemaWrite(doc: LexiconDoc): LexiconSchemaWrite { + return { + collection: LEXICON_SCHEMA_COLLECTION, + rkey: doc.id, + record: { $type: LEXICON_SCHEMA_COLLECTION, ...doc } + }; +} diff --git a/packages/lexicons/tests/schema-records.test.ts b/packages/lexicons/tests/schema-records.test.ts new file mode 100644 index 0000000..baa9d5d --- /dev/null +++ b/packages/lexicons/tests/schema-records.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; +import { Lexicons } from '@atproto/lexicon'; +import { loadLexicons, loadVendoredLexicons } from '../src/index'; +import { LEXICON_SCHEMA_COLLECTION, toSchemaWrite } from '../src/schema-records'; + +const loaded = loadLexicons(); + +describe('toSchemaWrite', () => { + it.each(loaded)('$relPath: rkey is the NSID', ({ doc }) => { + const write = toSchemaWrite(doc); + expect(write.collection).toBe(LEXICON_SCHEMA_COLLECTION); + expect(write.rkey).toBe(doc.id); + }); + + it.each(loaded)('$relPath: the record is the document plus $type', ({ doc }) => { + const write = toSchemaWrite(doc); + expect(write.record).toEqual({ $type: LEXICON_SCHEMA_COLLECTION, ...doc }); + expect(write.record.$type).toBe(LEXICON_SCHEMA_COLLECTION); + expect(write.record.lexicon).toBe(1); + expect(write.record.id).toBe(doc.id); + }); + + it('does not mutate the document', () => { + const doc = loaded[0].doc; + const before = JSON.stringify(doc); + toSchemaWrite(doc); + expect(JSON.stringify(doc)).toBe(before); + }); +}); + +describe('loadVendoredLexicons', () => { + it('includes the label defs our records reference', () => { + expect(loadVendoredLexicons().map((doc) => doc.id)).toContain('com.atproto.label.defs'); + }); +}); + +describe('records already in the wild', () => { + const lexicons = new Lexicons([...loaded.map((l) => l.doc), ...loadVendoredLexicons()]); + + // The first real page.mooring.page record, as written to Jacob's repo + // (at://did:web:malpercio.dev/page.mooring.page/3msvyleagb6uv). Draft edits + // must keep this shape valid: it exists on the network. + const aboutPage = { + $type: 'page.mooring.page', + content: { + $type: 'page.mooring.page#markdown', + text: '# About me' + }, + createdAt: '2026-08-12T20:43:09.750Z', + description: 'About me', + path: '/about', + publishedAt: '2026-08-12T20:43:09.750Z', + title: 'About', + visibility: 'public' + }; + + it('the live /about page record validates', () => { + const result = lexicons.validate('page.mooring.page', aboutPage); + expect(result.success, result.success ? '' : result.error.message).toBe(true); + }); +}); diff --git a/packages/lexicons/tsconfig.json b/packages/lexicons/tsconfig.json index 9660de6..e7348aa 100644 --- a/packages/lexicons/tsconfig.json +++ b/packages/lexicons/tsconfig.json @@ -3,10 +3,11 @@ "target": "ES2022", "module": "ESNext", "moduleResolution": "bundler", + "allowImportingTsExtensions": true, "strict": true, "skipLibCheck": true, "types": ["node"], "noEmit": true }, - "include": ["src", "tests"] + "include": ["src", "tests", "scripts"] } diff --git a/packages/lexicons/vendor/README.md b/packages/lexicons/vendor/README.md new file mode 100644 index 0000000..c3384e3 --- /dev/null +++ b/packages/lexicons/vendor/README.md @@ -0,0 +1,12 @@ +# Vendored lexicons + +Third-party lexicon documents copied verbatim from upstream. They live here +rather than in the repo-root `lexicons/` directory, which holds the +`page.mooring.*` documents we author and is covered by the convention tests — +the vendored documents are never published by the publication script. + +| File | Upstream | +| --- | --- | +| `com.atproto.label.defs.json` | https://raw.githubusercontent.com/bluesky-social/atproto/main/lexicons/com/atproto/label/defs.json | + +To refresh a copy, download the upstream file over it and run `npm test`. diff --git a/packages/lexicons/vendor/com.atproto.label.defs.json b/packages/lexicons/vendor/com.atproto.label.defs.json new file mode 100644 index 0000000..26b6a3f --- /dev/null +++ b/packages/lexicons/vendor/com.atproto.label.defs.json @@ -0,0 +1,153 @@ +{ + "lexicon": 1, + "id": "com.atproto.label.defs", + "defs": { + "label": { + "type": "object", + "description": "Metadata tag on an atproto resource (eg, repo or record).", + "required": ["src", "uri", "val", "cts"], + "properties": { + "ver": { + "type": "integer", + "description": "The AT Protocol version of the label object." + }, + "src": { + "type": "string", + "format": "did", + "description": "DID of the actor who created this label." + }, + "uri": { + "type": "string", + "format": "uri", + "description": "AT URI of the record, repository (account), or other resource that this label applies to." + }, + "cid": { + "type": "string", + "format": "cid", + "description": "Optionally, CID specifying the specific version of 'uri' resource this label applies to." + }, + "val": { + "type": "string", + "maxLength": 128, + "description": "The short string name of the value or type of this label." + }, + "neg": { + "type": "boolean", + "description": "If true, this is a negation label, overwriting a previous label." + }, + "cts": { + "type": "string", + "format": "datetime", + "description": "Timestamp when this label was created." + }, + "exp": { + "type": "string", + "format": "datetime", + "description": "Timestamp at which this label expires (no longer applies)." + }, + "sig": { + "type": "bytes", + "description": "Signature of dag-cbor encoded label." + } + } + }, + "selfLabels": { + "type": "object", + "description": "Metadata tags on an atproto record, published by the author within the record.", + "required": ["values"], + "properties": { + "values": { + "type": "array", + "items": { "type": "ref", "ref": "#selfLabel" }, + "maxLength": 10 + } + } + }, + "selfLabel": { + "type": "object", + "description": "Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel.", + "required": ["val"], + "properties": { + "val": { + "type": "string", + "maxLength": 128, + "description": "The short string name of the value or type of this label." + } + } + }, + "labelValueDefinition": { + "type": "object", + "description": "Declares a label value and its expected interpretations and behaviors.", + "required": ["identifier", "severity", "blurs", "locales"], + "properties": { + "identifier": { + "type": "string", + "description": "The value of the label being defined. Must only include lowercase ascii and the '-' character ([a-z-]+).", + "maxLength": 100, + "maxGraphemes": 100 + }, + "severity": { + "type": "string", + "description": "How should a client visually convey this label? 'inform' means neutral and informational; 'alert' means negative and warning; 'none' means show nothing.", + "knownValues": ["inform", "alert", "none"] + }, + "blurs": { + "type": "string", + "description": "What should this label hide in the UI, if applied? 'content' hides all of the target; 'media' hides the images/video/audio; 'none' hides nothing.", + "knownValues": ["content", "media", "none"] + }, + "defaultSetting": { + "type": "string", + "description": "The default setting for this label.", + "knownValues": ["ignore", "warn", "hide"], + "default": "warn" + }, + "adultOnly": { + "type": "boolean", + "description": "Does the user need to have adult content enabled in order to configure this label?" + }, + "locales": { + "type": "array", + "items": { "type": "ref", "ref": "#labelValueDefinitionStrings" } + } + } + }, + "labelValueDefinitionStrings": { + "type": "object", + "description": "Strings which describe the label in the UI, localized into a specific language.", + "required": ["lang", "name", "description"], + "properties": { + "lang": { + "type": "string", + "description": "The code of the language these strings are written in.", + "format": "language" + }, + "name": { + "type": "string", + "description": "A short human-readable name for the label.", + "maxGraphemes": 64, + "maxLength": 640 + }, + "description": { + "type": "string", + "description": "A longer description of what the label means and why it might be applied.", + "maxGraphemes": 10000, + "maxLength": 100000 + } + } + }, + "labelValue": { + "type": "string", + "knownValues": [ + "!hide", + "!warn", + "!no-unauthenticated", + "porn", + "sexual", + "nudity", + "graphic-media", + "bot" + ] + } + } +} -- 2.51.2