diff --git a/ingester/package.json b/ingester/package.json index b5d6387..e4d6beb 100644 --- a/ingester/package.json +++ b/ingester/package.json @@ -15,7 +15,8 @@ "drizzle-orm": "^0.45.1", "pg": "^8.16.3", "tsx": "^4.19.0", - "ws": "^8.18.3" + "ws": "^8.18.3", + "@atproto/lexicon": "^0.5.0" }, "devDependencies": { "@types/node": "^22.14.1", diff --git a/ingester/src/lexicons.ts b/ingester/src/lexicons.ts index c47f029..21c83bd 100644 --- a/ingester/src/lexicons.ts +++ b/ingester/src/lexicons.ts @@ -1,38 +1,41 @@ -import { jsonToLex } from "@atproto/lex-json"; -import { main as trail } from "../../lib/lexicons/app/sidetrail/trail.defs"; -import { main as walk } from "../../lib/lexicons/app/sidetrail/walk.defs"; -import { main as completion } from "../../lib/lexicons/app/sidetrail/completion.defs"; +import { readFileSync } from "node:fs"; +import { Lexicons, jsonToLex, type LexiconDoc } from "@atproto/lexicon"; -// Runtime lexicon schemas for every collection we index. Records that fail -// validation are rejected: PDSes don't reliably enforce third-party lexicons, -// so validating is the indexer's responsibility. -export const RECORD_SCHEMAS = { - "app.sidetrail.trail": trail, - "app.sidetrail.walk": walk, - "app.sidetrail.completion": completion, -} as const; +// Validate against the lexicon JSON documents directly (the same source the +// generated lib/lexicons code is built from). PDSes don't reliably enforce +// third-party lexicons, so validating is the indexer's responsibility. +const loadDoc = (path: string): LexiconDoc => + JSON.parse(readFileSync(new URL(`../../lexicons/${path}`, import.meta.url), "utf8")); -export type IndexedCollection = keyof typeof RECORD_SCHEMAS; +const lexicons = new Lexicons([ + loadDoc("app/sidetrail/trail.json"), + loadDoc("app/sidetrail/walk.json"), + loadDoc("app/sidetrail/completion.json"), + loadDoc("com/atproto/repo/strongRef.json"), +]); + +export const INDEXED_COLLECTIONS = [ + "app.sidetrail.trail", + "app.sidetrail.walk", + "app.sidetrail.completion", +] as const; + +export type IndexedCollection = (typeof INDEXED_COLLECTIONS)[number]; export function validateRecord( collection: IndexedCollection, record: unknown, ): { success: true } | { success: false; reason: string } { // Records arrive as JSON (jetstream, listRecords, jsonb storage), where CID - // links are {"$link": ...} objects. Schemas validate lex values, so convert - // first; a record that can't even be converted (e.g. malformed CID) is invalid. + // links are {"$link": ...} objects. The validator expects lex values, so + // convert first; a record that can't be converted (e.g. malformed CID) is invalid. let lexValue: unknown; try { - lexValue = jsonToLex(record as Parameters[0]); + lexValue = jsonToLex(record); } catch (err) { return { success: false, reason: `unparseable as lex: ${(err as Error).message}` }; } - const result = RECORD_SCHEMAS[collection].validate(lexValue); + const result = lexicons.validate(collection, lexValue); if (result.success) return { success: true }; - return { - success: false, - reason: result.error.issues - .map((issue) => `${issue.code} at ${issue.path.join(".") || "(root)"}`) - .join("; "), - }; + return { success: false, reason: result.error.message }; } diff --git a/package-lock.json b/package-lock.json index d2e247c..44dbc03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -75,6 +75,7 @@ "name": "sidetrail-ingester", "version": "0.1.0", "dependencies": { + "@atproto/lexicon": "^0.5.0", "@sidetrail/db": "*", "dotenv": "^17.2.3", "drizzle-orm": "^0.45.1", diff --git a/package.json b/package.json index 6e89715..ddd03e4 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,10 @@ "db:studio": "drizzle-kit studio", "test": "npx vitest run", "test:watch": "npx vitest", - "deploy:app": "railway link --service sidetrail && railway up", - "deploy:ingester": "railway link --service ingester && railway up", - "deploy:realtime": "railway link --service realtime && railway up", - "deploy:all": "npm run deploy:app & npm run deploy:ingester & npm run deploy:realtime & wait" + "deploy:app": "railway up --service sidetrail --ci", + "deploy:ingester": "railway up --service ingester --ci", + "deploy:realtime": "railway up --service realtime --ci", + "deploy:all": "npm run deploy:app && npm run deploy:ingester && npm run deploy:realtime" }, "dependencies": { "@atproto/api": "^0.17.4",