diff --git a/.claude/settings.local.json b/.claude/settings.local.json index dc335a9..1a1750b 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -6,7 +6,8 @@ "Bash(npx tsx:*)", "Bash(npx tsc:*)", "Bash(pnpm test:*)", - "Bash(npx vitest:*)" + "Bash(npx vitest:*)", + "WebFetch(domain:raw.githubusercontent.com)" ] } } diff --git a/.gitignore b/.gitignore index 0d2b6de..d17a0b7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ dist # Generated (internal only) lex.config.js src/lexicon-types/ -src/core/queryable.generated.ts \ No newline at end of file +src/core/queryable.generated.ts + +stuff/ \ No newline at end of file diff --git a/README.md b/README.md index e73c872..7204759 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ You can override any auto-detected field by specifying `queryable` manually in c | `relations.*.match` | `"uri"` | Match against parent's `"uri"` or `"did"` | | `relations.*.groupBy` | — | Split counts by this field's value | | `queries` | `{}` | Custom query handlers | +| `searchable` | auto-detected | FTS5 search fields. `string[]` = explicit fields, `false` = disabled, omitted = all non-range queryable fields | ### Profiles @@ -88,6 +89,7 @@ All endpoints at `/xrpc/{nsid}.{method}`: | `{collection}.listRecords` | List/filter records | | `{collection}.getRecord` | Get single record by URI | | `{namespace}.getProfile` | Get a user's profile by DID or handle | +| `{namespace}.notifyOfUpdate` | Notify of a record change for immediate indexing | | `{namespace}.admin.sync` | Discover + backfill (requires `ADMIN_SECRET`) | | `{namespace}.admin.getCursor` | Current cursor position | | `{namespace}.admin.getOverview` | All collections summary | @@ -101,38 +103,101 @@ All endpoints at `/xrpc/{nsid}.{method}`: |-------|---------|-------------| | `actor` | `?actor=did:plc:...` or `?actor=alice.bsky.social` | Filter by DID or handle (triggers on-demand backfill) | | `profiles` | `?profiles=true` | Include profile + identity info keyed by DID | +| `search` | `?search=meetup` | Full-text search across searchable fields (FTS5, ranked) | | `{field}` | `?status=going` | Equality filter on queryable string field | | `{field}Min` | `?startsAtMin=2026-03-16` | Range minimum (datetime/integer fields) | | `{field}Max` | `?endsAtMax=2026-04-01` | Range maximum (datetime/integer fields) | | `{rel}CountMin` | `?rsvpsCountMin=10` | Minimum total relation count | | `{rel}{Group}CountMin` | `?rsvpsGoingCountMin=10` | Minimum relation count for a specific groupBy value | -| `hydrate` | `?hydrate=rsvps:10` | Embed latest N related records per record | +| `hydrate{Rel}` | `?hydrateRsvps=10` | Embed latest N related records (per group if grouped) | +| `hydrate{Ref}` | `?hydrateEvent=true` | Embed the referenced record | +| `sort` | `?sort=startsAt` | Sort by a queryable field or count (see below) | +| `order` | `?order=asc` | Sort direction: `asc` or `desc` (default depends on field type) | | `limit` | `?limit=25` | Page size (1-100, default 50) | | `cursor` | `?cursor=...` | Pagination cursor | -**Hydration** returns related records grouped by `groupBy` value: +**Sorting** — `sort` accepts any queryable field param name or a count field: ``` -?hydrate=rsvps:5 # latest 5 per group (going, interested, etc.) -?hydrate=rsvps:5&hydrate=followers:10 # multiple hydrations +?sort=startsAt # by date (default: desc for range fields) +?sort=name&order=asc # by name ascending +?sort=rsvpsCount # by total RSVP count (default: desc) +?sort=rsvpsGoingCount&order=asc # by going count ascending +``` + +**Search** uses SQLite FTS5 for ranked full-text search. By default, all non-range queryable fields are searchable. Results are ranked by relevance (BM25) with `time_us` as tiebreaker. Supports FTS5 syntax including prefix (`meetup*`), phrases (`"rust meetup"`), and boolean (`rust OR typescript`). Combinable with all other filters. + +``` +?search=meetup # basic search +?search=meetup&mode=online # search + filter +?search=rust*&sort=startsAt&order=asc # search + sort override +``` + +**Hydration** embeds related or referenced records inline: + +``` +?hydrateRsvps=5 # latest 5 RSVPs per group (going, interested, etc.) +?hydrateEvent=true # embed the referenced event record +?hydrateRsvps=5&hydrateEvent=true # combine both ``` ### Examples (events) ``` # Upcoming events with 10+ going RSVPs, with RSVP records and profiles -/xrpc/community.lexicon.calendar.event.getRecords?startsAtMin=2026-03-16&rsvpsGoingCountMin=10&hydrate=rsvps:5&profiles=true +/xrpc/community.lexicon.calendar.event.listRecords?startsAtMin=2026-03-16&rsvpsGoingCountMin=10&hydrateRsvps=5&profiles=true # Events for a specific user (by handle) -/xrpc/community.lexicon.calendar.event.getRecords?actor=alice.bsky.social&profiles=true +/xrpc/community.lexicon.calendar.event.listRecords?actor=alice.bsky.social&profiles=true # Single event with counts, RSVPs, and profiles -/xrpc/community.lexicon.calendar.event.getRecord?uri=at://did:plc:.../community.lexicon.calendar.event/...&hydrate=rsvps:10&profiles=true +/xrpc/community.lexicon.calendar.event.getRecord?uri=at://did:plc:.../community.lexicon.calendar.event/...&hydrateRsvps=10&profiles=true + +# Search for events by name/description +/xrpc/community.lexicon.calendar.event.listRecords?search=meetup&profiles=true -# RSVPs for a specific event with profiles -/xrpc/community.lexicon.calendar.rsvp.getRecords?subjectUri=at://did:plc:.../community.lexicon.calendar.event/...&profiles=true +# RSVPs for a specific event, with the referenced event embedded +/xrpc/community.lexicon.calendar.rsvp.listRecords?subjectUri=at://did:plc:.../community.lexicon.calendar.event/...&hydrateEvent=true&profiles=true ``` +## Notify of Updates + +By default, Contrail ingests from Jetstream every minute. If your app writes to a user's PDS and needs the change reflected immediately, call `notifyOfUpdate` right after the write: + +```ts +// User creates an RSVP via their PDS +const { uri } = await agent.createRecord({ ... }); + +// Tell Contrail to fetch and index it now +await fetch("https://your-contrail.workers.dev/xrpc/com.example.notifyOfUpdate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri }), +}); +``` + +Contrail fetches the record from the user's PDS and figures out what to do: + +| PDS returns | Already indexed? | Action | +|---|---|---| +| Record (new CID) | No | **Create** — indexes it, updates relation counts | +| Record (new CID) | Yes | **Update** — upserts the record | +| Record (same CID) | Yes | **Skip** — nothing changed | +| 404 | Yes | **Delete** — removes it, decrements counts | +| 404 | No | **No-op** | + +You can also batch up to 25 URIs in one request: + +```ts +await fetch(".../xrpc/com.example.notifyOfUpdate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uris: [uri1, uri2, uri3] }), +}); +``` + +When Jetstream later delivers the same event, the duplicate is detected by CID and skipped. + ## Typesafe Client Usage You can get fully typed XRPC queries for any Contrail instance using [`@atcute/lex-cli`](https://github.com/mary-ext/atcute). The lexicon files are committed to the repo, so you can pull them directly via the git source. diff --git a/lexicons-generated/community/lexicon/calendar/event/listRecords.json b/lexicons-generated/community/lexicon/calendar/event/listRecords.json index f86bbcc..4254151 100644 --- a/lexicons-generated/community/lexicon/calendar/event/listRecords.json +++ b/lexicons-generated/community/lexicon/calendar/event/listRecords.json @@ -26,6 +26,10 @@ "type": "boolean", "description": "Include profile + identity info keyed by DID" }, + "search": { + "type": "string", + "description": "Full-text search across: mode, name, status, description" + }, "mode": { "type": "string", "description": "Filter by mode" diff --git a/lexicons-generated/community/lexicon/calendar/rsvp/listRecords.json b/lexicons-generated/community/lexicon/calendar/rsvp/listRecords.json index 1c9efbb..accc91c 100644 --- a/lexicons-generated/community/lexicon/calendar/rsvp/listRecords.json +++ b/lexicons-generated/community/lexicon/calendar/rsvp/listRecords.json @@ -26,6 +26,10 @@ "type": "boolean", "description": "Include profile + identity info keyed by DID" }, + "search": { + "type": "string", + "description": "Full-text search across: status, subject.uri" + }, "status": { "type": "string", "description": "Filter by status" diff --git a/lexicons-generated/rsvp/atmo/notifyOfUpdate.json b/lexicons-generated/rsvp/atmo/notifyOfUpdate.json new file mode 100644 index 0000000..a91c2e7 --- /dev/null +++ b/lexicons-generated/rsvp/atmo/notifyOfUpdate.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "rsvp.atmo.notifyOfUpdate", + "defs": { + "main": { + "type": "procedure", + "description": "Notify of a record change for immediate indexing. Fetches the record from the user's PDS and indexes (or deletes) it.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "format": "at-uri", + "description": "Single AT URI to fetch and index" + }, + "uris": { + "type": "array", + "items": { + "type": "string", + "format": "at-uri" + }, + "maxLength": 25, + "description": "Batch of AT URIs to fetch and index (max 25)" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "indexed", + "deleted" + ], + "properties": { + "indexed": { + "type": "integer", + "description": "Number of records created or updated" + }, + "deleted": { + "type": "integer", + "description": "Number of records deleted (not found on PDS)" + }, + "errors": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Errors for individual URIs that could not be processed" + } + } + } + } + } + } +} diff --git a/scripts/generate-lexicons.ts b/scripts/generate-lexicons.ts index 4414422..02891ec 100644 --- a/scripts/generate-lexicons.ts +++ b/scripts/generate-lexicons.ts @@ -1,877 +1,18 @@ /** - * Generates lexicon TypeScript files from the Contrail config. - * - * For each collection, generates: - * - {nsid}.listRecords — query with queryable field params - * - {nsid}.getUsers — query with limit/cursor - * - {nsid}.getStats — query returning collection stats - * - * Plus namespaced endpoints: - * - {namespace}.admin.getCursor - * - {namespace}.admin.getOverview - * - {namespace}.admin.sync - * - {namespace}.admin.reset - * - {namespace}.getProfile + * Generates lexicon files, lex.config.js, and queryable.generated.ts from config. * * Usage: npx tsx scripts/generate-lexicons.ts */ -import { writeFileSync, mkdirSync, rmSync, existsSync, readFileSync, readdirSync } from "fs"; import { join } from "path"; import { config } from "../src/config"; +import { generateLexicons } from "../src/generate"; const ROOT_DIR = join(__dirname, ".."); -const USER_LEXICONS_DIR = join(ROOT_DIR, "lexicons"); -const PULLED_LEXICONS_DIR = join(ROOT_DIR, "lexicons-pulled"); -const GENERATED_DIR = join(ROOT_DIR, "lexicons-generated"); - -function fieldToParam(field: string): string { - return field.replace(/\.(\w)/g, (_, c) => c.toUpperCase()); -} - -interface QueryableField { - type?: "range"; -} - -// Find a collection's lexicon file (user-provided takes priority over pulled) -function findCollectionLexicon(collection: string): string | null { - const segments = collection.split("."); - for (const dir of [USER_LEXICONS_DIR, PULLED_LEXICONS_DIR]) { - const filePath = join(dir, ...segments) + ".json"; - if (existsSync(filePath)) return filePath; - } - return null; -} - -// Analyze a collection's lexicon and return auto-detected queryable fields -function detectQueryableFields(collection: string): Record { - const filePath = findCollectionLexicon(collection); - if (!filePath) return {}; - try { - const doc = JSON.parse(readFileSync(filePath, "utf-8")); - const mainRecord = doc.defs?.main?.record; - if (!mainRecord?.properties) return {}; - return analyzeProperties(doc.defs, mainRecord.properties, ""); - } catch { - return {}; - } -} - -function analyzeProperties( - defs: Record, - properties: Record, - prefix: string -): Record { - const result: Record = {}; - - for (const [field, def] of Object.entries(properties)) { - const path = prefix ? `${prefix}.${field}` : field; - - if (def.type === "string") { - if (def.format === "datetime") { - result[path] = { type: "range" }; - } else if (def.format !== "uri" && def.format !== "at-uri") { - // Regular strings (enums, free text) → equality - result[path] = {}; - } - } else if (def.type === "integer" || def.type === "number") { - result[path] = { type: "range" }; - } else if (def.type === "ref" && def.ref === "com.atproto.repo.strongRef") { - result[`${path}.uri`] = {}; - } else if (def.type === "union" && Array.isArray(def.refs) && def.refs.includes("com.atproto.repo.strongRef")) { - result[`${path}.uri`] = {}; - } else if (def.type === "ref" && def.ref) { - // Resolve local ref (e.g. #mode → defs.mode) - const refId = def.ref.includes("#") ? def.ref.split("#")[1] : null; - if (refId && defs[refId]) { - const resolved = defs[refId]; - if (resolved.type === "string") { - // String enum (knownValues) → equality - result[path] = {}; - } - } - } - } - - return result; -} - -// Extract knownValues for a field from a collection's lexicon -function getKnownValues(collection: string, fieldName: string): string[] { - const filePath = findCollectionLexicon(collection); - if (!filePath) return []; - try { - const doc = JSON.parse(readFileSync(filePath, "utf-8")); - const props = doc.defs?.main?.record?.properties; - if (!props) return []; - const field = props[fieldName]; - if (!field) return []; - if (Array.isArray(field.knownValues)) return field.knownValues; - return []; - } catch { - return []; - } -} - -// Default mapping: "community.lexicon.calendar.rsvp#going" → "going" -function tokenShortName(token: string): string { - const hash = token.indexOf("#"); - return hash !== -1 ? token.slice(hash + 1) : token; -} - -// Clean generated dir (user-provided lexicons/ is untouched) -rmSync(GENERATED_DIR, { recursive: true, force: true }); - -function nsidToPath(nsid: string): string { - return join(GENERATED_DIR, ...nsid.split(".")) + ".json"; -} - -// Check if a collection lexicon exists (user-provided or pulled) -function getCollectionLexiconRef(collection: string): string | null { - const filePath = findCollectionLexicon(collection); - if (!filePath) return null; - try { - const doc = JSON.parse(readFileSync(filePath, "utf-8")); - if (doc.defs?.main) return `${collection}#main`; - } catch {} - return null; -} - -function ensureDir(filePath: string) { - mkdirSync(join(filePath, ".."), { recursive: true }); -} - -function writeLexicon(nsid: string, doc: object) { - const filePath = nsidToPath(nsid); - ensureDir(filePath); - writeFileSync(filePath, JSON.stringify(doc, null, 2) + "\n"); - console.log(` ${nsid}`); -} - -// Build record output shape, optionally typing the record field -// countFields: e.g. [{ name: "rsvpsTotal", description: "..." }, { name: "rsvpsGoing", ... }] -interface CountField { - name: string; - description: string; -} - -interface RelationDef { - relName: string; - collection: string; - groupBy?: string; - groups: Record; // shortName → full token -} - -interface ReferenceDef { - refName: string; - collection: string; -} - -function buildRecordDef( - collectionRef: string | null, - countFields?: CountField[], - relationDefs?: RelationDef[], - referenceDefs?: ReferenceDef[] -) { - const properties: Record = { - uri: { type: "string", format: "at-uri" }, - did: { type: "string", format: "did" }, - collection: { type: "string", format: "nsid" }, - rkey: { type: "string" }, - cid: { type: "string" }, - record: collectionRef - ? { type: "ref", ref: collectionRef } - : { type: "unknown" }, - time_us: { type: "integer" }, - }; - - if (countFields) { - for (const cf of countFields) { - properties[cf.name] = { type: "integer", description: cf.description }; - } - } - - if (relationDefs && relationDefs.length > 0) { - for (const rd of relationDefs) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - if (rd.groupBy && Object.keys(rd.groups).length > 0) { - properties[rd.relName] = { - type: "ref", - ref: `#hydrate${capitalize(rd.relName)}`, - }; - } else { - properties[rd.relName] = { - type: "array", - items: { type: "ref", ref: `#hydrate${capitalize(rd.relName)}Record` }, - }; - } - } - } - - if (referenceDefs && referenceDefs.length > 0) { - for (const rd of referenceDefs) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - properties[rd.refName] = { - type: "ref", - ref: `#ref${capitalize(rd.refName)}Record`, - }; - } - } - return { - type: "object", - required: ["uri", "did", "collection", "rkey", "time_us"], - properties, - }; -} - -function buildReferenceDefs(referenceDefs: ReferenceDef[]): Record { - const defs: Record = {}; - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - for (const rd of referenceDefs) { - const refCollectionRef = getCollectionLexiconRef(rd.collection); - const recordDefName = `ref${capitalize(rd.refName)}Record`; - defs[recordDefName] = { - type: "object", - required: ["uri", "did", "collection", "rkey", "time_us"], - properties: { - uri: { type: "string", format: "at-uri" }, - did: { type: "string", format: "did" }, - collection: { type: "string", format: "nsid" }, - rkey: { type: "string" }, - cid: { type: "string" }, - record: refCollectionRef - ? { type: "ref", ref: refCollectionRef } - : { type: "unknown" }, - time_us: { type: "integer" }, - }, - }; - } - - return defs; -} - -function buildHydrateDefs(relationDefs: RelationDef[]): Record { - const defs: Record = {}; - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - for (const rd of relationDefs) { - const relCollectionRef = getCollectionLexiconRef(rd.collection); - - // Def for each hydrated record - const recordDefName = `hydrate${capitalize(rd.relName)}Record`; - defs[recordDefName] = { - type: "object", - required: ["uri", "did", "collection", "rkey", "time_us"], - properties: { - uri: { type: "string", format: "at-uri" }, - did: { type: "string", format: "did" }, - collection: { type: "string", format: "nsid" }, - rkey: { type: "string" }, - cid: { type: "string" }, - record: relCollectionRef - ? { type: "ref", ref: relCollectionRef } - : { type: "unknown" }, - time_us: { type: "integer" }, - }, - }; - - if (rd.groupBy && Object.keys(rd.groups).length > 0) { - // Grouped relation: object with group keys - const groupDefName = `hydrate${capitalize(rd.relName)}`; - const groupProperties: Record = {}; - for (const shortName of Object.keys(rd.groups)) { - groupProperties[shortName] = { - type: "array", - items: { type: "ref", ref: `#${recordDefName}` }, - }; - } - groupProperties["other"] = { - type: "array", - items: { type: "ref", ref: `#${recordDefName}` }, - }; - defs[groupDefName] = { - type: "object", - properties: groupProperties, - }; - } - // Ungrouped relations are typed directly as arrays on the record (no wrapper def needed) - } - - return defs; -} - -// Read the inner record object schema from a collection's lexicon -function getRecordObjectSchema(collection: string): any | null { - const filePath = findCollectionLexicon(collection); - if (!filePath) return null; - try { - const doc = JSON.parse(readFileSync(filePath, "utf-8")); - const main = doc.defs?.main; - if (main?.type === "record" && main.record) return main.record; - return null; - } catch { - return null; - } -} - -function profileDefs() { - const profiles = config.profiles ?? ["app.bsky.actor.profile"]; - const extraDefs: Record = {}; - const objectRefs: string[] = []; - - for (const col of profiles) { - const schema = getRecordObjectSchema(col); - if (!schema) continue; - // Create a local def name from the NSID, e.g. "app.bsky.actor.profile" → "appBskyActorProfile" - const defName = col.split(".").map((p, i) => i === 0 ? p : p.charAt(0).toUpperCase() + p.slice(1)).join(""); - extraDefs[defName] = schema; - objectRefs.push(`#${defName}`); - } - - let recordField: any; - if (objectRefs.length === 1) { - recordField = { type: "ref", ref: objectRefs[0] }; - } else if (objectRefs.length > 1) { - recordField = { type: "union", refs: objectRefs }; - } else { - recordField = { type: "unknown" }; - } - - return { - profileEntry: { - type: "object", - required: ["did"], - properties: { - did: { type: "string", format: "did" }, - handle: { type: "string" }, - uri: { type: "string", format: "at-uri" }, - collection: { type: "string", format: "nsid" }, - rkey: { type: "string" }, - cid: { type: "string" }, - record: recordField, - }, - }, - ...extraDefs, - }; -} - -// --- Namespace --- - -const ns = config.namespace!; - -// --- Admin endpoints --- - -console.log("Generating admin endpoints..."); - -writeLexicon(`${ns}.admin.getCursor`, { - lexicon: 1, - id: `${ns}.admin.getCursor`, - defs: { - main: { - type: "query", - description: "Get the current cursor position", - output: { - encoding: "application/json", - schema: { - type: "object", - properties: { - time_us: { type: "integer" }, - date: { type: "string" }, - seconds_ago: { type: "integer" }, - }, - }, - }, - }, - }, +generateLexicons({ + config, + rootDir: ROOT_DIR, + outputDir: join(ROOT_DIR, "lexicons-generated"), + writeRuntimeFiles: true, }); - -writeLexicon(`${ns}.admin.getOverview`, { - lexicon: 1, - id: `${ns}.admin.getOverview`, - defs: { - main: { - type: "query", - description: "Get an overview of all indexed collections", - output: { - encoding: "application/json", - schema: { - type: "object", - required: ["total_records", "collections"], - properties: { - total_records: { type: "integer" }, - collections: { - type: "array", - items: { type: "ref", ref: "#collectionStats" }, - }, - }, - }, - }, - }, - collectionStats: { - type: "object", - required: ["collection", "records", "unique_users"], - properties: { - collection: { type: "string" }, - records: { type: "integer" }, - unique_users: { type: "integer" }, - }, - }, - }, -}); - -writeLexicon(`${ns}.admin.sync`, { - lexicon: 1, - id: `${ns}.admin.sync`, - defs: { - main: { - type: "query", - description: "Discover users from relays and backfill their records from PDS", - parameters: { - type: "params", - properties: { - concurrency: { - type: "integer", - minimum: 1, - maximum: 50, - default: 10, - }, - }, - }, - output: { - encoding: "application/json", - schema: { - type: "object", - required: ["discovered", "backfilled", "remaining", "done"], - properties: { - discovered: { type: "integer" }, - backfilled: { type: "integer" }, - remaining: { type: "integer" }, - done: { type: "boolean" }, - }, - }, - }, - }, - }, -}); - -writeLexicon(`${ns}.admin.reset`, { - lexicon: 1, - id: `${ns}.admin.reset`, - defs: { - main: { - type: "query", - description: "Delete all data from all tables", - output: { - encoding: "application/json", - schema: { - type: "object", - required: ["ok"], - properties: { - ok: { type: "boolean" }, - }, - }, - }, - }, - }, -}); - -// --- getProfile endpoint --- - -writeLexicon(`${ns}.getProfile`, { - lexicon: 1, - id: `${ns}.getProfile`, - defs: { - main: { - type: "query", - description: "Get a user's profile by DID or handle", - parameters: { - type: "params", - required: ["actor"], - properties: { - actor: { - type: "string", - format: "at-identifier", - description: "DID or handle of the user", - }, - }, - }, - output: { - encoding: "application/json", - schema: { - type: "ref", - ref: "#profileEntry", - }, - }, - }, - ...profileDefs(), - }, -}); - -// --- Per-collection endpoints --- - -console.log("Generating collection endpoints..."); - -// Collect resolved queryable fields for all collections -const resolvedQueryable: Record> = {}; -// Collect resolved relation mappings (short name → full token) for runtime -const resolvedRelations: Record }>> = {}; - -for (const [collection, colConfig] of Object.entries(config.collections)) { - const collectionRef = getCollectionLexiconRef(collection); - if (collectionRef) { - console.log(` → ${collection} record typed via lexicon`); - } - - // Auto-detect queryable fields from lexicon, then merge manual overrides - const autoDetected = detectQueryableFields(collection); - const manual = colConfig.queryable ?? {}; - const merged = { ...autoDetected, ...manual }; - resolvedQueryable[collection] = merged; - - if (Object.keys(autoDetected).length > 0) { - const autoOnly = Object.keys(autoDetected).filter((k) => !manual[k]); - if (autoOnly.length > 0) { - console.log(` → auto-detected queryable: ${autoOnly.join(", ")}`); - } - } - - // --- listRecords --- - const listRecordsParamProps: Record = { - limit: { type: "integer", minimum: 1, maximum: 100, default: 50 }, - cursor: { type: "string" }, - actor: { type: "string", format: "at-identifier", description: "Filter by DID or handle (triggers on-demand backfill)" }, - profiles: { type: "boolean", description: "Include profile + identity info keyed by DID" }, - }; - - for (const [field, fieldConfig] of Object.entries(merged)) { - const param = fieldToParam(field); - if (fieldConfig.type === "range") { - listRecordsParamProps[`${param}Min`] = { - type: "string", - description: `Minimum value for ${field}`, - }; - listRecordsParamProps[`${param}Max`] = { - type: "string", - description: `Maximum value for ${field}`, - }; - } else { - listRecordsParamProps[param] = { - type: "string", - description: `Filter by ${field}`, - }; - } - } - - // Build count fields, hydrate params, and relation defs from relations + knownValues - const countFields: CountField[] = []; - const relationDefs: RelationDef[] = []; - for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - // Total count - countFields.push({ name: `${relName}Count`, description: `Total ${relName} count` }); - listRecordsParamProps[`${relName}CountMin`] = { - type: "integer", - description: `Minimum total ${relName} count`, - }; - - // Per-relation hydrate param (e.g. hydrateRsvps=5) - listRecordsParamProps[`hydrate${capitalize(relName)}`] = { - type: "integer", - minimum: 1, - maximum: 50, - description: `Number of ${relName} records to embed per record`, - }; - - // Per-group counts from knownValues - const groupMapping: Record = {}; - if (rel.groupBy) { - const knownValues = getKnownValues(rel.collection, rel.groupBy); - for (const token of knownValues) { - const shortName = tokenShortName(token); - groupMapping[shortName] = token; - countFields.push({ - name: `${relName}${capitalize(shortName)}Count`, - description: `${relName} count where ${rel.groupBy} = ${shortName}`, - }); - listRecordsParamProps[`${relName}${capitalize(shortName)}CountMin`] = { - type: "integer", - description: `Minimum ${relName} count where ${rel.groupBy} = ${shortName}`, - }; - } - // Store mapping for runtime use - if (!resolvedRelations[collection]) resolvedRelations[collection] = {}; - resolvedRelations[collection][relName] = { - collection: rel.collection, - groupBy: rel.groupBy, - groups: groupMapping, - }; - } - - relationDefs.push({ - relName, - collection: rel.collection, - groupBy: rel.groupBy, - groups: groupMapping, - }); - } - - // Build reference defs - const referenceDefs: ReferenceDef[] = []; - for (const [refName, ref] of Object.entries(colConfig.references ?? {})) { - referenceDefs.push({ refName, collection: ref.collection }); - } - - // Add per-reference hydrate params (e.g. hydrateEvent=true) - for (const refName of Object.keys(colConfig.references ?? {})) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - listRecordsParamProps[`hydrate${capitalize(refName)}`] = { - type: "boolean", - description: `Embed the referenced ${refName} record`, - }; - } - - // Build sortable field values: queryable fields + count fields - const sortableValues: string[] = []; - for (const field of Object.keys(merged)) { - sortableValues.push(fieldToParam(field)); - } - for (const cf of countFields) { - sortableValues.push(cf.name); - } - - if (sortableValues.length > 0) { - listRecordsParamProps["sort"] = { - type: "string", - knownValues: sortableValues, - description: "Field to sort by (default: time_us)", - }; - listRecordsParamProps["order"] = { - type: "string", - knownValues: ["asc", "desc"], - description: "Sort direction (default: desc for dates/numbers/counts, asc for strings)", - }; - } - - const hydrateDefs = buildHydrateDefs(relationDefs); - const refDefs = buildReferenceDefs(referenceDefs); - - writeLexicon(`${collection}.listRecords`, { - lexicon: 1, - id: `${collection}.listRecords`, - defs: { - main: { - type: "query", - description: `Query ${collection} records with filters`, - parameters: { - type: "params", - properties: listRecordsParamProps, - }, - output: { - encoding: "application/json", - schema: { - type: "object", - required: ["records"], - properties: { - records: { type: "array", items: { type: "ref", ref: "#record" } }, - cursor: { type: "string" }, - profiles: { type: "array", items: { type: "ref", ref: "#profileEntry" } }, - }, - }, - }, - }, - record: buildRecordDef(collectionRef, countFields, relationDefs, referenceDefs), - ...hydrateDefs, - ...refDefs, - ...profileDefs(), - }, - }); - - // --- getRecord --- - const getRecordParamProps: Record = { - uri: { type: "string", format: "at-uri", description: "AT URI of the record" }, - profiles: { type: "boolean", description: "Include profile + identity info keyed by DID" }, - }; - - // Add per-relation hydrate params to getRecord too - for (const rd of relationDefs) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - getRecordParamProps[`hydrate${capitalize(rd.relName)}`] = { - type: "integer", - minimum: 1, - maximum: 50, - description: `Number of ${rd.relName} records to embed`, - }; - } - - // Add per-reference hydrate params to getRecord too - for (const refName of Object.keys(colConfig.references ?? {})) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - getRecordParamProps[`hydrate${capitalize(refName)}`] = { - type: "boolean", - description: `Embed the referenced ${refName} record`, - }; - } - - writeLexicon(`${collection}.getRecord`, { - lexicon: 1, - id: `${collection}.getRecord`, - defs: { - main: { - type: "query", - description: `Get a single ${collection} record by AT URI`, - parameters: { - type: "params", - required: ["uri"], - properties: getRecordParamProps, - }, - output: { - encoding: "application/json", - schema: { - type: "object", - required: ["uri", "did", "collection", "rkey", "time_us"], - properties: { - ...buildRecordDef(collectionRef, countFields, relationDefs, referenceDefs).properties, - profiles: { type: "array", items: { type: "ref", ref: "#profileEntry" } }, - }, - }, - }, - }, - ...hydrateDefs, - ...refDefs, - ...profileDefs(), - }, - }); - - - // --- Custom queries --- - for (const queryName of Object.keys(colConfig.queries ?? {})) { - writeLexicon(`${collection}.${queryName}`, { - lexicon: 1, - id: `${collection}.${queryName}`, - defs: { - main: { - type: "query", - description: `Custom query: ${queryName}`, - output: { - encoding: "application/json", - schema: { type: "object", properties: {} }, - }, - }, - }, - }); - } -} - -// --- Auto-generate lex.config.js --- - -// Collect all collection NSIDs from config -const collectionNsids = Object.keys(config.collections); - -// Scan pulled lexicons for external refs to find transitive deps -function findRefsInLexicon(filePath: string): string[] { - try { - const content = readFileSync(filePath, "utf-8"); - const refs: string[] = []; - // Match all "ref": "some.nsid.here" or "refs": ["some.nsid.here"] - const refPattern = /"ref":\s*"([a-z][a-zA-Z0-9]*(?:\.[a-zA-Z0-9]+)+)(?:#\w+)?"/g; - let match; - while ((match = refPattern.exec(content)) !== null) { - refs.push(match[1]); - } - // Also match refs arrays - const refsArrayPattern = /"refs":\s*\[([^\]]+)\]/g; - while ((match = refsArrayPattern.exec(content)) !== null) { - const inner = match[1]; - const nsidPattern = /"([a-z][a-zA-Z0-9]*(?:\.[a-zA-Z0-9]+)+)(?:#\w+)?"/g; - let innerMatch; - while ((innerMatch = nsidPattern.exec(inner)) !== null) { - refs.push(innerMatch[1]); - } - } - return refs; - } catch { - return []; - } -} - -function scanLexiconsDir(dir: string): string[] { - const files: string[] = []; - if (!existsSync(dir)) return files; - for (const entry of readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name); - if (entry.isDirectory()) { - files.push(...scanLexiconsDir(fullPath)); - } else if (entry.name.endsWith(".json")) { - files.push(fullPath); - } - } - return files; -} - -// Find all NSIDs referenced by pulled lexicons -const pulledFiles = [ - ...scanLexiconsDir(USER_LEXICONS_DIR), - ...scanLexiconsDir(PULLED_LEXICONS_DIR), -]; -const allRefs = new Set(); -for (const file of pulledFiles) { - for (const ref of findRefsInLexicon(file)) { - allRefs.add(ref); - } -} - -// Merge: collection NSIDs + profile NSIDs + transitive deps (excluding com.atproto.* which comes from imports) -const profileNsids = config.profiles ?? ["app.bsky.actor.profile"]; -const pullNsids = new Set([...collectionNsids, ...profileNsids]); -for (const ref of allRefs) { - if (!ref.startsWith("com.atproto.")) { - pullNsids.add(ref); - } -} - -const sortedNsids = [...pullNsids].sort(); - -const lexConfigContent = `import { defineLexiconConfig } from "@atcute/lex-cli"; - -export default defineLexiconConfig({ - files: ["lexicons/**/*.json", "lexicons-pulled/**/*.json", "lexicons-generated/**/*.json"], - outdir: "src/lexicon-types/", - imports: ["@atcute/atproto"], - pull: { - outdir: "lexicons-pulled/", - sources: [ - { - type: "atproto", - mode: "nsids", - nsids: ${JSON.stringify(sortedNsids, null, 10).replace(/^/gm, " ").trim()}, - }, - ], - }, -}); -`; - -writeFileSync(join(ROOT_DIR, "lex.config.js"), lexConfigContent); -console.log(`\nGenerated lex.config.js with ${sortedNsids.length} pull NSIDs`); - -// Generate resolved queryable config for runtime use -const queryableContent = `// Auto-generated — do not edit. Run \`pnpm generate\` to regenerate. -import type { QueryableField } from "./types"; - -export const resolvedQueryable: Record> = ${JSON.stringify(resolvedQueryable, null, 2)}; - -export interface ResolvedRelation { - collection: string; - groupBy: string; - groups: Record; // shortName → full token value -} - -export const resolvedRelationsMap: Record> = ${JSON.stringify(resolvedRelations, null, 2)}; -`; - -writeFileSync(join(ROOT_DIR, "src", "core", "queryable.generated.ts"), queryableContent); -console.log("Generated src/core/queryable.generated.ts"); - -console.log("\nDone!"); diff --git a/src/core/db/records.ts b/src/core/db/records.ts index a40c45d..71b030d 100644 --- a/src/core/db/records.ts +++ b/src/core/db/records.ts @@ -7,6 +7,7 @@ import type { RecordRow, } from "../types"; import { getNestedValue, getRelationField } from "../types"; +import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; // --- Counts --- @@ -71,6 +72,41 @@ function buildCountStatements( return statements; } +// --- FTS --- + +function buildFtsStatements( + db: Database, + event: IngestEvent, + config: ContrailConfig +): Statement[] { + const colConfig = config.collections[event.collection]; + if (!colConfig) return []; + + const fields = getSearchableFields(event.collection, colConfig); + if (!fields || fields.length === 0) return []; + + const table = ftsTableName(event.collection); + const stmts: Statement[] = []; + + if (event.operation === "delete") { + stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); + } else { + const record = event.record ? JSON.parse(event.record) : null; + if (!record) return []; + + const content = buildFtsContent(record, fields); + if (!content) return []; + + // Delete-then-insert handles both create and update + stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); + stmts.push( + db.prepare(`INSERT INTO ${table} (uri, content) VALUES (?, ?)`).bind(event.uri, content) + ); + } + + return stmts; +} + // --- Cursor --- export async function getLastCursor(db: Database): Promise { @@ -128,6 +164,7 @@ export async function applyEvents( if (config) { batch.push(...buildCountStatements(db, e, config)); + batch.push(...buildFtsStatements(db, e, config)); } } @@ -151,6 +188,7 @@ export interface QueryOptions { rangeFilters?: Record; countFilters?: Record; sort?: SortOption; + search?: string; } export async function queryRecords( @@ -167,6 +205,7 @@ export async function queryRecords( rangeFilters = {}, countFilters = {}, sort, + search, } = options; const limit = Math.min(Math.max(1, rawLimit ?? 50), 100); @@ -218,6 +257,19 @@ export async function queryRecords( } } + // FTS search + let ftsJoin = ""; + if (search) { + const colConfig2 = config.collections[collection]; + const fields = colConfig2 ? getSearchableFields(collection, colConfig2) : null; + if (fields && fields.length > 0) { + const table = ftsTableName(collection); + ftsJoin = `JOIN ${table} fts ON fts.uri = r.uri`; + conditions.push("fts.content MATCH ?"); + bindings.push(search); + } + } + const colConfig = config.collections[collection]; const relations = colConfig?.relations ?? {}; const sortByCount = sort?.countType != null; @@ -254,7 +306,10 @@ export async function queryRecords( const select = needsCounts ? "r.uri, r.did, r.collection, r.rkey, r.cid, r.record, r.time_us, r.indexed_at, GROUP_CONCAT(c.type || ':' || c.count) as _counts" : "r.uri, r.did, r.collection, r.rkey, r.cid, r.record, r.time_us, r.indexed_at"; - const join = needsCounts ? "LEFT JOIN counts c ON c.uri = r.uri" : ""; + const joinParts: string[] = []; + if (ftsJoin) joinParts.push(ftsJoin); + if (needsCounts) joinParts.push("LEFT JOIN counts c ON c.uri = r.uri"); + const join = joinParts.join(" "); const group = needsCounts ? "GROUP BY r.uri" : ""; const having = countHaving.length > 0 ? `HAVING ${countHaving.join(" AND ")}` : ""; @@ -267,6 +322,8 @@ export async function queryRecords( const dir = sort.direction === "desc" ? "DESC" : "ASC"; orderBy = `COALESCE(SUM(CASE WHEN c.type = ? THEN c.count END), 0) ${dir}, r.time_us DESC`; orderBindings.push(sort.countType); + } else if (ftsJoin) { + orderBy = "fts.rank, r.time_us DESC"; } else { orderBy = "r.time_us DESC"; } diff --git a/src/core/db/schema.ts b/src/core/db/schema.ts index 76f485b..6d204a4 100644 --- a/src/core/db/schema.ts +++ b/src/core/db/schema.ts @@ -1,6 +1,7 @@ import type { ContrailConfig, Database } from "../types"; import { getRelationField } from "../types"; import { resolvedQueryable } from "../queryable.generated"; +import { getSearchableFields, ftsTableName } from "../search"; const BASE_SCHEMA = ` CREATE TABLE IF NOT EXISTS records ( @@ -76,6 +77,19 @@ function buildDynamicIndexes(config: ContrailConfig): string[] { return indexes; } +function buildFtsTables(config: ContrailConfig): string[] { + const stmts: string[] = []; + for (const [collection, colConfig] of Object.entries(config.collections)) { + const fields = getSearchableFields(collection, colConfig); + if (!fields || fields.length === 0) continue; + const table = ftsTableName(collection); + stmts.push( + `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(uri UNINDEXED, content)` + ); + } + return stmts; +} + const MIGRATIONS = [ "ALTER TABLE backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", "ALTER TABLE backfills ADD COLUMN last_error TEXT", @@ -100,7 +114,8 @@ export async function initSchema( .filter((s) => s.length > 0); const indexStatements = buildDynamicIndexes(config); - const all = [...baseStatements, ...indexStatements]; + const ftsStatements = buildFtsTables(config); + const all = [...baseStatements, ...indexStatements, ...ftsStatements]; await db.batch(all.map((s) => db.prepare(s))); await runMigrations(db); diff --git a/src/core/router/collection.ts b/src/core/router/collection.ts index 5b6a091..6ed0697 100644 --- a/src/core/router/collection.ts +++ b/src/core/router/collection.ts @@ -110,6 +110,8 @@ export function registerCollectionRoutes( } } + const search = params.get("search") || undefined; + const result = await queryRecords(db, config, { collection, did, @@ -119,6 +121,7 @@ export function registerCollectionRoutes( rangeFilters, countFilters, sort, + search, }); const rows = result.records; diff --git a/src/core/router/index.ts b/src/core/router/index.ts index 9d56168..9539586 100644 --- a/src/core/router/index.ts +++ b/src/core/router/index.ts @@ -3,6 +3,7 @@ import { cors } from "hono/cors"; import type { Database, ContrailConfig } from "../types"; import { registerAdminRoutes } from "./admin"; import { registerCollectionRoutes } from "./collection"; +import { registerNotifyRoute } from "./notify"; import { resolveActor } from "../identity"; import { resolveProfiles } from "./profiles"; import { backfillUser } from "../backfill"; @@ -43,6 +44,7 @@ export function createApp( registerAdminRoutes(app, db, config, adminSecret); registerCollectionRoutes(app, db, config); + registerNotifyRoute(app, db, config); return app; } diff --git a/src/core/router/notify.ts b/src/core/router/notify.ts new file mode 100644 index 0000000..b01734f --- /dev/null +++ b/src/core/router/notify.ts @@ -0,0 +1,149 @@ +import type { Hono } from "hono"; +import type { Database, ContrailConfig, IngestEvent } from "../types"; +import { applyEvents } from "../db/records"; +import { getPDS } from "../client"; +import type { Did } from "@atcute/lexicons"; + +/** Parse an AT URI into its components. */ +export function parseAtUri(uri: string): { did: string; collection: string; rkey: string } | null { + const match = uri.match(/^at:\/\/(did:[^/]+)\/([^/]+)\/([^/]+)$/); + if (!match) return null; + return { did: match[1], collection: match[2], rkey: match[3] }; +} + +/** + * Fetch a single record from the user's PDS. + * Returns the record + cid on success, null if not found. + */ +async function fetchRecordFromPDS( + pds: string, + did: string, + collection: string, + rkey: string +): Promise<{ value: unknown; cid: string } | null> { + const url = new URL(`/xrpc/com.atproto.repo.getRecord`, pds); + url.searchParams.set("repo", did); + url.searchParams.set("collection", collection); + url.searchParams.set("rkey", rkey); + + const res = await fetch(url.toString()); + if (!res.ok) return null; + + const data = (await res.json()) as { value?: unknown; cid?: string }; + if (!data.value || !data.cid) return null; + return { value: data.value, cid: data.cid }; +} + +export function registerNotifyRoute( + app: Hono, + db: Database, + config: ContrailConfig +) { + const ns = config.namespace; + + app.post(`/xrpc/${ns}.notifyOfUpdate`, async (c) => { + const body = await c.req.json<{ uri?: string; uris?: string[] }>().catch(() => null); + const uris: string[] = []; + + if (body?.uris && Array.isArray(body.uris)) { + uris.push(...body.uris); + } else if (body?.uri) { + uris.push(body.uri); + } else { + return c.json({ error: "uri or uris required" }, 400); + } + + if (uris.length > 25) { + return c.json({ error: "max 25 URIs per request" }, 400); + } + + const events: IngestEvent[] = []; + const errors: string[] = []; + + for (const uri of uris) { + const parsed = parseAtUri(uri); + if (!parsed) { + errors.push(`invalid AT URI: ${uri}`); + continue; + } + + // Only accept collections we're tracking + if (!config.collections[parsed.collection]) { + errors.push(`collection not tracked: ${parsed.collection}`); + continue; + } + + const pds = await getPDS(parsed.did as Did, db); + if (!pds) { + errors.push(`could not resolve PDS for ${parsed.did}`); + continue; + } + + const result = await fetchRecordFromPDS( + pds, + parsed.did, + parsed.collection, + parsed.rkey + ); + + const now = Date.now() * 1000; // microseconds + + // Check if this record already exists locally + const existing = await db + .prepare("SELECT cid FROM records WHERE uri = ?") + .bind(uri) + .first<{ cid: string | null }>(); + + if (result) { + if (existing?.cid === result.cid) { + // Same CID — nothing changed, skip to avoid double-counting + continue; + } + + events.push({ + uri, + did: parsed.did, + collection: parsed.collection, + rkey: parsed.rkey, + // "update" skips count statements, "create" increments them. + // Only use "create" if the record is genuinely new. + operation: existing ? "update" : "create", + cid: result.cid, + record: JSON.stringify(result.value), + time_us: now, + indexed_at: now, + }); + } else if (existing) { + // Record gone from PDS but exists locally — delete it. + // We need the old record data so buildCountStatements can decrement counts. + const existingRecord = await db + .prepare("SELECT record FROM records WHERE uri = ?") + .bind(uri) + .first<{ record: string | null }>(); + + events.push({ + uri, + did: parsed.did, + collection: parsed.collection, + rkey: parsed.rkey, + operation: "delete", + cid: null, + record: existingRecord?.record ?? null, + time_us: now, + indexed_at: now, + }); + } + // If not on PDS and not local, nothing to do + } + + if (events.length > 0) { + await applyEvents(db, events, config); + } + + return c.json({ + indexed: events.filter((e) => e.operation === "create" || e.operation === "update").length, + deleted: events.filter((e) => e.operation === "delete").length, + errors: errors.length > 0 ? errors : undefined, + }); + }); +} diff --git a/src/core/search.ts b/src/core/search.ts new file mode 100644 index 0000000..26726de --- /dev/null +++ b/src/core/search.ts @@ -0,0 +1,40 @@ +import type { CollectionConfig } from "./types"; +import { getNestedValue } from "./types"; +import { resolvedQueryable } from "./queryable.generated"; + +/** + * Resolve which fields are searchable for a collection. + * Returns null if search is disabled or no fields found. + */ +export function getSearchableFields( + collection: string, + colConfig: CollectionConfig +): string[] | null { + if (colConfig.searchable === false) return null; + if (Array.isArray(colConfig.searchable)) { + return colConfig.searchable.length > 0 ? colConfig.searchable : null; + } + // Auto-detect: all non-range queryable fields + const queryable = resolvedQueryable[collection] ?? colConfig.queryable ?? {}; + const fields = Object.entries(queryable) + .filter(([, f]) => f.type !== "range") + .map(([name]) => name); + return fields.length > 0 ? fields : null; +} + +/** Sanitized FTS table name for a collection. */ +export function ftsTableName(collection: string): string { + return `fts_${collection.replace(/[^a-zA-Z0-9]/g, "_")}`; +} + +/** Extract searchable field values from a record and join them into a single string. */ +export function buildFtsContent(record: unknown, fields: string[]): string | null { + const parts: string[] = []; + for (const field of fields) { + const value = getNestedValue(record, field); + if (typeof value === "string" && value.length > 0) { + parts.push(value); + } + } + return parts.length > 0 ? parts.join(" ") : null; +} diff --git a/src/core/types.ts b/src/core/types.ts index f98f133..ce15f26 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -44,6 +44,8 @@ export interface CollectionConfig { /** Forward references: fields on this collection's records that point at another collection. */ references?: Record; queries?: Record; + /** FTS5 search fields. string[] = explicit fields, false = disabled, omitted = auto-detect non-range queryable fields */ + searchable?: string[] | false; } export const DEFAULT_PROFILES = ["app.bsky.actor.profile"]; @@ -133,6 +135,11 @@ export function validateConfig(config: ContrailConfig): void { if (rel.field) validateFieldName(rel.field); if (rel.groupBy) validateFieldName(rel.groupBy); } + if (Array.isArray(colConfig.searchable)) { + for (const field of colConfig.searchable) { + validateFieldName(field); + } + } } } diff --git a/src/generate.ts b/src/generate.ts new file mode 100644 index 0000000..bb8d386 --- /dev/null +++ b/src/generate.ts @@ -0,0 +1,525 @@ +/** + * Core lexicon generation logic, importable for testing. + * + * Builds lexicon JSON objects from a ContrailConfig. Separated from the + * script entry point so tests can call it with custom configs and output dirs. + */ + +import { writeFileSync, mkdirSync, rmSync, existsSync, readFileSync, readdirSync } from "fs"; +import { join } from "path"; +import type { ContrailConfig } from "./core/types"; + +export interface GenerateOptions { + config: ContrailConfig; + /** Root project directory (for finding lexicon source files). */ + rootDir: string; + /** Directory to write generated lexicons into (will be cleaned first). Omit for in-memory only. */ + outputDir?: string; + /** Additional lexicon source directories to search for collection schemas. */ + lexiconDirs?: string[]; + /** If true, also writes lex.config.js and queryable.generated.ts. */ + writeRuntimeFiles?: boolean; + /** Suppress console output. */ + quiet?: boolean; +} + +function fieldToParam(field: string): string { + return field.replace(/\.(\w)/g, (_, c) => c.toUpperCase()); +} + +interface QueryableField { + type?: "range"; +} + +interface CountField { + name: string; + description: string; +} + +interface RelationDef { + relName: string; + collection: string; + groupBy?: string; + groups: Record; +} + +interface ReferenceDef { + refName: string; + collection: string; +} + +export function generateLexicons(options: GenerateOptions): Record { + const { config, rootDir, outputDir, quiet } = options; + const lexiconDirs = options.lexiconDirs ?? [ + join(rootDir, "lexicons"), + join(rootDir, "lexicons-pulled"), + ]; + + const log = quiet ? () => {} : console.log; + const generated: Record = {}; + + // --- Helpers that depend on lexiconDirs --- + + function findCollectionLexicon(collection: string): string | null { + const segments = collection.split("."); + for (const dir of lexiconDirs) { + const filePath = join(dir, ...segments) + ".json"; + if (existsSync(filePath)) return filePath; + } + return null; + } + + function detectQueryableFields(collection: string): Record { + const filePath = findCollectionLexicon(collection); + if (!filePath) return {}; + try { + const doc = JSON.parse(readFileSync(filePath, "utf-8")); + const mainRecord = doc.defs?.main?.record; + if (!mainRecord?.properties) return {}; + return analyzeProperties(doc.defs, mainRecord.properties, ""); + } catch { + return {}; + } + } + + function getKnownValues(collection: string, fieldName: string): string[] { + const filePath = findCollectionLexicon(collection); + if (!filePath) return []; + try { + const doc = JSON.parse(readFileSync(filePath, "utf-8")); + const props = doc.defs?.main?.record?.properties; + if (!props) return []; + const field = props[fieldName]; + if (!field) return []; + if (Array.isArray(field.knownValues)) return field.knownValues; + return []; + } catch { + return []; + } + } + + function getCollectionLexiconRef(collection: string): string | null { + const filePath = findCollectionLexicon(collection); + if (!filePath) return null; + try { + const doc = JSON.parse(readFileSync(filePath, "utf-8")); + if (doc.defs?.main) return `${collection}#main`; + } catch {} + return null; + } + + function getRecordObjectSchema(collection: string): any | null { + const filePath = findCollectionLexicon(collection); + if (!filePath) return null; + try { + const doc = JSON.parse(readFileSync(filePath, "utf-8")); + const main = doc.defs?.main; + if (main?.type === "record" && main.record) return main.record; + return null; + } catch { + return null; + } + } + + // --- Writing --- + + if (outputDir) { + rmSync(outputDir, { recursive: true, force: true }); + } + + function writeLexicon(nsid: string, doc: object) { + if (outputDir) { + const filePath = join(outputDir, ...nsid.split(".")) + ".json"; + mkdirSync(join(filePath, ".."), { recursive: true }); + writeFileSync(filePath, JSON.stringify(doc, null, 2) + "\n"); + } + generated[nsid] = doc; + log(` ${nsid}`); + } + + // --- Building --- + + function buildRecordDef( + collectionRef: string | null, + countFields?: CountField[], + relationDefs?: RelationDef[], + referenceDefs?: ReferenceDef[] + ) { + const properties: Record = { + uri: { type: "string", format: "at-uri" }, + did: { type: "string", format: "did" }, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + cid: { type: "string" }, + record: collectionRef ? { type: "ref", ref: collectionRef } : { type: "unknown" }, + time_us: { type: "integer" }, + }; + if (countFields) { + for (const cf of countFields) { + properties[cf.name] = { type: "integer", description: cf.description }; + } + } + if (relationDefs && relationDefs.length > 0) { + for (const rd of relationDefs) { + const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + if (rd.groupBy && Object.keys(rd.groups).length > 0) { + properties[rd.relName] = { type: "ref", ref: `#hydrate${cap(rd.relName)}` }; + } else { + properties[rd.relName] = { type: "array", items: { type: "ref", ref: `#hydrate${cap(rd.relName)}Record` } }; + } + } + } + if (referenceDefs && referenceDefs.length > 0) { + for (const rd of referenceDefs) { + const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + properties[rd.refName] = { type: "ref", ref: `#ref${cap(rd.refName)}Record` }; + } + } + return { type: "object", required: ["uri", "did", "collection", "rkey", "time_us"], properties }; + } + + function buildHydrateDefs(relationDefs: RelationDef[]): Record { + const defs: Record = {}; + const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + for (const rd of relationDefs) { + const relCollectionRef = getCollectionLexiconRef(rd.collection); + const recordDefName = `hydrate${cap(rd.relName)}Record`; + defs[recordDefName] = { + type: "object", + required: ["uri", "did", "collection", "rkey", "time_us"], + properties: { + uri: { type: "string", format: "at-uri" }, + did: { type: "string", format: "did" }, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + cid: { type: "string" }, + record: relCollectionRef ? { type: "ref", ref: relCollectionRef } : { type: "unknown" }, + time_us: { type: "integer" }, + }, + }; + if (rd.groupBy && Object.keys(rd.groups).length > 0) { + const groupDefName = `hydrate${cap(rd.relName)}`; + const groupProperties: Record = {}; + for (const shortName of Object.keys(rd.groups)) { + groupProperties[shortName] = { type: "array", items: { type: "ref", ref: `#${recordDefName}` } }; + } + groupProperties["other"] = { type: "array", items: { type: "ref", ref: `#${recordDefName}` } }; + defs[groupDefName] = { type: "object", properties: groupProperties }; + } + } + return defs; + } + + function buildReferenceDefs(referenceDefs: ReferenceDef[]): Record { + const defs: Record = {}; + const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + for (const rd of referenceDefs) { + const refCollectionRef = getCollectionLexiconRef(rd.collection); + const recordDefName = `ref${cap(rd.refName)}Record`; + defs[recordDefName] = { + type: "object", + required: ["uri", "did", "collection", "rkey", "time_us"], + properties: { + uri: { type: "string", format: "at-uri" }, + did: { type: "string", format: "did" }, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + cid: { type: "string" }, + record: refCollectionRef ? { type: "ref", ref: refCollectionRef } : { type: "unknown" }, + time_us: { type: "integer" }, + }, + }; + } + return defs; + } + + function profileDefs() { + const profiles = config.profiles ?? ["app.bsky.actor.profile"]; + const extraDefs: Record = {}; + const objectRefs: string[] = []; + for (const col of profiles) { + const schema = getRecordObjectSchema(col); + if (!schema) continue; + const defName = col.split(".").map((p, i) => i === 0 ? p : p.charAt(0).toUpperCase() + p.slice(1)).join(""); + extraDefs[defName] = schema; + objectRefs.push(`#${defName}`); + } + let recordField: any; + if (objectRefs.length === 1) recordField = { type: "ref", ref: objectRefs[0] }; + else if (objectRefs.length > 1) recordField = { type: "union", refs: objectRefs }; + else recordField = { type: "unknown" }; + return { + profileEntry: { + type: "object", + required: ["did"], + properties: { + did: { type: "string", format: "did" }, + handle: { type: "string" }, + uri: { type: "string", format: "at-uri" }, + collection: { type: "string", format: "nsid" }, + rkey: { type: "string" }, + cid: { type: "string" }, + record: recordField, + }, + }, + ...extraDefs, + }; + } + + function tokenShortName(token: string): string { + const hash = token.indexOf("#"); + return hash !== -1 ? token.slice(hash + 1) : token; + } + + // --- Generate --- + + const ns = config.namespace; + + log("Generating admin endpoints..."); + + writeLexicon(`${ns}.admin.getCursor`, { + lexicon: 1, id: `${ns}.admin.getCursor`, + defs: { main: { type: "query", description: "Get the current cursor position", output: { encoding: "application/json", schema: { type: "object", properties: { time_us: { type: "integer" }, date: { type: "string" }, seconds_ago: { type: "integer" } } } } } }, + }); + + writeLexicon(`${ns}.admin.getOverview`, { + lexicon: 1, id: `${ns}.admin.getOverview`, + defs: { main: { type: "query", description: "Get an overview of all indexed collections", output: { encoding: "application/json", schema: { type: "object", required: ["total_records", "collections"], properties: { total_records: { type: "integer" }, collections: { type: "array", items: { type: "ref", ref: "#collectionStats" } } } } } }, collectionStats: { type: "object", required: ["collection", "records", "unique_users"], properties: { collection: { type: "string" }, records: { type: "integer" }, unique_users: { type: "integer" } } } }, + }); + + writeLexicon(`${ns}.admin.sync`, { + lexicon: 1, id: `${ns}.admin.sync`, + defs: { main: { type: "query", description: "Discover users from relays and backfill their records from PDS", parameters: { type: "params", properties: { concurrency: { type: "integer", minimum: 1, maximum: 50, default: 10 } } }, output: { encoding: "application/json", schema: { type: "object", required: ["discovered", "backfilled", "remaining", "done"], properties: { discovered: { type: "integer" }, backfilled: { type: "integer" }, remaining: { type: "integer" }, done: { type: "boolean" } } } } } }, + }); + + writeLexicon(`${ns}.admin.reset`, { + lexicon: 1, id: `${ns}.admin.reset`, + defs: { main: { type: "query", description: "Delete all data from all tables", output: { encoding: "application/json", schema: { type: "object", required: ["ok"], properties: { ok: { type: "boolean" } } } } } }, + }); + + writeLexicon(`${ns}.getProfile`, { + lexicon: 1, id: `${ns}.getProfile`, + defs: { main: { type: "query", description: "Get a user's profile by DID or handle", parameters: { type: "params", required: ["actor"], properties: { actor: { type: "string", format: "at-identifier", description: "DID or handle of the user" } } }, output: { encoding: "application/json", schema: { type: "ref", ref: "#profileEntry" } } }, ...profileDefs() }, + }); + + writeLexicon(`${ns}.notifyOfUpdate`, { + lexicon: 1, id: `${ns}.notifyOfUpdate`, + defs: { main: { type: "procedure", description: "Notify of a record change for immediate indexing. Fetches the record from the user's PDS and indexes (or deletes) it.", input: { encoding: "application/json", schema: { type: "object", properties: { uri: { type: "string", format: "at-uri", description: "Single AT URI to fetch and index" }, uris: { type: "array", items: { type: "string", format: "at-uri" }, maxLength: 25, description: "Batch of AT URIs to fetch and index (max 25)" } } } }, output: { encoding: "application/json", schema: { type: "object", required: ["indexed", "deleted"], properties: { indexed: { type: "integer", description: "Number of records created or updated" }, deleted: { type: "integer", description: "Number of records deleted (not found on PDS)" }, errors: { type: "array", items: { type: "string" }, description: "Errors for individual URIs that could not be processed" } } } } } }, + }); + + // --- Per-collection --- + + log("Generating collection endpoints..."); + + const resolvedQueryableMap: Record> = {}; + const resolvedRelationsMap: Record }>> = {}; + + for (const [collection, colConfig] of Object.entries(config.collections)) { + const collectionRef = getCollectionLexiconRef(collection); + + const autoDetected = detectQueryableFields(collection); + const manual = colConfig.queryable ?? {}; + const merged = { ...autoDetected, ...manual }; + resolvedQueryableMap[collection] = merged; + + // --- listRecords --- + const listParams: Record = { + limit: { type: "integer", minimum: 1, maximum: 100, default: 50 }, + cursor: { type: "string" }, + actor: { type: "string", format: "at-identifier", description: "Filter by DID or handle (triggers on-demand backfill)" }, + profiles: { type: "boolean", description: "Include profile + identity info keyed by DID" }, + }; + + // Search param + if (colConfig.searchable !== false) { + const allQueryable = { ...autoDetected, ...manual }; + const searchableFields = Array.isArray(colConfig.searchable) + ? colConfig.searchable + : Object.entries(allQueryable).filter(([, f]) => f.type !== "range").map(([name]) => name); + if (searchableFields.length > 0) { + listParams["search"] = { + type: "string", + description: `Full-text search across: ${searchableFields.join(", ")}`, + }; + } + } + + for (const [field, fieldConfig] of Object.entries(merged)) { + const param = fieldToParam(field); + if (fieldConfig.type === "range") { + listParams[`${param}Min`] = { type: "string", description: `Minimum value for ${field}` }; + listParams[`${param}Max`] = { type: "string", description: `Maximum value for ${field}` }; + } else { + listParams[param] = { type: "string", description: `Filter by ${field}` }; + } + } + + const countFields: CountField[] = []; + const relationDefs: RelationDef[] = []; + const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + + for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { + countFields.push({ name: `${relName}Count`, description: `Total ${relName} count` }); + listParams[`${relName}CountMin`] = { type: "integer", description: `Minimum total ${relName} count` }; + listParams[`hydrate${cap(relName)}`] = { type: "integer", minimum: 1, maximum: 50, description: `Number of ${relName} records to embed per record` }; + + const groupMapping: Record = {}; + if (rel.groupBy) { + const knownValues = getKnownValues(rel.collection, rel.groupBy); + for (const token of knownValues) { + const shortName = tokenShortName(token); + groupMapping[shortName] = token; + countFields.push({ name: `${relName}${cap(shortName)}Count`, description: `${relName} count where ${rel.groupBy} = ${shortName}` }); + listParams[`${relName}${cap(shortName)}CountMin`] = { type: "integer", description: `Minimum ${relName} count where ${rel.groupBy} = ${shortName}` }; + } + if (!resolvedRelationsMap[collection]) resolvedRelationsMap[collection] = {}; + resolvedRelationsMap[collection][relName] = { collection: rel.collection, groupBy: rel.groupBy, groups: groupMapping }; + } + + relationDefs.push({ relName, collection: rel.collection, groupBy: rel.groupBy, groups: groupMapping }); + } + + const referenceDefs: ReferenceDef[] = []; + for (const [refName, ref] of Object.entries(colConfig.references ?? {})) { + referenceDefs.push({ refName, collection: ref.collection }); + } + for (const refName of Object.keys(colConfig.references ?? {})) { + listParams[`hydrate${cap(refName)}`] = { type: "boolean", description: `Embed the referenced ${refName} record` }; + } + + const sortableValues: string[] = []; + for (const field of Object.keys(merged)) sortableValues.push(fieldToParam(field)); + for (const cf of countFields) sortableValues.push(cf.name); + if (sortableValues.length > 0) { + listParams["sort"] = { type: "string", knownValues: sortableValues, description: "Field to sort by (default: time_us)" }; + listParams["order"] = { type: "string", knownValues: ["asc", "desc"], description: "Sort direction (default: desc for dates/numbers/counts, asc for strings)" }; + } + + const hydrateDefs = buildHydrateDefs(relationDefs); + const refDefs = buildReferenceDefs(referenceDefs); + + writeLexicon(`${collection}.listRecords`, { + lexicon: 1, id: `${collection}.listRecords`, + defs: { + main: { type: "query", description: `Query ${collection} records with filters`, parameters: { type: "params", properties: listParams }, output: { encoding: "application/json", schema: { type: "object", required: ["records"], properties: { records: { type: "array", items: { type: "ref", ref: "#record" } }, cursor: { type: "string" }, profiles: { type: "array", items: { type: "ref", ref: "#profileEntry" } } } } } }, + record: buildRecordDef(collectionRef, countFields, relationDefs, referenceDefs), + ...hydrateDefs, ...refDefs, ...profileDefs(), + }, + }); + + // --- getRecord --- + const getParams: Record = { + uri: { type: "string", format: "at-uri", description: "AT URI of the record" }, + profiles: { type: "boolean", description: "Include profile + identity info keyed by DID" }, + }; + for (const rd of relationDefs) { + getParams[`hydrate${cap(rd.relName)}`] = { type: "integer", minimum: 1, maximum: 50, description: `Number of ${rd.relName} records to embed` }; + } + for (const refName of Object.keys(colConfig.references ?? {})) { + getParams[`hydrate${cap(refName)}`] = { type: "boolean", description: `Embed the referenced ${refName} record` }; + } + + writeLexicon(`${collection}.getRecord`, { + lexicon: 1, id: `${collection}.getRecord`, + defs: { + main: { type: "query", description: `Get a single ${collection} record by AT URI`, parameters: { type: "params", required: ["uri"], properties: getParams }, output: { encoding: "application/json", schema: { type: "object", required: ["uri", "did", "collection", "rkey", "time_us"], properties: { ...buildRecordDef(collectionRef, countFields, relationDefs, referenceDefs).properties, profiles: { type: "array", items: { type: "ref", ref: "#profileEntry" } } } } } }, + ...hydrateDefs, ...refDefs, ...profileDefs(), + }, + }); + + for (const queryName of Object.keys(colConfig.queries ?? {})) { + writeLexicon(`${collection}.${queryName}`, { + lexicon: 1, id: `${collection}.${queryName}`, + defs: { main: { type: "query", description: `Custom query: ${queryName}`, output: { encoding: "application/json", schema: { type: "object", properties: {} } } } }, + }); + } + } + + // --- Runtime files (only when called from script) --- + if (options.writeRuntimeFiles) { + // lex.config.js + const collectionNsids = Object.keys(config.collections); + const pulledFiles = [...scanLexiconsDir(lexiconDirs), ...scanLexiconsDir([])].flat(); + const allRefs = new Set(); + for (const file of pulledFiles) { + for (const ref of findRefsInLexicon(file)) allRefs.add(ref); + } + const profileNsids = config.profiles ?? ["app.bsky.actor.profile"]; + const pullNsids = new Set([...collectionNsids, ...profileNsids]); + for (const ref of allRefs) { + if (!ref.startsWith("com.atproto.")) pullNsids.add(ref); + } + const sortedNsids = [...pullNsids].sort(); + const lexConfigContent = `import { defineLexiconConfig } from "@atcute/lex-cli";\n\nexport default defineLexiconConfig({\n files: ["lexicons/**/*.json", "lexicons-pulled/**/*.json", "lexicons-generated/**/*.json"],\n outdir: "src/lexicon-types/",\n imports: ["@atcute/atproto"],\n pull: {\n outdir: "lexicons-pulled/",\n sources: [\n {\n type: "atproto",\n mode: "nsids",\n nsids: ${JSON.stringify(sortedNsids, null, 10).replace(/^/gm, " ").trim()},\n },\n ],\n },\n});\n`; + writeFileSync(join(rootDir, "lex.config.js"), lexConfigContent); + log(`\nGenerated lex.config.js with ${sortedNsids.length} pull NSIDs`); + + // queryable.generated.ts + const queryableContent = `// Auto-generated — do not edit. Run \`pnpm generate\` to regenerate.\nimport type { QueryableField } from "./types";\n\nexport const resolvedQueryable: Record> = ${JSON.stringify(resolvedQueryableMap, null, 2)};\n\nexport interface ResolvedRelation {\n collection: string;\n groupBy: string;\n groups: Record; // shortName → full token value\n}\n\nexport const resolvedRelationsMap: Record> = ${JSON.stringify(resolvedRelationsMap, null, 2)};\n`; + writeFileSync(join(rootDir, "src", "core", "queryable.generated.ts"), queryableContent); + log("Generated src/core/queryable.generated.ts"); + } + + log("\nDone!"); + return generated; +} + +// --- Helpers used by writeRuntimeFiles --- + +function analyzeProperties( + defs: Record, + properties: Record, + prefix: string +): Record { + const result: Record = {}; + for (const [field, def] of Object.entries(properties)) { + const path = prefix ? `${prefix}.${field}` : field; + if (def.type === "string") { + if (def.format === "datetime") result[path] = { type: "range" }; + else if (def.format !== "uri" && def.format !== "at-uri") result[path] = {}; + } else if (def.type === "integer" || def.type === "number") { + result[path] = { type: "range" }; + } else if (def.type === "ref" && def.ref === "com.atproto.repo.strongRef") { + result[`${path}.uri`] = {}; + } else if (def.type === "union" && Array.isArray(def.refs) && def.refs.includes("com.atproto.repo.strongRef")) { + result[`${path}.uri`] = {}; + } else if (def.type === "ref" && def.ref) { + const refId = def.ref.includes("#") ? def.ref.split("#")[1] : null; + if (refId && defs[refId]?.type === "string") result[path] = {}; + } + } + return result; +} + +function scanLexiconsDir(dirs: string[]): string[] { + const files: string[] = []; + for (const dir of dirs) { + if (!existsSync(dir)) continue; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) files.push(...scanLexiconsDir([fullPath])); + else if (entry.name.endsWith(".json")) files.push(fullPath); + } + } + return files; +} + +function findRefsInLexicon(filePath: string): string[] { + try { + const content = readFileSync(filePath, "utf-8"); + const refs: string[] = []; + const refPattern = /"ref":\s*"([a-z][a-zA-Z0-9]*(?:\.[a-zA-Z0-9]+)+)(?:#\w+)?"/g; + let match; + while ((match = refPattern.exec(content)) !== null) refs.push(match[1]); + const refsArrayPattern = /"refs":\s*\[([^\]]+)\]/g; + while ((match = refsArrayPattern.exec(content)) !== null) { + const inner = match[1]; + const nsidPattern = /"([a-z][a-zA-Z0-9]*(?:\.[a-zA-Z0-9]+)+)(?:#\w+)?"/g; + let innerMatch; + while ((innerMatch = nsidPattern.exec(inner)) !== null) refs.push(innerMatch[1]); + } + return refs; + } catch { + return []; + } +} diff --git a/tests/generate.test.ts b/tests/generate.test.ts new file mode 100644 index 0000000..c01d067 --- /dev/null +++ b/tests/generate.test.ts @@ -0,0 +1,265 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { join } from "path"; +import { generateLexicons } from "../src/generate"; +import type { ContrailConfig } from "../src/core/types"; + +const ROOT_DIR = join(__dirname, ".."); + +function getParams(lexicon: any): Record { + return lexicon?.defs?.main?.parameters?.properties ?? {}; +} + +function getInputSchema(lexicon: any): any { + return lexicon?.defs?.main?.input?.schema; +} + +function getOutputSchema(lexicon: any): any { + return lexicon?.defs?.main?.output?.schema; +} + +function generate(config: ContrailConfig) { + return generateLexicons({ + config, + rootDir: ROOT_DIR, + lexiconDirs: [], + quiet: true, + }); +} + +// --- Test configs --- + +const BASIC_CONFIG: ContrailConfig = { + namespace: "test.app", + collections: { + "com.example.post": { + queryable: { + title: {}, + body: {}, + createdAt: { type: "range" }, + }, + }, + }, +}; + +const RELATIONS_CONFIG: ContrailConfig = { + namespace: "test.app", + collections: { + "com.example.post": { + queryable: { title: {} }, + relations: { + likes: { + collection: "com.example.like", + }, + }, + }, + "com.example.like": { + queryable: { status: {} }, + references: { + post: { + collection: "com.example.post", + field: "subject.uri", + }, + }, + }, + }, +}; + +const SEARCH_EXPLICIT_CONFIG: ContrailConfig = { + namespace: "test.app", + collections: { + "com.example.post": { + queryable: { + title: {}, + body: {}, + category: {}, + createdAt: { type: "range" }, + }, + searchable: ["title", "body"], + }, + }, +}; + +const SEARCH_DISABLED_CONFIG: ContrailConfig = { + namespace: "test.app", + collections: { + "com.example.post": { + queryable: { title: {}, body: {} }, + searchable: false, + }, + }, +}; + +const SEARCH_AUTO_CONFIG: ContrailConfig = { + namespace: "test.app", + collections: { + "com.example.post": { + queryable: { + title: {}, + body: {}, + score: { type: "range" }, + }, + }, + }, +}; + +describe("basic generation", () => { + let lexicons: Record; + + beforeAll(() => { + lexicons = generate(BASIC_CONFIG); + }); + + it("generates admin endpoints", () => { + expect(lexicons["test.app.admin.getCursor"]).toBeDefined(); + expect(lexicons["test.app.admin.getOverview"]).toBeDefined(); + expect(lexicons["test.app.admin.sync"]).toBeDefined(); + expect(lexicons["test.app.admin.reset"]).toBeDefined(); + }); + + it("generates getProfile", () => { + const lex = lexicons["test.app.getProfile"]; + expect(lex).toBeDefined(); + const params = getParams(lex); + expect(params.actor).toBeDefined(); + expect(params.actor.format).toBe("at-identifier"); + }); + + it("generates notifyOfUpdate as a procedure", () => { + const lex = lexicons["test.app.notifyOfUpdate"]; + expect(lex).toBeDefined(); + expect(lex.defs.main.type).toBe("procedure"); + + const input = getInputSchema(lex); + expect(input.properties.uri).toBeDefined(); + expect(input.properties.uri.format).toBe("at-uri"); + expect(input.properties.uris.type).toBe("array"); + expect(input.properties.uris.maxLength).toBe(25); + + const output = getOutputSchema(lex); + expect(output.required).toContain("indexed"); + expect(output.required).toContain("deleted"); + expect(output.properties.errors.type).toBe("array"); + }); + + it("generates listRecords with standard params", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.limit).toBeDefined(); + expect(params.cursor).toBeDefined(); + expect(params.actor).toBeDefined(); + expect(params.profiles).toBeDefined(); + }); + + it("generates queryable field params", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.title).toBeDefined(); + expect(params.title.type).toBe("string"); + expect(params.body).toBeDefined(); + expect(params.createdAtMin).toBeDefined(); + expect(params.createdAtMax).toBeDefined(); + }); + + it("generates sort and order params", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.sort).toBeDefined(); + expect(params.sort.knownValues).toContain("title"); + expect(params.sort.knownValues).toContain("body"); + expect(params.sort.knownValues).toContain("createdAt"); + expect(params.order.knownValues).toEqual(["asc", "desc"]); + }); + + it("generates getRecord with uri param", () => { + const params = getParams(lexicons["com.example.post.getRecord"]); + expect(params.uri).toBeDefined(); + expect(params.uri.format).toBe("at-uri"); + }); + + it("does not include search on getRecord", () => { + const params = getParams(lexicons["com.example.post.getRecord"]); + expect(params.search).toBeUndefined(); + }); +}); + +describe("relations and references", () => { + let lexicons: Record; + + beforeAll(() => { + lexicons = generate(RELATIONS_CONFIG); + }); + + it("generates count filter params for relations", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.likesCountMin).toBeDefined(); + expect(params.likesCountMin.type).toBe("integer"); + }); + + it("generates hydrate params for relations", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.hydrateLikes).toBeDefined(); + expect(params.hydrateLikes.type).toBe("integer"); + }); + + it("generates hydrate params for references", () => { + const params = getParams(lexicons["com.example.like.listRecords"]); + expect(params.hydratePost).toBeDefined(); + expect(params.hydratePost.type).toBe("boolean"); + }); + + it("includes count fields in record def", () => { + const recordDef = lexicons["com.example.post.listRecords"].defs.record; + expect(recordDef.properties.likesCount).toBeDefined(); + expect(recordDef.properties.likesCount.type).toBe("integer"); + }); + + it("includes relation shape in record def (ungrouped → array)", () => { + const recordDef = lexicons["com.example.post.listRecords"].defs.record; + expect(recordDef.properties.likes).toBeDefined(); + expect(recordDef.properties.likes.type).toBe("array"); + }); + + it("includes reference shape in record def", () => { + const recordDef = lexicons["com.example.like.listRecords"].defs.record; + expect(recordDef.properties.post).toBeDefined(); + expect(recordDef.properties.post.type).toBe("ref"); + }); + + it("sort knownValues includes count fields", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.sort.knownValues).toContain("likesCount"); + }); +}); + +describe("search: explicit fields", () => { + let lexicons: Record; + + beforeAll(() => { + lexicons = generate(SEARCH_EXPLICIT_CONFIG); + }); + + it("includes search param listing only explicit fields", () => { + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.search).toBeDefined(); + expect(params.search.description).toContain("title"); + expect(params.search.description).toContain("body"); + expect(params.search.description).not.toContain("category"); + expect(params.search.description).not.toContain("createdAt"); + }); +}); + +describe("search: disabled", () => { + it("does not include search param", () => { + const lexicons = generate(SEARCH_DISABLED_CONFIG); + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.search).toBeUndefined(); + }); +}); + +describe("search: auto-detect", () => { + it("includes search param with non-range fields only", () => { + const lexicons = generate(SEARCH_AUTO_CONFIG); + const params = getParams(lexicons["com.example.post.listRecords"]); + expect(params.search).toBeDefined(); + expect(params.search.description).toContain("title"); + expect(params.search.description).toContain("body"); + expect(params.search.description).not.toContain("score"); + }); +}); diff --git a/tests/notify.test.ts b/tests/notify.test.ts new file mode 100644 index 0000000..3263133 --- /dev/null +++ b/tests/notify.test.ts @@ -0,0 +1,471 @@ +import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; +import type { Database } from "../src/core/types"; +import { createTestDbWithSchema, makeEvent, TEST_CONFIG } from "./helpers"; +import { parseAtUri } from "../src/core/router/notify"; +import { createApp } from "../src/core/router/index"; +import { applyEvents, queryRecords } from "../src/core/db/records"; +import type { Hono } from "hono"; + +let db: Database; +let app: Hono; + +beforeEach(async () => { + db = await createTestDbWithSchema(); + app = createApp(db, TEST_CONFIG); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("parseAtUri", () => { + it("parses a valid AT URI", () => { + const result = parseAtUri( + "at://did:plc:abc123/community.lexicon.calendar.event/rkey1" + ); + expect(result).toEqual({ + did: "did:plc:abc123", + collection: "community.lexicon.calendar.event", + rkey: "rkey1", + }); + }); + + it("parses did:web URIs", () => { + const result = parseAtUri( + "at://did:web:example.com/app.bsky.feed.post/abc" + ); + expect(result).toEqual({ + did: "did:web:example.com", + collection: "app.bsky.feed.post", + rkey: "abc", + }); + }); + + it("returns null for invalid URIs", () => { + expect(parseAtUri("")).toBeNull(); + expect(parseAtUri("https://example.com")).toBeNull(); + expect(parseAtUri("at://did:plc:abc")).toBeNull(); // missing collection and rkey + expect(parseAtUri("at://did:plc:abc/collection")).toBeNull(); // missing rkey + expect(parseAtUri("at://did:plc:abc/collection/rkey/extra")).toBeNull(); // too many segments + }); +}); + +describe("POST notifyOfUpdate", () => { + const endpoint = `/xrpc/${TEST_CONFIG.namespace}.notifyOfUpdate`; + + function mockFetch( + records: Record + ) { + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + const u = new URL(url); + const repo = u.searchParams.get("repo"); + const collection = u.searchParams.get("collection"); + const rkey = u.searchParams.get("rkey"); + const uri = `at://${repo}/${collection}/${rkey}`; + + if (records[uri]) { + return new Response(JSON.stringify(records[uri]), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response("not found", { status: 404 }); + }) + ); + } + + /** Seed the identities table so getPDS resolves without hitting slingshot. */ + async function seedIdentity(did: string, pds: string) { + await db + .prepare( + "INSERT OR REPLACE INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?)" + ) + .bind(did, "test.handle", pds, Date.now()) + .run(); + } + + it("returns 400 when no uri provided", async () => { + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/uri/); + }); + + it("returns 400 when too many URIs", async () => { + const uris = Array.from({ length: 26 }, (_, i) => + `at://did:plc:test/community.lexicon.calendar.event/r${i}` + ); + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uris }), + }); + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatch(/max 25/); + }); + + it("reports error for invalid AT URI", async () => { + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri: "not-a-valid-uri" }), + }); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.errors).toContain("invalid AT URI: not-a-valid-uri"); + expect(body.indexed).toBe(0); + }); + + it("reports error for untracked collection", async () => { + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + uri: "at://did:plc:test/app.bsky.feed.post/abc", + }), + }); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.errors).toContain("collection not tracked: app.bsky.feed.post"); + }); + + it("fetches and indexes a record from PDS", async () => { + const did = "did:plc:test"; + const uri = `at://${did}/community.lexicon.calendar.event/evt1`; + const record = { name: "Test Event", startsAt: "2026-04-01T10:00:00Z" }; + + await seedIdentity(did, "https://pds.example.com"); + mockFetch({ [uri]: { value: record, cid: "bafytest" } }); + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri }), + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.indexed).toBe(1); + expect(body.deleted).toBe(0); + expect(body.errors).toBeUndefined(); + + // Verify the record is in the database + const result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records).toHaveLength(1); + expect(result.records[0].uri).toBe(uri); + expect(result.records[0].cid).toBe("bafytest"); + expect(JSON.parse(result.records[0].record!)).toEqual(record); + }); + + it("deletes locally when record not found on PDS", async () => { + const did = "did:plc:test"; + const uri = `at://${did}/community.lexicon.calendar.event/evt1`; + + // Pre-populate a record + await applyEvents(db, [ + makeEvent({ uri, did, rkey: "evt1", record: { name: "Old" } }), + ]); + + await seedIdentity(did, "https://pds.example.com"); + mockFetch({}); // PDS returns 404 for everything + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri }), + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.indexed).toBe(0); + expect(body.deleted).toBe(1); + + // Verify the record is gone + const result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records).toHaveLength(0); + }); + + it("handles batch of URIs", async () => { + const did = "did:plc:test"; + const uri1 = `at://${did}/community.lexicon.calendar.event/e1`; + const uri2 = `at://${did}/community.lexicon.calendar.event/e2`; + + await seedIdentity(did, "https://pds.example.com"); + mockFetch({ + [uri1]: { value: { name: "Event 1" }, cid: "cid1" }, + [uri2]: { value: { name: "Event 2" }, cid: "cid2" }, + }); + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uris: [uri1, uri2] }), + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.indexed).toBe(2); + + const result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records).toHaveLength(2); + }); + + it("updates an existing record (upsert)", async () => { + const did = "did:plc:test"; + const uri = `at://${did}/community.lexicon.calendar.event/evt1`; + + // Insert original + await applyEvents(db, [ + makeEvent({ uri, did, rkey: "evt1", record: { name: "V1" } }), + ]); + + await seedIdentity(did, "https://pds.example.com"); + mockFetch({ + [uri]: { value: { name: "V2" }, cid: "newcid" }, + }); + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri }), + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.indexed).toBe(1); + + const result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records).toHaveLength(1); + expect(JSON.parse(result.records[0].record!).name).toBe("V2"); + expect(result.records[0].cid).toBe("newcid"); + }); + + it("updates counts when notifying about a relation record", async () => { + const did = "did:plc:test"; + const eventUri = `at://${did}/community.lexicon.calendar.event/evt1`; + const rsvpUri = `at://${did}/community.lexicon.calendar.rsvp/r1`; + + // Insert parent event + await applyEvents(db, [ + makeEvent({ uri: eventUri, did, rkey: "evt1", record: { name: "Event" } }), + ]); + + await seedIdentity(did, "https://pds.example.com"); + mockFetch({ + [rsvpUri]: { + value: { subject: { uri: eventUri }, status: "going" }, + cid: "rsvpcid", + }, + }); + + await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri: rsvpUri }), + }); + + // Check that the event now has an RSVP count + const result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records).toHaveLength(1); + expect( + result.records[0].counts?.["community.lexicon.calendar.rsvp"] + ).toBe(1); + }); + + it("skips when record already exists with same CID (no double-counting)", async () => { + const did = "did:plc:test"; + const eventUri = `at://${did}/community.lexicon.calendar.event/evt1`; + const rsvpUri = `at://${did}/community.lexicon.calendar.rsvp/r1`; + const rsvpRecord = { subject: { uri: eventUri }, status: "going" }; + + // Insert parent event and RSVP via normal ingestion + await applyEvents(db, [ + makeEvent({ uri: eventUri, did, rkey: "evt1", record: { name: "Event" } }), + ]); + await applyEvents( + db, + [ + makeEvent({ + uri: rsvpUri, + did, + collection: "community.lexicon.calendar.rsvp", + rkey: "r1", + cid: "rsvpcid", + record: rsvpRecord, + time_us: 2000000, + }), + ], + TEST_CONFIG + ); + + // Verify count is 1 + let result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(1); + + // Now notify with the same RSVP (same CID) — should be a no-op + await seedIdentity(did, "https://pds.example.com"); + mockFetch({ + [rsvpUri]: { value: rsvpRecord, cid: "rsvpcid" }, + }); + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri: rsvpUri }), + }); + + const body = await res.json(); + expect(body.indexed).toBe(0); // skipped, nothing changed + expect(body.deleted).toBe(0); + + // Count should still be 1, not 2 + result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(1); + }); + + it("uses update (not create) when record exists with different CID", async () => { + const did = "did:plc:test"; + const eventUri = `at://${did}/community.lexicon.calendar.event/evt1`; + const rsvpUri = `at://${did}/community.lexicon.calendar.rsvp/r1`; + + // Insert parent event and RSVP via normal ingestion + await applyEvents(db, [ + makeEvent({ uri: eventUri, did, rkey: "evt1", record: { name: "Event" } }), + ]); + await applyEvents( + db, + [ + makeEvent({ + uri: rsvpUri, + did, + collection: "community.lexicon.calendar.rsvp", + rkey: "r1", + cid: "oldcid", + record: { subject: { uri: eventUri }, status: "going" }, + time_us: 2000000, + }), + ], + TEST_CONFIG + ); + + // Now notify with updated record (different CID) + await seedIdentity(did, "https://pds.example.com"); + mockFetch({ + [rsvpUri]: { + value: { subject: { uri: eventUri }, status: "interested" }, + cid: "newcid", + }, + }); + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri: rsvpUri }), + }); + + const body = await res.json(); + expect(body.indexed).toBe(1); + + // Record should be updated + const rsvpResult = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.rsvp", + }); + expect(rsvpResult.records).toHaveLength(1); + expect(rsvpResult.records[0].cid).toBe("newcid"); + + // Count should still be 1 (not double-counted) + const result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(1); + }); + + it("does nothing when record not on PDS and not local", async () => { + const did = "did:plc:test"; + const uri = `at://${did}/community.lexicon.calendar.event/nonexistent`; + + await seedIdentity(did, "https://pds.example.com"); + mockFetch({}); // 404 for everything + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri }), + }); + + const body = await res.json(); + expect(body.indexed).toBe(0); + expect(body.deleted).toBe(0); + }); + + it("decrements counts when deleting a relation record", async () => { + const did = "did:plc:test"; + const eventUri = `at://${did}/community.lexicon.calendar.event/evt1`; + const rsvpUri = `at://${did}/community.lexicon.calendar.rsvp/r1`; + + // Insert event + RSVP + await applyEvents(db, [ + makeEvent({ uri: eventUri, did, rkey: "evt1", record: { name: "Event" } }), + ]); + await applyEvents( + db, + [ + makeEvent({ + uri: rsvpUri, + did, + collection: "community.lexicon.calendar.rsvp", + rkey: "r1", + record: { subject: { uri: eventUri }, status: "going" }, + time_us: 2000000, + }), + ], + TEST_CONFIG + ); + + // Verify count is 1 + let result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"]).toBe(1); + + // Notify with RSVP URI — PDS returns 404 (deleted) + await seedIdentity(did, "https://pds.example.com"); + mockFetch({}); + + const res = await app.request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ uri: rsvpUri }), + }); + + const body = await res.json(); + expect(body.deleted).toBe(1); + + // Count should be back to 0 + result = await queryRecords(db, TEST_CONFIG, { + collection: "community.lexicon.calendar.event", + }); + expect(result.records[0].counts?.["community.lexicon.calendar.rsvp"] ?? 0).toBe(0); + }); +}); diff --git a/tests/search.test.ts b/tests/search.test.ts new file mode 100644 index 0000000..0edafa4 --- /dev/null +++ b/tests/search.test.ts @@ -0,0 +1,321 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import type { Database, ContrailConfig } from "../src/core/types"; +import { createTestDb, makeEvent } from "./helpers"; +import { initSchema } from "../src/core/db/schema"; +import { applyEvents, queryRecords } from "../src/core/db/records"; + +const SEARCH_CONFIG: ContrailConfig = { + namespace: "com.example", + collections: { + "community.lexicon.calendar.event": { + queryable: { + mode: {}, + name: {}, + description: {}, + startsAt: { type: "range" }, + }, + // searchable omitted → auto-detect non-range: mode, name, description + }, + "test.explicit.collection": { + queryable: { + title: {}, + body: {}, + category: {}, + }, + searchable: ["title", "body"], // explicit: only title and body + }, + "test.disabled.collection": { + queryable: { + name: {}, + }, + searchable: false, // disabled + }, + }, +}; + +let db: Database; + +beforeEach(async () => { + db = createTestDb(); + await initSchema(db, SEARCH_CONFIG); +}); + +describe("FTS auto-detect (searchable omitted)", () => { + const collection = "community.lexicon.calendar.event"; + + beforeEach(async () => { + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/community.lexicon.calendar.event/1", + did: "did:plc:a", + collection, + rkey: "1", + record: { name: "Rust Meetup", mode: "in-person", description: "A gathering of Rustaceans" }, + time_us: 3000, + }), + makeEvent({ + uri: "at://did:plc:b/community.lexicon.calendar.event/2", + did: "did:plc:b", + collection, + rkey: "2", + record: { name: "TypeScript Workshop", mode: "online", description: "Learn advanced TypeScript" }, + time_us: 2000, + }), + makeEvent({ + uri: "at://did:plc:c/community.lexicon.calendar.event/3", + did: "did:plc:c", + collection, + rkey: "3", + record: { name: "Rust and TypeScript", mode: "hybrid", description: "Best of both worlds" }, + time_us: 1000, + }), + ], + SEARCH_CONFIG + ); + }); + + it("finds records matching a search term", async () => { + const result = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Rust", + }); + expect(result.records).toHaveLength(2); + const names = result.records.map((r) => JSON.parse(r.record!).name); + expect(names).toContain("Rust Meetup"); + expect(names).toContain("Rust and TypeScript"); + }); + + it("searches across multiple fields", async () => { + const result = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Rustaceans", + }); + expect(result.records).toHaveLength(1); + expect(JSON.parse(result.records[0].record!).name).toBe("Rust Meetup"); + }); + + it("returns nothing for non-matching search", async () => { + const result = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Python", + }); + expect(result.records).toHaveLength(0); + }); + + it("supports prefix search", async () => { + const result = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Type*", + }); + expect(result.records).toHaveLength(2); + }); + + it("combines search with filters", async () => { + const result = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Rust", + filters: { mode: "in-person" }, + }); + expect(result.records).toHaveLength(1); + expect(JSON.parse(result.records[0].record!).name).toBe("Rust Meetup"); + }); + + it("does not search range fields (startsAt)", async () => { + // startsAt is range, so not included in FTS. Searching for its value should not match. + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:d/community.lexicon.calendar.event/4", + did: "did:plc:d", + collection, + rkey: "4", + record: { name: "Date Event", startsAt: "2026-04-01T10:00:00Z", mode: "online", description: "Nothing special" }, + time_us: 500, + }), + ], + SEARCH_CONFIG + ); + // "T10" would appear in the startsAt value but not in any searchable field + const result = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "T10", + }); + expect(result.records).toHaveLength(0); + }); +}); + +describe("FTS sync", () => { + const collection = "community.lexicon.calendar.event"; + + it("updates FTS on record update", async () => { + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/community.lexicon.calendar.event/1", + collection, + rkey: "1", + record: { name: "Old Name", mode: "online", description: "test" }, + time_us: 1000, + }), + ], + SEARCH_CONFIG + ); + + // Update the record + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/community.lexicon.calendar.event/1", + collection, + rkey: "1", + record: { name: "New Name", mode: "online", description: "test" }, + operation: "update", + time_us: 2000, + }), + ], + SEARCH_CONFIG + ); + + const oldResult = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Old" }); + expect(oldResult.records).toHaveLength(0); + + const newResult = await queryRecords(db, SEARCH_CONFIG, { collection, search: "New" }); + expect(newResult.records).toHaveLength(1); + }); + + it("removes from FTS on delete", async () => { + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/community.lexicon.calendar.event/1", + collection, + rkey: "1", + record: { name: "Deletable", mode: "online", description: "test" }, + time_us: 1000, + }), + ], + SEARCH_CONFIG + ); + + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/community.lexicon.calendar.event/1", + collection, + rkey: "1", + operation: "delete", + record: { name: "Deletable", mode: "online", description: "test" }, + time_us: 2000, + }), + ], + SEARCH_CONFIG + ); + + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Deletable" }); + expect(result.records).toHaveLength(0); + }); +}); + +describe("explicit searchable fields", () => { + const collection = "test.explicit.collection"; + + beforeEach(async () => { + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/test.explicit.collection/1", + did: "did:plc:a", + collection, + rkey: "1", + record: { title: "Interesting Article", body: "Some content here", category: "tech" }, + time_us: 1000, + }), + ], + SEARCH_CONFIG + ); + }); + + it("searches in explicitly listed fields", async () => { + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Interesting" }); + expect(result.records).toHaveLength(1); + + const result2 = await queryRecords(db, SEARCH_CONFIG, { collection, search: "content" }); + expect(result2.records).toHaveLength(1); + }); + + it("does not search non-listed fields", async () => { + // "tech" is in category, which is not in searchable + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "tech" }); + expect(result.records).toHaveLength(0); + }); +}); + +describe("searchable: false", () => { + const collection = "test.disabled.collection"; + + it("search param is ignored when FTS is disabled", async () => { + await applyEvents( + db, + [ + makeEvent({ + uri: "at://did:plc:a/test.disabled.collection/1", + did: "did:plc:a", + collection, + rkey: "1", + record: { name: "Should Not Be Searchable" }, + time_us: 1000, + }), + ], + SEARCH_CONFIG + ); + + // Search is a no-op — returns all records (no FTS join) + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Searchable" }); + expect(result.records).toHaveLength(1); // returned because no FTS filtering applied + }); +}); + +describe("search pagination", () => { + const collection = "community.lexicon.calendar.event"; + + beforeEach(async () => { + const events = Array.from({ length: 5 }, (_, i) => + makeEvent({ + uri: `at://did:plc:a/community.lexicon.calendar.event/e${i}`, + did: "did:plc:a", + collection, + rkey: `e${i}`, + record: { name: `Rust Event ${i}`, mode: "online", description: "test" }, + time_us: (i + 1) * 1000, + }) + ); + await applyEvents(db, events, SEARCH_CONFIG); + }); + + it("paginates search results", async () => { + const page1 = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Rust", + limit: 3, + }); + expect(page1.records).toHaveLength(3); + expect(page1.cursor).toBeDefined(); + + const page2 = await queryRecords(db, SEARCH_CONFIG, { + collection, + search: "Rust", + limit: 3, + cursor: page1.cursor, + }); + expect(page2.records).toHaveLength(2); + expect(page2.cursor).toBeUndefined(); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 9ee54eb..565f91b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,5 +13,5 @@ "isolatedModules": true }, "include": ["src"], - "exclude": ["src/adapters/sqlite.ts"] + "exclude": ["src/adapters/sqlite.ts", "src/generate.ts"] }