From 7dafe449d2f308c9c437fcbca50920a2085bf631 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 2 Apr 2026 01:14:56 +0200 Subject: [PATCH] switch sqlite to node:sqlite --- .github/workflows/release.yml | 3 - README.md | 7 +- package.json | 2 - pnpm-lock.yaml | 15 --- src/adapters/sqlite.ts | 6 +- src/core/db/schema.ts | 11 +- tests/search.test.ts | 200 +++++++++------------------------- tsup.config.ts | 2 +- 8 files changed, 67 insertions(+), 179 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a323ea4..9148167 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,9 +32,6 @@ jobs: - name: Install dependencies run: pnpm install - - name: Rebuild native modules - run: pnpm rebuild better-sqlite3 - - name: Build run: pnpm build diff --git a/README.md b/README.md index 7491c68..f334147 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,14 @@ export default { ### SQLite adapter (Node.js / local dev) ```ts -import { SqliteDatabase } from "@atmo-dev/contrail/sqlite"; -import Database from "better-sqlite3"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; -const db = new SqliteDatabase(new Database("data.db")); +const db = createSqliteDatabase("data.db"); const contrail = new Contrail({ ...config, db }); ``` +> **Note:** The SQLite adapter uses Node's built-in `node:sqlite` (Node 22+). Full-text search (`searchable`) is not supported with this adapter because `node:sqlite` doesn't include the FTS5 extension. Search works on Cloudflare D1, which has FTS5 enabled. + ## Running the example (Cloudflare Workers) This repo includes a working example that indexes AT Protocol calendar events and RSVPs on Cloudflare Workers + D1. diff --git a/package.json b/package.json index 7c9c525..36814f5 100644 --- a/package.json +++ b/package.json @@ -64,9 +64,7 @@ "@atcute/lex-cli": "^2.5.3", "@atcute/lexicon-doc": "^2.1.2", "@cloudflare/workers-types": "^4.20250124.0", - "@types/better-sqlite3": "^7.6.13", "@types/node": "^25.5.0", - "better-sqlite3": "^12.8.0", "@changesets/cli": "^2.29.4", "tsup": "^8.5.0", "tsx": "^4.21.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8d603ad..0ec7542 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,15 +39,9 @@ importers: '@cloudflare/workers-types': specifier: ^4.20250124.0 version: 4.20260205.0 - '@types/better-sqlite3': - specifier: ^7.6.13 - version: 7.6.13 '@types/node': specifier: ^25.5.0 version: 25.5.0 - better-sqlite3: - specifier: ^12.8.0 - version: 12.8.0 tsup: specifier: ^8.5.0 version: 8.5.1(postcss@8.5.8)(tsx@4.21.0)(typescript@5.9.3) @@ -996,10 +990,6 @@ packages: better-sqlite3@11.10.0: resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==, tarball: https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz} - better-sqlite3@12.8.0: - resolution: {integrity: sha512-RxD2Vd96sQDjQr20kdP+F+dK/1OUNiVOl200vKBZY8u0vTwysfolF6Hq+3ZK2+h8My9YvZhHsF+RSGZW2VYrPQ==} - engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x} - bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -2712,11 +2702,6 @@ snapshots: bindings: 1.5.0 prebuild-install: 7.1.3 - better-sqlite3@12.8.0: - dependencies: - bindings: 1.5.0 - prebuild-install: 7.1.3 - bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 diff --git a/src/adapters/sqlite.ts b/src/adapters/sqlite.ts index 74af363..25c6d9f 100644 --- a/src/adapters/sqlite.ts +++ b/src/adapters/sqlite.ts @@ -1,9 +1,9 @@ -import BetterSqlite3 from "better-sqlite3"; +import { DatabaseSync } from "node:sqlite"; import type { Database, Statement } from "../core/types"; export function createSqliteDatabase(path: string): Database { - const raw = new BetterSqlite3(path); - raw.pragma("journal_mode = WAL"); + const raw = new DatabaseSync(path); + raw.exec("PRAGMA journal_mode = WAL"); function wrapStatement(sql: string, boundValues: any[] = []): Statement { return { diff --git a/src/core/db/schema.ts b/src/core/db/schema.ts index 93699d7..b2d8a29 100644 --- a/src/core/db/schema.ts +++ b/src/core/db/schema.ts @@ -207,9 +207,18 @@ export async function initSchema( const indexStatements = buildDynamicIndexes(config); const ftsStatements = buildFtsTables(config); const feedStatements = buildFeedTables(config); - const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...ftsStatements, ...feedStatements]; + const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; await db.batch(all.map((s) => db.prepare(s))); + + // FTS5 may not be available (e.g. node:sqlite) — skip gracefully + for (const stmt of ftsStatements) { + try { + await db.prepare(stmt).run(); + } catch { + // FTS5 not supported in this environment + } + } await runMigrations(db); // Add count columns (ALTER TABLE — may already exist) diff --git a/tests/search.test.ts b/tests/search.test.ts index 71ac87b..c0f91ce 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -5,6 +5,14 @@ import { createTestDb, makeEvent } from "./helpers"; import { initSchema } from "../src/core/db/schema"; import { applyEvents, queryRecords } from "../src/core/db/records"; +// Detect FTS5 support at module level (node:sqlite doesn't include it) +let hasFts = false; +try { + const testDb = createTestDb(); + await testDb.prepare("CREATE VIRTUAL TABLE __fts_test USING fts5(content)").run(); + hasFts = true; +} catch {} + const SEARCH_CONFIG = resolveConfig({ namespace: "com.example", collections: { @@ -23,13 +31,13 @@ const SEARCH_CONFIG = resolveConfig({ body: {}, category: {}, }, - searchable: ["title", "body"], // explicit: only title and body + searchable: ["title", "body"], }, "test.disabled.collection": { queryable: { name: {}, }, - searchable: false, // disabled + searchable: false, }, }, }); @@ -37,11 +45,12 @@ const SEARCH_CONFIG = resolveConfig({ let db: Database; beforeEach(async () => { + if (!hasFts) return; db = createTestDb(); await initSchema(db, SEARCH_CONFIG); }); -describe("FTS with explicit searchable fields", () => { +describe.skipIf(!hasFts)("FTS with explicit searchable fields", () => { const collection = "community.lexicon.calendar.event"; beforeEach(async () => { @@ -78,10 +87,7 @@ describe("FTS with explicit searchable fields", () => { }); it("finds records matching a search term", async () => { - const result = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Rust", - }); + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Rust" }); expect(result.records).toHaveLength(2); const names = result.records.map((r) => JSON.parse(r.record!).name); expect(names).toContain("Rust Meetup"); @@ -89,42 +95,28 @@ describe("FTS with explicit searchable fields", () => { }); it("searches across multiple fields", async () => { - const result = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Rustaceans", - }); + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Rustaceans" }); expect(result.records).toHaveLength(1); expect(JSON.parse(result.records[0].record!).name).toBe("Rust Meetup"); }); it("returns nothing for non-matching search", async () => { - const result = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Python", - }); + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Python" }); expect(result.records).toHaveLength(0); }); it("supports prefix search", async () => { - const result = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Type*", - }); + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Type*" }); expect(result.records).toHaveLength(2); }); it("combines search with filters", async () => { - const result = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Rust", - filters: { mode: "in-person" }, - }); + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Rust", filters: { mode: "in-person" } }); expect(result.records).toHaveLength(1); expect(JSON.parse(result.records[0].record!).name).toBe("Rust Meetup"); }); it("does not search range fields (startsAt)", async () => { - // startsAt is range, so not included in FTS. Searching for its value should not match. await applyEvents( db, [ @@ -139,152 +131,68 @@ describe("FTS with explicit searchable fields", () => { ], SEARCH_CONFIG ); - // "T10" would appear in the startsAt value but not in any searchable field - const result = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "T10", - }); + const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "T10" }); expect(result.records).toHaveLength(0); }); }); -describe("FTS sync", () => { +describe.skipIf(!hasFts)("FTS sync", () => { const collection = "community.lexicon.calendar.event"; it("updates FTS on record update", async () => { - await applyEvents( - db, - [ - makeEvent({ - uri: "at://did:plc:a/community.lexicon.calendar.event/1", - collection, - rkey: "1", - record: { name: "Old Name", mode: "online", description: "test" }, - time_us: 1000, - }), - ], - SEARCH_CONFIG - ); - - // Update the record - await applyEvents( - db, - [ - makeEvent({ - uri: "at://did:plc:a/community.lexicon.calendar.event/1", - collection, - rkey: "1", - record: { name: "New Name", mode: "online", description: "test" }, - operation: "update", - time_us: 2000, - }), - ], - SEARCH_CONFIG - ); - - const oldResult = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Old" }); - expect(oldResult.records).toHaveLength(0); - - const newResult = await queryRecords(db, SEARCH_CONFIG, { collection, search: "New" }); - expect(newResult.records).toHaveLength(1); + await applyEvents(db, [ + makeEvent({ uri: "at://did:plc:a/community.lexicon.calendar.event/1", collection, rkey: "1", record: { name: "Old Name", mode: "online", description: "test" }, time_us: 1000 }), + ], SEARCH_CONFIG); + await applyEvents(db, [ + makeEvent({ uri: "at://did:plc:a/community.lexicon.calendar.event/1", collection, rkey: "1", record: { name: "New Name", mode: "online", description: "test" }, operation: "update", time_us: 2000 }), + ], SEARCH_CONFIG); + expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "Old" })).records).toHaveLength(0); + expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "New" })).records).toHaveLength(1); }); it("removes from FTS on delete", async () => { - await applyEvents( - db, - [ - makeEvent({ - uri: "at://did:plc:a/community.lexicon.calendar.event/1", - collection, - rkey: "1", - record: { name: "Deletable", mode: "online", description: "test" }, - time_us: 1000, - }), - ], - SEARCH_CONFIG - ); - - await applyEvents( - db, - [ - makeEvent({ - uri: "at://did:plc:a/community.lexicon.calendar.event/1", - collection, - rkey: "1", - operation: "delete", - record: { name: "Deletable", mode: "online", description: "test" }, - time_us: 2000, - }), - ], - SEARCH_CONFIG - ); - - const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Deletable" }); - expect(result.records).toHaveLength(0); + await applyEvents(db, [ + makeEvent({ uri: "at://did:plc:a/community.lexicon.calendar.event/1", collection, rkey: "1", record: { name: "Deletable", mode: "online", description: "test" }, time_us: 1000 }), + ], SEARCH_CONFIG); + await applyEvents(db, [ + makeEvent({ uri: "at://did:plc:a/community.lexicon.calendar.event/1", collection, rkey: "1", operation: "delete", record: { name: "Deletable", mode: "online", description: "test" }, time_us: 2000 }), + ], SEARCH_CONFIG); + expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "Deletable" })).records).toHaveLength(0); }); }); -describe("explicit searchable fields", () => { +describe.skipIf(!hasFts)("explicit searchable fields", () => { const collection = "test.explicit.collection"; beforeEach(async () => { - await applyEvents( - db, - [ - makeEvent({ - uri: "at://did:plc:a/test.explicit.collection/1", - did: "did:plc:a", - collection, - rkey: "1", - record: { title: "Interesting Article", body: "Some content here", category: "tech" }, - time_us: 1000, - }), - ], - SEARCH_CONFIG - ); + await applyEvents(db, [ + makeEvent({ uri: "at://did:plc:a/test.explicit.collection/1", did: "did:plc:a", collection, rkey: "1", record: { title: "Interesting Article", body: "Some content here", category: "tech" }, time_us: 1000 }), + ], SEARCH_CONFIG); }); it("searches in explicitly listed fields", async () => { - const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Interesting" }); - expect(result.records).toHaveLength(1); - - const result2 = await queryRecords(db, SEARCH_CONFIG, { collection, search: "content" }); - expect(result2.records).toHaveLength(1); + expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "Interesting" })).records).toHaveLength(1); + expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "content" })).records).toHaveLength(1); }); it("does not search non-listed fields", async () => { - // "tech" is in category, which is not in searchable - const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "tech" }); - expect(result.records).toHaveLength(0); + expect((await queryRecords(db, SEARCH_CONFIG, { collection, search: "tech" })).records).toHaveLength(0); }); }); -describe("searchable: false", () => { +describe.skipIf(!hasFts)("searchable: false", () => { const collection = "test.disabled.collection"; it("search param is ignored when FTS is disabled", async () => { - await applyEvents( - db, - [ - makeEvent({ - uri: "at://did:plc:a/test.disabled.collection/1", - did: "did:plc:a", - collection, - rkey: "1", - record: { name: "Should Not Be Searchable" }, - time_us: 1000, - }), - ], - SEARCH_CONFIG - ); - - // Search is a no-op — returns all records (no FTS join) + await applyEvents(db, [ + makeEvent({ uri: "at://did:plc:a/test.disabled.collection/1", did: "did:plc:a", collection, rkey: "1", record: { name: "Should Not Be Searchable" }, time_us: 1000 }), + ], SEARCH_CONFIG); const result = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Searchable" }); - expect(result.records).toHaveLength(1); // returned because no FTS filtering applied + expect(result.records).toHaveLength(1); }); }); -describe("search pagination", () => { +describe.skipIf(!hasFts)("search pagination", () => { const collection = "community.lexicon.calendar.event"; beforeEach(async () => { @@ -302,20 +210,10 @@ describe("search pagination", () => { }); it("paginates search results", async () => { - const page1 = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Rust", - limit: 3, - }); + const page1 = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Rust", limit: 3 }); expect(page1.records).toHaveLength(3); expect(page1.cursor).toBeDefined(); - - const page2 = await queryRecords(db, SEARCH_CONFIG, { - collection, - search: "Rust", - limit: 3, - cursor: page1.cursor, - }); + const page2 = await queryRecords(db, SEARCH_CONFIG, { collection, search: "Rust", limit: 3, cursor: page1.cursor }); expect(page2.records).toHaveLength(2); expect(page2.cursor).toBeUndefined(); }); diff --git a/tsup.config.ts b/tsup.config.ts index 58e12ba..cf6d592 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -12,5 +12,5 @@ export default defineConfig({ sourcemap: true, clean: true, tsconfig: "tsconfig.build.json", - external: ["better-sqlite3"], + external: ["node:sqlite"], }); -- 2.51.2