From 5b01ff7557df936717bcf2a865935592f8a0d06a Mon Sep 17 00:00:00 2001 From: "nekomimi.pet" Date: Sat, 3 Oct 2026 07:36:22 -0400 Subject: [PATCH] select file_objects when reading site_cache in hosting and firehose found on the devnet: neither getSiteCache selected the new column, so hosting answered 503 for every file of a CAS site and the writer's update ledger never saw the previous mapping. the unit tests missed it because they fake the row. the new postgres tests read through the real queries in a throwaway schema. --- .../src/lib/db.site-cache.pg.test.ts | 43 +++++++++++++++++ apps/firehose-service/src/lib/db.ts | 2 +- apps/hosting-service/src/lib/db.test.ts | 46 ++++++++++++++++++- apps/hosting-service/src/lib/db.ts | 2 +- 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 apps/firehose-service/src/lib/db.site-cache.pg.test.ts diff --git a/apps/firehose-service/src/lib/db.site-cache.pg.test.ts b/apps/firehose-service/src/lib/db.site-cache.pg.test.ts new file mode 100644 index 0000000..eef5df7 --- /dev/null +++ b/apps/firehose-service/src/lib/db.site-cache.pg.test.ts @@ -0,0 +1,43 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import { CAS_SCHEMA_STATEMENTS } from '@wispplace/database' +import postgres from 'postgres' + +// The module binds its connection at import, so point it at a throwaway schema first: +// WISP_TEST_DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/wisp bun test src/lib/db.site-cache.pg.test.ts +const url = process.env.WISP_TEST_DATABASE_URL +const schema = `fh_site_cache_${Math.random().toString(36).slice(2, 10)}` +if (url) process.env.DATABASE_URL = `${url}${url.includes('?') ? '&' : '?'}search_path=${schema}` +const { getSiteCache } = await import('./db') + +const suite = url ? describe : describe.skip + +suite('getSiteCache (postgres)', () => { + let admin: postgres.Sql + + beforeAll(async () => { + admin = postgres(url as string, { max: 1, onnotice: () => {} }) + await admin.unsafe(`CREATE SCHEMA ${schema}`) + await admin.unsafe(`SET search_path TO ${schema}`) + await admin.unsafe( + "CREATE TABLE site_cache (did TEXT NOT NULL, rkey TEXT NOT NULL, record_cid TEXT NOT NULL, file_cids JSONB NOT NULL DEFAULT '{}', cached_at BIGINT, updated_at BIGINT, cold_synced BOOLEAN NOT NULL DEFAULT true, PRIMARY KEY (did, rkey))", + ) + for (const statement of CAS_SCHEMA_STATEMENTS) await admin.unsafe(statement) + }) + + afterAll(async () => { + await admin.unsafe(`DROP SCHEMA ${schema} CASCADE`) + await admin.end() + }) + + test('returns the path to CAS key mapping the update ledger is built from', async () => { + const objects = { 'index.html': 'cas/bafkreiaaa.deadbeef.html' } + await admin` + INSERT INTO site_cache (did, rkey, record_cid, file_cids, file_objects) + VALUES ('did:plc:test', 'site', 'cid', ${admin.json({ 'index.html': 'bafkreiaaa' })}, ${admin.json(objects)}) + ` + + const row = await getSiteCache('did:plc:test', 'site') + + expect(row?.file_objects).toEqual(objects) + }) +}) diff --git a/apps/firehose-service/src/lib/db.ts b/apps/firehose-service/src/lib/db.ts index e8d293d..f3c249a 100644 --- a/apps/firehose-service/src/lib/db.ts +++ b/apps/firehose-service/src/lib/db.ts @@ -877,7 +877,7 @@ export async function withSiteWriteLock( export async function getSiteCache(did: string, rkey: string): Promise { const result = await sql` - SELECT did, rkey, record_cid, file_cids, cached_at, updated_at, cold_synced + SELECT did, rkey, record_cid, file_cids, file_objects, cached_at, updated_at, cold_synced FROM site_cache WHERE did = ${did} AND rkey = ${rkey} LIMIT 1 diff --git a/apps/hosting-service/src/lib/db.test.ts b/apps/hosting-service/src/lib/db.test.ts index 6cf8942..5bdbbba 100644 --- a/apps/hosting-service/src/lib/db.test.ts +++ b/apps/hosting-service/src/lib/db.test.ts @@ -1,5 +1,14 @@ -import { describe, expect, test } from 'bun:test' -import { type ClosableDatabasePool, createDatabasePoolCloser } from './db' +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' +import { CAS_SCHEMA_STATEMENTS } from '@wispplace/database' +import postgres from 'postgres' + +// The module binds its connection at import, so point it at a throwaway schema first: +// WISP_TEST_DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/wisp bun test src/lib/db.test.ts +const url = process.env.WISP_TEST_DATABASE_URL +const schema = `hosting_db_${Math.random().toString(36).slice(2, 10)}` +if (url) process.env.DATABASE_URL = `${url}${url.includes('?') ? '&' : '?'}search_path=${schema}` +const { createDatabasePoolCloser, getSiteCache } = await import('./db') +type ClosableDatabasePool = Parameters[0] describe('database pool closer', () => { test('ends the pool once across repeated calls', async () => { @@ -32,3 +41,36 @@ describe('database pool closer', () => { expect(reported).toBe(1) }) }) + +const suite = url ? describe : describe.skip + +suite('getSiteCache (postgres)', () => { + let admin: postgres.Sql + + beforeAll(async () => { + admin = postgres(url as string, { max: 1, onnotice: () => {} }) + await admin.unsafe(`CREATE SCHEMA ${schema}`) + await admin.unsafe(`SET search_path TO ${schema}`) + await admin.unsafe( + "CREATE TABLE site_cache (did TEXT NOT NULL, rkey TEXT NOT NULL, record_cid TEXT NOT NULL, file_cids JSONB NOT NULL DEFAULT '{}', cached_at BIGINT, updated_at BIGINT, absent_since BIGINT, PRIMARY KEY (did, rkey))", + ) + for (const statement of CAS_SCHEMA_STATEMENTS) await admin.unsafe(statement) + }) + + afterAll(async () => { + await admin.unsafe(`DROP SCHEMA ${schema} CASCADE`) + await admin.end() + }) + + test('returns the path to CAS key mapping the files are read from', async () => { + const objects = { 'index.html': 'cas/bafkreiaaa.deadbeef.html' } + await admin` + INSERT INTO site_cache (did, rkey, record_cid, file_cids, file_objects) + VALUES ('did:plc:test', 'site', 'cid', ${admin.json({ 'index.html': 'bafkreiaaa' })}, ${admin.json(objects)}) + ` + + const row = await getSiteCache('did:plc:test', 'site') + + expect(row?.file_objects).toEqual(objects) + }) +}) diff --git a/apps/hosting-service/src/lib/db.ts b/apps/hosting-service/src/lib/db.ts index 9701d52..6c5ba1b 100644 --- a/apps/hosting-service/src/lib/db.ts +++ b/apps/hosting-service/src/lib/db.ts @@ -131,7 +131,7 @@ export async function getSiteCache(did: string, rkey: string): Promise { const result = await sql` - SELECT did, rkey, record_cid, file_cids, cached_at, updated_at, absent_since + SELECT did, rkey, record_cid, file_cids, file_objects, cached_at, updated_at, absent_since FROM site_cache WHERE did = ${did} AND rkey = ${rkey} LIMIT 1 -- 2.54.0