diff --git a/apps/web/src/routes/api/cron/+server.ts b/apps/web/src/routes/api/cron/+server.ts index 6f19117..6308d67 100644 --- a/apps/web/src/routes/api/cron/+server.ts +++ b/apps/web/src/routes/api/cron/+server.ts @@ -25,7 +25,7 @@ export const POST: RequestHandler = async ({ request, platform }) => { // Firehose ingest — guarded so an over-budget cycle can't 500 the whole tick // (which previously also took the bot down with it). try { - await ensureInit(db); + await ensureInit(db, platform!.env); // Two deliberately-split budgets. contrail saves the jetstream cursor LAST in // runIngestCycle — after the drain, applyEvents, and the per-DID // refreshStaleIdentities network tail. If this handler aborts before that -- 2.51.2 From 4dba3da064e222c6891452e6021805f2b3ef6743 Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Tue, 7 Jul 2026 22:42:25 -0400 Subject: [PATCH 2/6] test(cron): assert the cron tick arms the sink by passing env Pins the regression: ensureInit must be called with the platform env, not just the db, or the search sink never arms and cron ingest writes nothing to Meili. Also asserts the tick still runs the isolated bot/notify/drip stages on a valid request. --- apps/web/src/routes/api/cron/server.test.ts | 80 +++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 apps/web/src/routes/api/cron/server.test.ts diff --git a/apps/web/src/routes/api/cron/server.test.ts b/apps/web/src/routes/api/cron/server.test.ts new file mode 100644 index 0000000..88556be --- /dev/null +++ b/apps/web/src/routes/api/cron/server.test.ts @@ -0,0 +1,80 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +// Mock the cron handler's collaborators so the test exercises only the handler's +// wiring: which functions it calls, and — the key regression — that it arms the +// Meili search sink by passing env to ensureInit. vi.hoisted keeps the spies +// available to the hoisted vi.mock factories below. +const { ensureInit, ingest, processBotMentions, runNotifications, runGeocodeDrip } = vi.hoisted( + () => ({ + ensureInit: vi.fn(async (_db: unknown, _env?: unknown) => {}), + ingest: vi.fn(async () => {}), + processBotMentions: vi.fn(async () => {}), + runNotifications: vi.fn(async () => {}), + runGeocodeDrip: vi.fn(async () => {}) + }) +); + +vi.mock('$lib/contrail/index', () => ({ + ensureInit, + contrail: { ingest } +})); +vi.mock('$lib/bot/process-mentions', () => ({ processBotMentions })); +vi.mock('$lib/notify/process', () => ({ runNotifications })); +vi.mock('$lib/geocode/process', () => ({ runGeocodeDrip })); + +import { POST } from './+server'; + +const CRON_SECRET = 'cron-secret'; +const DB = { __brand: 'D1' }; + +// Minimal RequestEvent shape the handler actually reads: request headers + +// platform.env (CRON_SECRET, DB, and the sink creds ensureInit reads). +function event(opts: { secret?: string | null } = {}) { + const headers = new Headers(); + if (opts.secret != null) headers.set('X-Cron-Secret', opts.secret); + const env = { + CRON_SECRET, + DB, + SEARCH_SINK_URL: 'https://search.example', + SEARCH_SINK_API_KEY: 'sink-key' + }; + return { + request: new Request('https://atmo.rsvp/api/cron', { method: 'POST', headers }), + platform: { env } + } as unknown as Parameters[0]; +} + +describe('POST /api/cron', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('rejects a request without the cron secret (no sink arming, no ingest)', async () => { + const res = await POST(event({ secret: 'wrong' })); + expect(res.status).toBe(401); + expect(ensureInit).not.toHaveBeenCalled(); + expect(ingest).not.toHaveBeenCalled(); + }); + + it('arms the Meili sink by passing env to ensureInit', async () => { + const res = await POST(event({ secret: CRON_SECRET })); + expect(res.status).toBe(200); + + // The regression fix: ensureInit must be called WITH the platform env, not + // just the db, or searchSinkBackend never arms and cron ingest writes + // nothing to Meili. + expect(ensureInit).toHaveBeenCalledTimes(1); + expect(ensureInit).toHaveBeenCalledWith(DB, expect.objectContaining({ CRON_SECRET, DB })); + // Second arg (env) must be present — the whole bug was omitting it. + expect(ensureInit.mock.calls[0].length).toBe(2); + expect(ensureInit.mock.calls[0][1]).toBeDefined(); + }); + + it('still runs ingest and the isolated bot/notify/drip stages on a valid tick', async () => { + await POST(event({ secret: CRON_SECRET })); + expect(processBotMentions).toHaveBeenCalledTimes(1); + expect(ingest).toHaveBeenCalledTimes(1); + expect(runNotifications).toHaveBeenCalledTimes(1); + expect(runGeocodeDrip).toHaveBeenCalledTimes(1); + }); +}); -- 2.51.2 From 00f91c2fc788397474b089a1522e6a2247418bc4 Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Wed, 8 Jul 2026 11:31:08 -0400 Subject: [PATCH 3/6] fix(search): bound the Meili settings/arming fetch with a timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit applyMeiliSettings' PATCH ran unbounded. On the cron path it fires inside ensureInit BEFORE the ingest hard-timeout race, so a stalling (not erroring) Meili endpoint would hang the whole tick past the 55s backstop and starve notify/drip; the xrpc handler ran the same unbounded fetch on user requests. Thread an AbortSignal.timeout (SETTINGS_TIMEOUT_MS=8s, overridable for tests) through applyMeiliSettings so a stall degrades to "sink disabled this cycle" (ensureInit's existing try/catch), retried next tick. Upsert/remove stay unbounded — they run only inside contrail.ingest, already under the cron race. --- .../src/lib/search/server/meili-sink.test.ts | 35 +++++++++- apps/web/src/lib/search/server/meili-sink.ts | 66 +++++++++++++------ 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/apps/web/src/lib/search/server/meili-sink.test.ts b/apps/web/src/lib/search/server/meili-sink.test.ts index 32d1d00..99f1f78 100644 --- a/apps/web/src/lib/search/server/meili-sink.test.ts +++ b/apps/web/src/lib/search/server/meili-sink.test.ts @@ -236,7 +236,11 @@ describe('createMeiliSink onRecords', () => { it('applies index settings once, before the first write (fresh-index safety)', async () => { const { fn, calls } = fakeFetch(); - const sink = createMeiliSink(() => BACKEND, () => null, fn); + const sink = createMeiliSink( + () => BACKEND, + () => null, + fn + ); // Two batches on the same sink: a fresh-rollout `pnpm backfill` must not // let PUT /documents auto-create a bare index whose _geo/startsAt searches @@ -292,6 +296,35 @@ describe('fetch is invoked detached (workerd Illegal invocation guard)', () => { }); }); +describe('applyMeiliSettings bounds the settings fetch', () => { + it('passes an AbortSignal on the settings request', async () => { + let sawSignal: AbortSignal | null | undefined; + const fn = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + sawSignal = init?.signal; + return new Response(null, { status: 202 }); + }) as unknown as typeof fetch; + + await applyMeiliSettings(BACKEND, fn); + expect(sawSignal).toBeInstanceOf(AbortSignal); + }); + + it('rejects when the settings fetch stalls past the timeout', async () => { + // A fetch that never settles on its own — only the AbortSignal can end it. + // A short timeout keeps this fast and deterministic without fake timers. + const stalling = vi.fn( + (_input: RequestInfo | URL, init?: RequestInit) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + if (signal) { + signal.addEventListener('abort', () => reject(signal.reason ?? new Error('aborted'))); + } + }) + ) as unknown as typeof fetch; + + await expect(applyMeiliSettings(BACKEND, stalling, 10)).rejects.toThrow(); + }); +}); + /** A minimal D1 double: prepare().bind().all() returns the seeded resolved * rows whose address_norm is in the bound args. Throw mode exercises the * best-effort swallow. */ diff --git a/apps/web/src/lib/search/server/meili-sink.ts b/apps/web/src/lib/search/server/meili-sink.ts index ca2219b..2e39434 100644 --- a/apps/web/src/lib/search/server/meili-sink.ts +++ b/apps/web/src/lib/search/server/meili-sink.ts @@ -27,6 +27,16 @@ type RecordEvent = Parameters[0][number]; /** The one collection we index for search. */ export const EVENT_COLLECTION = 'community.lexicon.calendar.event'; +/** How long the settings PATCH (which also arms/creates the index) may run + * before we abort it. This request fires on the ensureInit arming path — and + * for the cron handler that runs BEFORE the ingest hard-timeout race, so an + * unbounded fetch to a *stalling* (not erroring) Meili endpoint would hang the + * whole tick past the 55s backstop and starve notify/drip. A short bound + * degrades that to "sink disabled this cycle" (caught by ensureInit's + * try/catch), retried next tick. Upsert/remove stay unbounded here: they only + * run inside contrail.ingest, already under the cron race. */ +const SETTINGS_TIMEOUT_MS = 8_000; + export interface MeiliSinkBackend { url: string; apiKey: string; @@ -82,21 +92,28 @@ export class MeiliEventIndex { /** PATCHing settings auto-creates the index, so this doubles as ensure-index. * _geo in filterable enables _geoRadius/_geoBoundingBox; in sortable, _geoPoint. - * Must mirror the filters ./meili.ts issues (startsAt/endsAt range, _geo). */ - async applySettings(): Promise { - await this.request('PATCH', `/indexes/${this.indexUid}/settings`, { - searchableAttributes: ['name', 'description'], - filterableAttributes: [ - '_geo', - 'startsAt', - 'endsAt', - 'status', - 'mode', - 'did', - 'locationTypes' - ], - sortableAttributes: ['_geo', 'startsAt', 'endsAt'] - }); + * Must mirror the filters ./meili.ts issues (startsAt/endsAt range, _geo). + * Bounded by an AbortSignal so a stalling Meili endpoint can't hang the + * arming path (see SETTINGS_TIMEOUT_MS). */ + async applySettings(timeoutMs: number = SETTINGS_TIMEOUT_MS): Promise { + await this.request( + 'PATCH', + `/indexes/${this.indexUid}/settings`, + { + searchableAttributes: ['name', 'description'], + filterableAttributes: [ + '_geo', + 'startsAt', + 'endsAt', + 'status', + 'mode', + 'did', + 'locationTypes' + ], + sortableAttributes: ['_geo', 'startsAt', 'endsAt'] + }, + AbortSignal.timeout(timeoutMs) + ); } async upsert(docs: SearchDoc[]): Promise { @@ -109,7 +126,12 @@ export class MeiliEventIndex { await this.request('POST', `/indexes/${this.indexUid}/documents/delete-batch`, ids); } - private async request(method: string, path: string, body: unknown): Promise { + private async request( + method: string, + path: string, + body: unknown, + signal?: AbortSignal + ): Promise { // Call fetch detached, not as `this.fetch(...)`: on workerd the global // fetch throws "Illegal invocation" when invoked with `this` bound to a // non-global object (which method-call syntax would do). A bare call @@ -122,7 +144,8 @@ export class MeiliEventIndex { authorization: `Bearer ${this.apiKey}`, 'content-type': 'application/json' }, - body: JSON.stringify(body) + body: JSON.stringify(body), + signal }); if (!res.ok) { // No body/key echoed — it can end up in logs or error pages. @@ -133,12 +156,15 @@ export class MeiliEventIndex { /** PATCH the index settings (and auto-create the index) for a backend. Call * once per worker before the sink starts upserting so the read path's filters - * resolve. */ + * resolve. Bounded by SETTINGS_TIMEOUT_MS (override for tests) so a stalling + * Meili endpoint can't hang the caller — the arming path runs outside the cron + * ingest race. */ export async function applyMeiliSettings( backend: MeiliSinkBackend, - fetchFn?: typeof fetch + fetchFn?: typeof fetch, + timeoutMs: number = SETTINGS_TIMEOUT_MS ): Promise { - await new MeiliEventIndex(backend, fetchFn).applySettings(); + await new MeiliEventIndex(backend, fetchFn).applySettings(timeoutMs); } /** Best-effort fill of doc._geo from the geocode_cache. Read-only; a missing -- 2.51.2 From 0198233e8df33e716789a997e3a09bf5ddcb2620 Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Thu, 9 Jul 2026 10:24:48 -0400 Subject: [PATCH 4/6] fix(cron): race sink arming inside the ingest backstop Arming (ensureInit) was awaited before the Promise.race hard backstop, so its bounded settings fetch was additive to the 55s ingest budget: a stalling search endpoint could stretch a tick to ~63s, past the 60s cron interval, overlapping the next tick and breaking the documented "ticks don't overlap" invariant. Move arming inside the race, ahead of ingest, so arming and ingest share ONE budget bounded by HARD_TIMEOUT_MS. Ordering is preserved (the sink must be armed before ingest upserts). Widen the failure log to cover the arming step now that it runs inside the guarded block. --- apps/web/src/routes/api/cron/+server.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/web/src/routes/api/cron/+server.ts b/apps/web/src/routes/api/cron/+server.ts index 6308d67..d1bf03a 100644 --- a/apps/web/src/routes/api/cron/+server.ts +++ b/apps/web/src/routes/api/cron/+server.ts @@ -25,7 +25,6 @@ export const POST: RequestHandler = async ({ request, platform }) => { // Firehose ingest — guarded so an over-budget cycle can't 500 the whole tick // (which previously also took the bot down with it). try { - await ensureInit(db, platform!.env); // Two deliberately-split budgets. contrail saves the jetstream cursor LAST in // runIngestCycle — after the drain, applyEvents, and the per-DID // refreshStaleIdentities network tail. If this handler aborts before that @@ -40,16 +39,26 @@ export const POST: RequestHandler = async ({ request, platform }) => { // normal cycle, and under the cron interval so ticks don't overlap. // Scheduled invocations get ~15min wall-clock and this is I/O-bound, // so the tail has ample room. + // The HARD budget covers ARMING too: ensureInit runs INSIDE the race, ahead + // of ingest (the sink must be armed before ingest upserts — see the ordering + // rationale in $lib/contrail/index.ts). ensureInit's own settings fetch is + // separately time-bounded, but that bound would be ADDITIVE to HARD if arming + // were awaited before the race — a stalling search endpoint could then push a + // tick past the cron interval and overlap the next one. Racing arming + + // ingest together keeps the whole tick under HARD, and thus under the interval. const DRAIN_TIMEOUT_MS = 20_000; const HARD_TIMEOUT_MS = 55_000; await Promise.race([ - contrail.ingest({ timeoutMs: DRAIN_TIMEOUT_MS }, db), + (async () => { + await ensureInit(db, platform!.env); + await contrail.ingest({ timeoutMs: DRAIN_TIMEOUT_MS }, db); + })(), new Promise((_, reject) => setTimeout(() => reject(new Error('ingest hard timeout')), HARD_TIMEOUT_MS) ) ]); } catch (e) { - console.error('[cron] contrail.ingest failed:', e); + console.error('[cron] ingest cycle failed:', e); } // atmo.pub notifications: event reminders + host RSVP alerts. Runs after -- 2.51.2 From 02ad0b247af7bfde2e0a38bbc7c37d00a24ef3dd Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Thu, 9 Jul 2026 10:25:02 -0400 Subject: [PATCH 5/6] test(cron): pin arming-failure isolation and the shared arming/ingest budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two properties the cron tick relied on but nothing pinned: - Arming-failure isolation: when the search sink's settings/arming fetch rejects, the tick must still 200 and ingest must still run. The existing cron test mocks the contrail module wholesale, replacing the real ensureInit, so a regression deleting ensureInit's try/catch passed the suite. The new arming-isolation test runs the REAL ensureInit (stubbing only the Contrail engine and forcing applyMeiliSettings to reject) so that try/catch is actually exercised — deleting it turns the test red (ingest stops running). - Shared budget: a slow arm plus a stalling ingest must still settle within the hard backstop and reach the post-ingest stages. With fake timers, advancing exactly the hard budget settles the tick; if arming were awaited before the race (the old additive shape) the tick would not settle. --- .../routes/api/cron/arming-isolation.test.ts | 100 ++++++++++++++++++ apps/web/src/routes/api/cron/server.test.ts | 32 ++++++ 2 files changed, 132 insertions(+) create mode 100644 apps/web/src/routes/api/cron/arming-isolation.test.ts diff --git a/apps/web/src/routes/api/cron/arming-isolation.test.ts b/apps/web/src/routes/api/cron/arming-isolation.test.ts new file mode 100644 index 0000000..9ca848a --- /dev/null +++ b/apps/web/src/routes/api/cron/arming-isolation.test.ts @@ -0,0 +1,100 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +// Pins the arming-failure ISOLATION property: when the search sink's +// settings/arming fetch fails (a stalling or erroring search endpoint), the cron +// tick must still return 200 AND ingest must still run. That property rests +// entirely on the try/catch inside the REAL ensureInit ($lib/contrail/index.ts). +// +// Unlike the sibling server.test.ts — which mocks $lib/contrail/index wholesale +// and therefore replaces ensureInit with a stub — this suite runs the REAL +// ensureInit so the try/catch is actually exercised. Delete that try/catch and +// this test goes red (ingest stops running once arming rejects). We replace only: +// - applyMeiliSettings (in the search-sink module): forced to reject, standing +// in for an unreachable/stalling search endpoint on the arming path; +// - the Contrail engine (init/ingest): stubbed so no real D1 / firehose I/O +// runs, and so we can assert ingest was still invoked after arming failed; +// - the sibling cron stages (bot / notify / drip): no-op spies. +const { ingest, contrailInit, processBotMentions, runNotifications, runGeocodeDrip, applyMeiliSettings } = + vi.hoisted(() => ({ + ingest: vi.fn(async () => {}), + contrailInit: vi.fn(async () => {}), + processBotMentions: vi.fn(async () => {}), + runNotifications: vi.fn(async () => {}), + runGeocodeDrip: vi.fn(async () => {}), + applyMeiliSettings: vi.fn(async () => { + throw new Error('search endpoint unreachable'); + }) + })); + +// Stub the Contrail engine so the REAL $lib/contrail/index can load and its real +// ensureInit run without touching a live DB or jetstream. The real module builds +// `new Contrail(...)`, so this instance is what ensureInit.init and the handler's +// contrail.ingest resolve to. +vi.mock('@atmo-dev/contrail', async (importActual) => { + const actual = await importActual(); + return { + ...actual, + Contrail: class { + init = contrailInit; + ingest = ingest; + } + }; +}); +vi.mock('@atmo-dev/contrail/server', () => ({ createHandler: () => () => new Response(null) })); + +// The real ensureInit calls applyMeiliSettings from this module on the arming +// path; force that one call to reject. Everything else stays real (notably +// meiliSinkBackendFromEnv, which must return a backend from the env below). +vi.mock('$lib/search/server/meili-sink', async (importActual) => { + const actual = await importActual(); + return { ...actual, applyMeiliSettings }; +}); + +vi.mock('$lib/bot/process-mentions', () => ({ processBotMentions })); +vi.mock('$lib/notify/process', () => ({ runNotifications })); +vi.mock('$lib/geocode/process', () => ({ runGeocodeDrip })); + +import { POST } from './+server'; + +const CRON_SECRET = 'cron-secret'; +const DB = { __brand: 'D1' }; + +function event(opts: { secret?: string | null } = {}) { + const headers = new Headers(); + if (opts.secret != null) headers.set('X-Cron-Secret', opts.secret); + const env = { + CRON_SECRET, + DB, + // Present + complete so meiliSinkBackendFromEnv yields a backend and the real + // ensureInit reaches applyMeiliSettings (which we force to reject). + SEARCH_SINK_URL: 'https://search.example', + SEARCH_SINK_API_KEY: 'sink-key' + }; + return { + request: new Request('https://atmo.rsvp/api/cron', { method: 'POST', headers }), + platform: { env } + } as unknown as Parameters[0]; +} + +describe('POST /api/cron — arming-failure isolation (real ensureInit)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('still returns 200 and still runs ingest + notify/drip when sink arming rejects', async () => { + const res = await POST(event({ secret: CRON_SECRET })); + + // Arming was actually attempted and failed (real ensureInit -> real + // meiliSinkBackendFromEnv -> the rejecting applyMeiliSettings). + expect(applyMeiliSettings).toHaveBeenCalledTimes(1); + + // The load-bearing invariant: a failing arm must be isolated inside + // ensureInit's try/catch, so the tick 200s AND ingest still runs. Deleting + // that try/catch makes ensureInit reject, which skips ingest — this + // assertion is what catches that regression. + expect(res.status).toBe(200); + expect(ingest).toHaveBeenCalledTimes(1); + expect(runNotifications).toHaveBeenCalledTimes(1); + expect(runGeocodeDrip).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/web/src/routes/api/cron/server.test.ts b/apps/web/src/routes/api/cron/server.test.ts index 88556be..df73ce7 100644 --- a/apps/web/src/routes/api/cron/server.test.ts +++ b/apps/web/src/routes/api/cron/server.test.ts @@ -77,4 +77,36 @@ describe('POST /api/cron', () => { expect(runNotifications).toHaveBeenCalledTimes(1); expect(runGeocodeDrip).toHaveBeenCalledTimes(1); }); + + it('bounds arming + ingest under ONE hard budget: a slow arm + stalling ingest still settles under the cron interval and reaches notify/drip', async () => { + vi.useFakeTimers(); + // Arming resolves only after a delay, then ingest hangs forever. The tick can + // end ONLY via the handler's hard backstop. Because arming now runs INSIDE + // that backstop race (not awaited before it), arming and ingest share ONE + // budget: the whole tick is bounded by the hard timeout (55s) — not arming + + // 55s. Advancing exactly the hard budget must settle the tick and reach the + // post-ingest stages. If arming were awaited before the race (the old, + // additive shape), the backstop would only start counting after arming, so + // advancing 55s would NOT settle the tick and this test would hang. + ensureInit.mockImplementation(() => new Promise((resolve) => setTimeout(resolve, 10_000))); + ingest.mockImplementation(() => new Promise(() => {})); + try { + const resPromise = POST(event({ secret: CRON_SECRET })); + await vi.advanceTimersByTimeAsync(55_000); + const res = await resPromise; + + expect(res.status).toBe(200); + // Arming ran first (it resolved), then ingest was reached and hung; once + // the backstop fired, the isolated post-ingest stages still ran. + expect(ingest).toHaveBeenCalledTimes(1); + expect(runNotifications).toHaveBeenCalledTimes(1); + expect(runGeocodeDrip).toHaveBeenCalledTimes(1); + } finally { + ensureInit.mockReset(); + ingest.mockReset(); + ensureInit.mockImplementation(async (_db: unknown, _env?: unknown) => {}); + ingest.mockImplementation(async () => {}); + vi.useRealTimers(); + } + }); }); -- 2.51.2 From 7cfb4447c4e2c6b067b85bbec8b53ce6bdd14f76 Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Thu, 9 Jul 2026 10:39:17 -0400 Subject: [PATCH 6/6] docs(cron,search): correct comments the arming-in-race move made false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving sink arming inside the cron ingest race invalidated three comments that now asserted the opposite of the code (this ships to a public upstream): - cron/+server.ts: dropped the claim that racing arming+ingest keeps "the whole tick" under the interval. The race bounds only the raced section (arming + ingest); the tick also spans the un-raced bot/notify/drip stages. State that the bound guards the search-endpoint hazard, not the tick's total duration. - meili-sink.ts (SETTINGS_TIMEOUT_MS doc): the arming fetch no longer runs "before the ingest race" — it runs inside it. Describe the current topology: the bound makes arming self-terminate within the shared budget, and also backs the xrpc arming path (outside any race). Also correct the false "upsert/remove only run inside contrail.ingest under the cron race": upsert is also reached from the geocode drip (after the race) and xrpc notify re-ingests (no race), so those write paths stay unbounded — a pre-existing exposure. - meili-sink.ts (applyMeiliSettings doc): "the arming path runs outside the cron ingest race" is now false for cron; distinguish cron (inside the race) from xrpc (outside any race). Comments only; no logic change. check: 0 errors / 7 pre-existing warnings; test: 283 passed / 4 skipped (unchanged). --- apps/web/src/lib/search/server/meili-sink.ts | 26 +++++++++++++------- apps/web/src/routes/api/cron/+server.ts | 5 +++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/apps/web/src/lib/search/server/meili-sink.ts b/apps/web/src/lib/search/server/meili-sink.ts index 2e39434..38318aa 100644 --- a/apps/web/src/lib/search/server/meili-sink.ts +++ b/apps/web/src/lib/search/server/meili-sink.ts @@ -28,13 +28,20 @@ type RecordEvent = Parameters[0][number]; export const EVENT_COLLECTION = 'community.lexicon.calendar.event'; /** How long the settings PATCH (which also arms/creates the index) may run - * before we abort it. This request fires on the ensureInit arming path — and - * for the cron handler that runs BEFORE the ingest hard-timeout race, so an - * unbounded fetch to a *stalling* (not erroring) Meili endpoint would hang the - * whole tick past the 55s backstop and starve notify/drip. A short bound - * degrades that to "sink disabled this cycle" (caught by ensureInit's - * try/catch), retried next tick. Upsert/remove stay unbounded here: they only - * run inside contrail.ingest, already under the cron race. */ + * before we abort it. This request fires on the ensureInit arming path. On the + * cron handler that path now runs INSIDE the ingest hard-timeout race, ahead of + * ingest, so the bound is what makes arming self-terminate within the shared + * budget instead of consuming it: without it, a *stalling* (not erroring) Meili + * endpoint would eat the whole race budget and starve the ingest it runs ahead + * of. The same bound also covers the user-facing xrpc arming path, which is NOT + * inside any race and would otherwise hang the request unboundedly. A short + * bound degrades either case to "sink disabled this cycle" (caught by + * ensureInit's try/catch), retried next tick. + * Upsert/remove are NOT bounded here — they pass no AbortSignal. They run under + * the cron race (via contrail.ingest) but ALSO outside it: MeiliEventIndex.upsert + * is reached from the geocode drip (runGeocodeDrip, after the race) and from + * xrpc notify re-ingests (no race at all), so a stalling Meili can still hang + * those write paths. Pre-existing exposure, tracked separately. */ const SETTINGS_TIMEOUT_MS = 8_000; export interface MeiliSinkBackend { @@ -157,8 +164,9 @@ export class MeiliEventIndex { /** PATCH the index settings (and auto-create the index) for a backend. Call * once per worker before the sink starts upserting so the read path's filters * resolve. Bounded by SETTINGS_TIMEOUT_MS (override for tests) so a stalling - * Meili endpoint can't hang the caller — the arming path runs outside the cron - * ingest race. */ + * Meili endpoint can't hang the caller — on cron the arming path runs inside the + * ingest race and must self-terminate within that shared budget; on xrpc it runs + * outside any race, where this bound is the only backstop. */ export async function applyMeiliSettings( backend: MeiliSinkBackend, fetchFn?: typeof fetch, diff --git a/apps/web/src/routes/api/cron/+server.ts b/apps/web/src/routes/api/cron/+server.ts index d1bf03a..9576e6b 100644 --- a/apps/web/src/routes/api/cron/+server.ts +++ b/apps/web/src/routes/api/cron/+server.ts @@ -45,7 +45,10 @@ export const POST: RequestHandler = async ({ request, platform }) => { // separately time-bounded, but that bound would be ADDITIVE to HARD if arming // were awaited before the race — a stalling search endpoint could then push a // tick past the cron interval and overlap the next one. Racing arming + - // ingest together keeps the whole tick under HARD, and thus under the interval. + // ingest together bounds the RACED SECTION by HARD, so a stalling search + // endpoint can no longer push that section past the interval. (The whole + // tick also spans the un-raced stages below — bot/notify/drip — so this + // bounds the search-endpoint hazard, not the tick's total duration.) const DRAIN_TIMEOUT_MS = 20_000; const HARD_TIMEOUT_MS = 55_000; await Promise.race([