From 90cbbfa1ce64344378bd0acf0678fff039ee561b Mon Sep 17 00:00:00 2001 From: Tom Scanlan Date: Wed, 29 Jul 2026 13:56:17 -0400 Subject: [PATCH] fix(search): give the band a Meilisearch path, and index the instant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork the Happening Now band needs before it can run on the backend the list beneath it runs on. A band ranked by D1 sitting above a list ranked by Meilisearch can disagree with itself about which events are live, and does. Adds the ongoing query to the search backend and registers its two cursor names in the pagination envelope, so a band page resumes and deep-links like any other list. Normalises the two fields the index range-filters and sorts on. Meilisearch has no date type, so every bound is a string comparison, which is only chronological while every value is written the same way — and RFC 3339 does not require that. An .ics from Denver yields 2026-08-10T09:00:00-06:00, which collates by its wall-clock digits rather than by the instant it names, so it filters as if it were later than a UTC value it actually precedes. Sub-second precision is the same hazard in miniature: a stored 12:00:00Z compares greater than 12:00:00.500Z because 'Z' sorts above '.'. Storing the instant settles both. Unparseable input still passes through as written; dropping the field would silently remove the event from every bounded query. --- apps/web/src/lib/contrail/cursor.ts | 10 ++- .../src/lib/search/server/meili-sink.test.ts | 4 +- apps/web/src/lib/search/server/meili.test.ts | 49 +++++++++++++-- apps/web/src/lib/search/server/meili.ts | 63 ++++++++++++++++++- .../src/lib/search/server/normalize.test.ts | 44 +++++++++++++ apps/web/src/lib/search/server/normalize.ts | 38 ++++++++++- apps/web/src/lib/search/server/query.ts | 25 ++++++++ 7 files changed, 223 insertions(+), 10 deletions(-) diff --git a/apps/web/src/lib/contrail/cursor.ts b/apps/web/src/lib/contrail/cursor.ts index b424261..6d9d1e8 100644 --- a/apps/web/src/lib/contrail/cursor.ts +++ b/apps/web/src/lib/contrail/cursor.ts @@ -82,6 +82,8 @@ export function parseCursor(cursor: string | null | undefined): ParsedCursor { */ export const CURSOR_QUERIES = [ 'events', + 'happening-now', + 'happening-now-meili', 'hosting', 'past-events', 'topic', @@ -247,7 +249,13 @@ function argsEqual(a: CursorArgs | undefined, b: CursorArgs | undefined): boolea * (`q` + `args`). Search is excluded — its defining term rides `?q=`, not the * envelope, so a search cursor can't be validated against the route. */ -const DEEP_LINKABLE: readonly CursorQuery[] = ['events', 'hosting', 'past-events', 'topic']; +const DEEP_LINKABLE: readonly CursorQuery[] = [ + 'events', + 'happening-now', + 'hosting', + 'past-events', + 'topic' +]; /** * Deep-link guard for a first-page load: return the inbound `?cursor=`'s opaque diff --git a/apps/web/src/lib/search/server/meili-sink.test.ts b/apps/web/src/lib/search/server/meili-sink.test.ts index 99f1f78..0023980 100644 --- a/apps/web/src/lib/search/server/meili-sink.test.ts +++ b/apps/web/src/lib/search/server/meili-sink.test.ts @@ -101,7 +101,9 @@ describe('createMeiliSink onRecords', () => { id: searchDocId('at://did:plc:alice/community.lexicon.calendar.event/1'), uri: 'at://did:plc:alice/community.lexicon.calendar.event/1', name: 'Coffee', - startsAt: '2026-07-01T10:00:00Z', + // Indexed as the instant, not as the record wrote it: the index compares + // this field as a string, so it is normalized (see normalize.ts). + startsAt: '2026-07-01T10:00:00.000Z', _geo: { lat: 40, lng: -105 } }); }); diff --git a/apps/web/src/lib/search/server/meili.test.ts b/apps/web/src/lib/search/server/meili.test.ts index 0da14e4..8add500 100644 --- a/apps/web/src/lib/search/server/meili.test.ts +++ b/apps/web/src/lib/search/server/meili.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi } from 'vitest'; -import { searchEvents, nearMeEvents } from './meili'; +import { searchEvents, nearMeEvents, ongoingEvents } from './meili'; // Fake fetch capturing the request and returning a canned Meilisearch response. function fakeFetch(body: unknown, status = 200) { @@ -85,9 +85,9 @@ describe('searchEvents', () => { await expect(searchEvents(cfg(fetchFn), { q: 'x', limit: 1, offset: 0 })).rejects.toThrow( /search request failed: 403/i ); - await expect( - searchEvents(cfg(fetchFn), { q: 'x', limit: 1, offset: 0 }) - ).rejects.not.toThrow(/read-only-key/); + await expect(searchEvents(cfg(fetchFn), { q: 'x', limit: 1, offset: 0 })).rejects.not.toThrow( + /read-only-key/ + ); }); }); @@ -169,3 +169,44 @@ describe('nearMeEvents', () => { expect(calls).toHaveLength(0); }); }); + +describe('ongoingEvents', () => { + it('bounds BOTH ends of the window and sorts soonest-ending first', async () => { + const { fetchFn, calls } = fakeFetch({ hits: [], estimatedTotalHits: 0 }); + + await ongoingEvents(cfg(fetchFn), { + q: 'town', + limit: 60, + offset: 0, + now: '2026-07-28T12:00:00.000Z' + }); + + expect(JSON.parse(String(calls[0].init.body))).toEqual({ + q: 'town', + limit: 60, + offset: 0, + // Started, and not yet ended — the window every upcoming list drops. + filter: '(startsAt <= "2026-07-28T12:00:00.000Z" AND endsAt >= "2026-07-28T12:00:00.000Z")', + // Overrides relevance ranking: for "what is on now", when it ends is what + // decides whether a reader can still get to it. + sort: ['endsAt:asc'], + attributesToRetrieve: ['uri'] + }); + }); + + it('has no missing-endsAt fallback, unlike the upcoming bound', async () => { + // An event with no endsAt is over once it has started (hasEnded), so it can + // never be ongoing. The bound excludes exactly the records that cannot answer + // "is it still on?" — the same exclusion D1 gets free from `NULL >= 'x'`. + const { fetchFn, calls } = fakeFetch({ hits: [], estimatedTotalHits: 0 }); + + await ongoingEvents(cfg(fetchFn), { + q: 'town', + limit: 60, + offset: 0, + now: '2026-07-28T12:00:00.000Z' + }); + + expect(JSON.parse(String(calls[0].init.body)).filter).not.toContain('NOT EXISTS'); + }); +}); diff --git a/apps/web/src/lib/search/server/meili.ts b/apps/web/src/lib/search/server/meili.ts index b21e5fc..8223034 100644 --- a/apps/web/src/lib/search/server/meili.ts +++ b/apps/web/src/lib/search/server/meili.ts @@ -27,7 +27,10 @@ interface MeiliHit { _geoDistance?: unknown; } -async function querySearchIndex(backend: SearchBackend, body: Record): Promise { +async function querySearchIndex( + backend: SearchBackend, + body: Record +): Promise { const base = backend.url.replace(/\/+$/, ''); const indexUid = backend.indexUid ?? 'events'; const fetchFn = backend.fetch ?? globalThis.fetch; @@ -59,7 +62,15 @@ async function querySearchIndex(backend: SearchBackend, body: Record=now rule the in-memory surfaces use. */ + * fallback. Matches the endsAt||startsAt>=now rule the in-memory surfaces use. + * + * THE "UTC" IN THAT SENTENCE IS AN INVARIANT THIS FILE DEPENDS ON AND DOES NOT + * ESTABLISH. Meilisearch has no date type, so every bound below is a STRING + * comparison, and text order is chronological order only while both sides are + * written the same way. `now` is a toISOString(); the indexed side is made to + * match by `utcInstant` in ./normalize.ts, which is where an offset-bearing + * RFC 3339 timestamp (the importers preserve them) is resolved to its instant. + * Change one side and the other has to move with it. */ function upcomingFilter(now: string): string { return `(endsAt >= "${now}" OR (endsAt NOT EXISTS AND startsAt >= "${now}"))`; } @@ -82,6 +93,54 @@ export async function searchEvents( }); } +/** Filter clause restricting to events UNDER WAY: started, and not yet ended. + * + * There is no missing-endsAt fallback here, and its absence is the rule rather + * than an oversight in it. An event with no endsAt is treated everywhere in this + * app as over once it has started (`hasEnded`, $lib/past-events), so it can never + * be ongoing — the records this bound excludes are exactly the ones that have no + * answer to "is it still on?". Same reasoning as the D1 band, which gets the + * exclusion for free from SQL's `NULL >= 'x'`. + * + * Both bounds are string comparisons against a UTC `now` — see upcomingFilter + * above for the normalization that makes that chronological. This clause is the + * one most exposed to it: it is bounded on BOTH sides, so a timestamp compared + * by its wall-clock digits can fall outside a window it is genuinely inside. */ +function ongoingFilter(now: string): string { + return `(startsAt <= "${now}" AND endsAt >= "${now}")`; +} + +/** + * The happening-now band, ranked by the search backend. + * + * Exists so a TERM-SCOPED band and the page its "see all" links to run the same + * query on the same backend. The band on /search sits beside a Meili-ranked list + * and promotes any live event out of it; a D1-only destination cannot contain + * what Meili ranked and D1 did not, so "see all" led to a smaller list than the + * block it was offered beside. + */ +export async function ongoingEvents( + backend: SearchBackend, + { + q, + limit, + offset, + now = new Date().toISOString() + }: { q: string; limit: number; offset: number; now?: string } +): Promise { + return querySearchIndex(backend, { + q, + limit, + offset, + filter: ongoingFilter(now), + // Soonest-ENDING first, overriding relevance ON PURPOSE and matching the D1 + // band. For "what is on right now", when it ends is what decides whether a + // reader can still get to it; how well it matched the term does not. + sort: ['endsAt:asc'], + attributesToRetrieve: ['uri'] + }); +} + export async function nearMeEvents( backend: SearchBackend, { diff --git a/apps/web/src/lib/search/server/normalize.test.ts b/apps/web/src/lib/search/server/normalize.test.ts index 1926467..d7d28d9 100644 --- a/apps/web/src/lib/search/server/normalize.test.ts +++ b/apps/web/src/lib/search/server/normalize.test.ts @@ -86,3 +86,47 @@ describe('recordGeo', () => { expect(recordGeo(record)).toEqual(eventToSearchDoc(payload(record))._geo); }); }); + +describe('eventToSearchDoc timestamp normalization', () => { + // Meilisearch compares startsAt/endsAt as STRINGS, so what gets indexed has to + // be the instant, not the way the source wrote it. These are the shapes the + // importers actually produce (see lib/import/*.test.ts). + it('rewrites an offset-bearing timestamp to the instant it names', () => { + const doc = eventToSearchDoc( + payload({ startsAt: '2026-08-10T09:00:00-06:00', endsAt: '2026-08-10T17:00:00-06:00' }) + ); + expect(doc.startsAt).toBe('2026-08-10T15:00:00.000Z'); + expect(doc.endsAt).toBe('2026-08-10T23:00:00.000Z'); + }); + + it('orders an offset timestamp against a UTC one by instant, not by text', () => { + // The bug this guards: as raw text '2026-08-10T13:00:00+02:00' sorts AFTER + // '2026-08-10T12:00:00Z', though it is the earlier instant by an hour — so + // a live event could be filtered out of the happening-now band. + const offset = eventToSearchDoc(payload({ startsAt: '2026-08-10T13:00:00+02:00' })).startsAt!; + const utc = eventToSearchDoc(payload({ startsAt: '2026-08-10T12:00:00Z' })).startsAt!; + expect(offset < utc).toBe(true); + expect('2026-08-10T13:00:00+02:00' < '2026-08-10T12:00:00Z').toBe(false); + }); + + it('gives whole seconds the same millisecond precision `now` is written with', () => { + // `now` is a toISOString(); against a bare-seconds string 'Z' > '.', so + // 12:00:00Z used to compare as later than 12:00:00.500Z. + const doc = eventToSearchDoc(payload({ startsAt: '2026-08-10T12:00:00Z' })); + expect(doc.startsAt).toBe('2026-08-10T12:00:00.000Z'); + expect(doc.startsAt! < new Date('2026-08-10T12:00:00.500Z').toISOString()).toBe(true); + }); + + it('passes an unparseable timestamp through rather than dropping the field', () => { + // Dropping it would remove the event from every bounded query; leaving it is + // what the index did before normalization existed. + const doc = eventToSearchDoc(payload({ startsAt: 'not a date' })); + expect(doc.startsAt).toBe('not a date'); + }); + + it('leaves a missing timestamp undefined', () => { + const doc = eventToSearchDoc(payload({ name: 'x' })); + expect(doc.startsAt).toBeUndefined(); + expect(doc.endsAt).toBeUndefined(); + }); +}); diff --git a/apps/web/src/lib/search/server/normalize.ts b/apps/web/src/lib/search/server/normalize.ts index 0dd0e63..b48cbb5 100644 --- a/apps/web/src/lib/search/server/normalize.ts +++ b/apps/web/src/lib/search/server/normalize.ts @@ -101,6 +101,38 @@ function str(v: unknown): string | undefined { return typeof v === 'string' ? v : undefined; } +/** The instant a timestamp names, as one canonical UTC string — for the two + * fields the index RANGE-FILTERS and SORTS on. + * + * Meilisearch has no date type: `startsAt`/`endsAt` are strings, and every + * bound ./meili.ts issues is a string comparison. That is only chronological + * while every value is written the same way, and RFC 3339 does not require + * that — it permits a zone offset, and the importers keep whatever the source + * wrote (an .ics in Denver yields `2026-08-10T09:00:00-06:00`; see + * import/ical.test.ts). Compared as text, such a value collates by its + * wall-clock digits rather than by the instant it names, so `13:00+02:00` + * sorts and filters as if it were later than a `12:00Z` it actually precedes. + * + * Sub-second precision is the same hazard in miniature: `now` is a + * toISOString() with milliseconds, so a stored `12:00:00Z` compares as GREATER + * than `12:00:00.500Z` ('Z' > '.'), off by up to a second even with all-UTC + * data. Emitting toISOString() on both sides settles both cases at once. + * + * Unparseable input passes through as written — that is what the index stores + * today, whereas dropping the field would silently remove the event from every + * bounded query. Only these two fields are normalized: nothing filters or + * sorts on `createdAt`, and no read path retrieves any of them (searches ask + * for `uri` alone), so this changes what the index COMPARES, never what a + * reader is shown. Existing documents converge as they are re-upserted; all + * currently indexed timestamps are already UTC, so the two forms differ only + * in milliseconds until then. */ +function utcInstant(v: unknown): string | undefined { + const s = str(v); + if (s === undefined) return undefined; + const ms = Date.parse(s); + return Number.isNaN(ms) ? s : new Date(ms).toISOString(); +} + /** The record's locations[] narrowed to the location objects deriveGeo and the * doc builder read. */ function recordLocations(record: Record): Loc[] { @@ -132,8 +164,10 @@ export function eventToSearchDoc(payload: EventRecordPayload): SearchDoc { rkey: payload.rkey, name: str(record.name), description: str(record.description), - startsAt: str(record.startsAt), - endsAt: str(record.endsAt), + // The two fields the index range-filters and sorts on, so they carry the + // instant rather than the way the source happened to write it. + startsAt: utcInstant(record.startsAt), + endsAt: utcInstant(record.endsAt), mode: str(record.mode), status: str(record.status), createdAt: str(record.createdAt), diff --git a/apps/web/src/lib/search/server/query.ts b/apps/web/src/lib/search/server/query.ts index 795ea9b..4f84865 100644 --- a/apps/web/src/lib/search/server/query.ts +++ b/apps/web/src/lib/search/server/query.ts @@ -5,6 +5,7 @@ import type { Client } from '@atcute/client'; import { searchEvents, nearMeEvents, + ongoingEvents, type SearchBackend, type SearchHit, type SearchResult @@ -111,6 +112,30 @@ export async function runEventSearchPage( return hydrateToPage(client, result, offset); } +/** + * One page of events UNDER WAY matching a term, ranked by the search backend. + * + * The term-scoped band and the `/events/now` page its "see all" links to both go + * through here, so the two cannot disagree about what matches. They did: the band + * ran on D1 while the list beside it ran on Meili, the page promoted any live + * event out of that list into the band, and the D1-only destination could not + * contain what Meili had ranked and D1 had not. "See all" then led to a shorter + * list than the block it was offered beside. + */ +export async function runOngoingSearchPage( + backend: SearchBackend, + client: Client, + { q, cursor }: { q: string; cursor?: string | null } +): Promise { + const offset = parseOffsetCursor(cursor); + const result = await ongoingEvents(backend, { + q, + limit: SEARCH_PAGE_SIZE * SEARCH_OVERFETCH, + offset + }); + return hydrateToPage(client, result, offset); +} + export async function runNearMePage( backend: SearchBackend, client: Client, -- 2.51.2