diff --git a/src/app/(inbox)/calendar/page.tsx b/src/app/(inbox)/calendar/page.tsx index bfc0592..09e6f97 100644 --- a/src/app/(inbox)/calendar/page.tsx +++ b/src/app/(inbox)/calendar/page.tsx @@ -4,7 +4,7 @@ import CalendarEventLink from "@/components/CalendarEventLink"; import MobileCalendarAgenda from "@/components/MobileCalendarAgenda"; import { resolveCalendarEvents } from "@/lib/calendarDetect"; import { addMonths, buildCalendarEntries, buildMonthDays, filterEventsForMonth, monthTitle, normalizeMonthKey } from "@/lib/calendarView"; -import { listCalendarCandidateEmails } from "@/lib/jmap"; +import { hydrateCalendarBodyValues, listCalendarCandidateEmails } from "@/lib/jmap"; import { getJmapMailboxContext } from "@/lib/jmapServer"; import { mapWithConcurrency } from "@/lib/promisePool"; @@ -41,8 +41,13 @@ export default async function CalendarPage({ searchParams }: Props) { ); }); - const resolvedEvents = (await mapWithConcurrency( + const hydratedInviteEmails = await hydrateCalendarBodyValues( + session.apiUrl, + accountId, inviteEmails, + ); + const resolvedEvents = (await mapWithConcurrency( + hydratedInviteEmails, CALENDAR_DOWNLOAD_CONCURRENCY, (email) => resolveCalendarEvents(email, session.downloadUrl, accountId) diff --git a/src/lib/__tests__/jmap.test.ts b/src/lib/__tests__/jmap.test.ts index 2134afb..a151358 100644 --- a/src/lib/__tests__/jmap.test.ts +++ b/src/lib/__tests__/jmap.test.ts @@ -3,7 +3,7 @@ process.env.FASTMAIL_API_TOKEN = "test-token"; import { describe, it } from "node:test"; import assert from "node:assert/strict"; -import { buildMailPanelMethodCalls, clearRecipientSuggestionCaches, deleteDraft, destroyAllEmailsInMailbox, destroyEmails, downloadBlobAsText, getAccountId, getContactsAccountId, getEmailState, getUnreadInboxTotal, listCalendarCandidateEmails, listInboxEmails, loadMailPanelData, loadMoreEmailsFiltered, moveEmailsToMailbox, parseAddresses, saveDraft, searchContacts, searchEmails, searchRecipientSuggestions, sendEmail, setKeywordsOnMany } from "../jmap"; +import { buildMailPanelMethodCalls, clearCalendarBodyCache, clearRecipientSuggestionCaches, deleteDraft, destroyAllEmailsInMailbox, destroyEmails, downloadBlobAsText, getAccountId, getContactsAccountId, getEmailState, getUnreadInboxTotal, hydrateCalendarBodyValues, listCalendarCandidateEmails, listInboxEmails, loadMailPanelData, loadMoreEmailsFiltered, moveEmailsToMailbox, parseAddresses, saveDraft, searchContacts, searchEmails, searchRecipientSuggestions, sendEmail, setKeywordsOnMany } from "../jmap"; const MAIL_CAP = "urn:ietf:params:jmap:mail"; @@ -315,31 +315,110 @@ describe("listInboxEmails", () => { }); describe("listCalendarCandidateEmails", () => { - it("pages through the complete mailbox instead of stopping at a fixed ceiling", async () => { + function makeCalendarEmail(id: string, location: "attachment" | "inline") { + const calendarPart = { + partId: `part-${id}`, + blobId: `blob-${id}`, + size: 100, + type: "text/calendar", + name: "invite.ics", + }; + return { + ...makeEmailResponse(id, true), + messageId: null, + cc: null, + textBody: location === "inline" ? [calendarPart] : [], + attachments: location === "attachment" ? [calendarPart] : [], + bodyValues: {}, + }; + } + + it("pages only attachment-bearing mail and adds a bounded recent fallback", async () => { capturedBodies = []; mockResponses = [ makeJmapResponse([ - ["Email/query", { ids: ["e1", "e2"], total: 3 }, "q"], - ["Email/get", { list: [makeEmailResponse("e1", true), makeEmailResponse("e2", true)] }, "g"], + ["Email/query", { ids: ["e1", "e2"], total: 3 }, "aq0"], + ["Email/get", { list: [makeCalendarEmail("e1", "attachment"), makeEmailResponse("e2", true)] }, "ag0"], + ["Email/query", { ids: ["e4"] }, "rq"], + ["Email/get", { list: [makeCalendarEmail("e4", "inline")] }, "rg"], ]), makeJmapResponse([ - ["Email/query", { ids: ["e3"], total: 3 }, "q"], - ["Email/get", { list: [makeEmailResponse("e3", true)] }, "g"], + ["Email/query", { ids: ["e3"], total: 3 }, "aq1"], + ["Email/get", { list: [makeCalendarEmail("e3", "attachment")] }, "ag1"], ]), ]; const result = await listCalendarCandidateEmails( "https://api.example.com/jmap", "acct1", - 2 + 2, + 1, ); - assert.deepEqual(result.map((email) => email.id), ["e1", "e2", "e3"]); + assert.deepEqual(result.map((email) => email.id), ["e1", "e4", "e3"]); assert.equal(capturedBodies.length, 2); assert.equal((capturedBodies[0] as any).methodCalls[0][1].position, 0); assert.equal((capturedBodies[1] as any).methodCalls[0][1].position, 2); + assert.deepEqual((capturedBodies[0] as any).methodCalls[0][1].filter, { + hasAttachment: true, + }); + assert.equal((capturedBodies[0] as any).methodCalls[2][1].limit, 1); + assert.equal((capturedBodies[1] as any).methodCalls.length, 2); assert.equal((capturedBodies[0] as any).methodCalls[0][1].calculateTotal, true); }); + + it("hydrates all calendar bodies in one request and reuses immutable blobs", async () => { + clearCalendarBodyCache(); + capturedBodies = []; + const emails = [ + makeCalendarEmail("e1", "attachment"), + makeCalendarEmail("e2", "attachment"), + ] as any[]; + mockResponses = [ + makeJmapResponse([ + [ + "Email/get", + { + list: emails.map((email) => ({ + id: email.id, + bodyValues: { + [`part-${email.id}`]: { + value: `BEGIN:VCALENDAR\r\nX-ID:${email.id}\r\nEND:VCALENDAR\r\n`, + charset: "utf-8", + isEncodingProblem: false, + isTruncated: false, + }, + }, + })), + }, + "g", + ], + ]), + ]; + + const first = await hydrateCalendarBodyValues( + "https://api.example.com/jmap", + "acct1", + emails, + ); + + assert.equal(capturedBodies.length, 1); + assert.deepEqual((capturedBodies[0] as any).methodCalls[0][1].ids, ["e1", "e2"]); + assert.equal((capturedBodies[0] as any).methodCalls[0][1].fetchAllBodyValues, true); + assert.match(first[0].bodyValues["part-e1"].value, /X-ID:e1/); + + capturedBodies = []; + mockResponses = []; + const second = await hydrateCalendarBodyValues( + "https://api.example.com/jmap", + "acct1", + emails, + ); + + assert.equal(capturedBodies.length, 0); + assert.match(second[1].bodyValues["part-e2"].value, /X-ID:e2/); + clearCalendarBodyCache(); + }); }); describe("downloadBlobAsText", () => { diff --git a/src/lib/calendarDetect.ts b/src/lib/calendarDetect.ts index 80db4d1..8ed995d 100644 --- a/src/lib/calendarDetect.ts +++ b/src/lib/calendarDetect.ts @@ -40,8 +40,11 @@ export async function resolveCalendarEvents( try { let icsText: string | null = null; - if (calPart.partId && email.bodyValues?.[calPart.partId]) { - icsText = email.bodyValues[calPart.partId].value; + const bodyValue = calPart.partId + ? email.bodyValues?.[calPart.partId] + : undefined; + if (bodyValue?.value && bodyValue.isTruncated !== true) { + icsText = bodyValue.value; } else if (calPart.blobId) { icsText = await downloadBlobAsText( downloadUrl, diff --git a/src/lib/jmap.ts b/src/lib/jmap.ts index 0b7ed46..cd21438 100644 --- a/src/lib/jmap.ts +++ b/src/lib/jmap.ts @@ -57,7 +57,7 @@ export async function jmapCall( const data: { methodResponses: [string, Record, string][] } = await res.json(); if (options.logSuccess !== false) { - log.info( + log.debug( { methods, method_count: methodCalls.length, account_id: accountId, response_count: data.methodResponses.length, duration_ms: Date.now() - t }, "jmap.call" ); @@ -367,6 +367,11 @@ export async function loadMailPanelData( // returned as truncated bodyValues. Keep a generous bounded cap so full // layouts render without turning normal list queries into body downloads. const MAX_RENDERED_BODY_BYTES = 10 * 1024 * 1024; +const MAX_CALENDAR_BODY_BYTES = 1024 * 1024; +const CALENDAR_QUERY_BATCH_SIZE = 4000; +const RECENT_INLINE_CALENDAR_CANDIDATES = 500; +const CALENDAR_BODY_BATCH_SIZE = 1000; +const CALENDAR_BODY_CACHE_MAX_ENTRIES = 2000; const CALENDAR_CANDIDATE_PROPERTIES = [ "id", @@ -384,6 +389,58 @@ const CALENDAR_CANDIDATE_PROPERTIES = [ "attachments", ]; +const calendarBodyCache = new Map(); + +function calendarPart(email: Pick) { + return ( + email.textBody?.find((part) => part.type === "text/calendar") ?? + email.attachments?.find((part) => part.type === "text/calendar") + ); +} + +function calendarBodyCacheKey(accountId: string, blobId: string) { + return `${accountId}:${blobId}`; +} + +function cacheCalendarBody(accountId: string, blobId: string, value: string) { + const key = calendarBodyCacheKey(accountId, blobId); + calendarBodyCache.delete(key); + calendarBodyCache.set(key, value); + + while (calendarBodyCache.size > CALENDAR_BODY_CACHE_MAX_ENTRIES) { + const oldestKey = calendarBodyCache.keys().next().value; + if (oldestKey === undefined) break; + calendarBodyCache.delete(oldestKey); + } +} + +function getCachedCalendarBody(accountId: string, blobId: string): string | undefined { + const key = calendarBodyCacheKey(accountId, blobId); + const value = calendarBodyCache.get(key); + if (value === undefined) return undefined; + cacheCalendarBody(accountId, blobId, value); + return value; +} + +function withCalendarBodyValue(email: Email, partId: string, value: string): Email { + return { + ...email, + bodyValues: { + ...(email.bodyValues ?? {}), + [partId]: { + value, + charset: "utf-8", + isEncodingProblem: false, + isTruncated: false, + }, + }, + }; +} + +export function clearCalendarBodyCache() { + calendarBodyCache.clear(); +} + async function queryEmailPage( apiUrl: string, accountId: string, @@ -709,42 +766,86 @@ export async function setPin( export async function listCalendarCandidateEmails( apiUrl: string, accountId: string, - batchSize = 250 + batchSize = CALENDAR_QUERY_BATCH_SIZE, + recentLimit = RECENT_INLINE_CALENDAR_CANDIDATES, ): Promise { const t = Date.now(); - const emails: Email[] = []; + const candidates = new Map(); let position = 0; let total = Number.POSITIVE_INFINITY; let pageCount = 0; while (position < total) { - const data = await jmapCall(apiUrl, [ + const attachmentQueryId = `aq${pageCount}`; + const attachmentGetId = `ag${pageCount}`; + const methodCalls: MethodCall[] = [ [ "Email/query", { accountId, + filter: { hasAttachment: true }, sort: [{ property: "receivedAt", isAscending: false }], calculateTotal: true, limit: batchSize, position, }, - "q", + attachmentQueryId, ], [ "Email/get", { accountId, - "#ids": { resultOf: "q", name: "Email/query", path: "/ids" }, + "#ids": { + resultOf: attachmentQueryId, + name: "Email/query", + path: "/ids", + }, properties: CALENDAR_CANDIDATE_PROPERTIES, }, - "g", + attachmentGetId, ], - ]); - const [, queryResult] = data.methodResponses.find(([name]) => name === "Email/query") ?? []; - const [, getResult] = data.methodResponses.find(([name]) => name === "Email/get") ?? []; + ]; + + if (pageCount === 0 && recentLimit > 0) { + methodCalls.push( + [ + "Email/query", + { + accountId, + sort: [{ property: "receivedAt", isAscending: false }], + limit: recentLimit, + position: 0, + }, + "rq", + ], + [ + "Email/get", + { + accountId, + "#ids": { resultOf: "rq", name: "Email/query", path: "/ids" }, + properties: CALENDAR_CANDIDATE_PROPERTIES, + }, + "rg", + ], + ); + } + + const data = await jmapCall(apiUrl, methodCalls, { logSuccess: false }); + const queryResult = data.methodResponses.find(([, , id]) => id === attachmentQueryId)?.[1]; + const getResult = data.methodResponses.find(([, , id]) => id === attachmentGetId)?.[1]; const ids = ((queryResult as { ids?: string[] } | undefined)?.ids) ?? []; total = (queryResult as { total?: number } | undefined)?.total ?? position + ids.length; - emails.push(...(((getResult as { list?: Email[] } | undefined)?.list) ?? [])); + + const emails = ((getResult as { list?: Email[] } | undefined)?.list) ?? []; + const recentEmails = + pageCount === 0 + ? ((data.methodResponses.find(([, , id]) => id === "rg")?.[1] as + | { list?: Email[] } + | undefined)?.list ?? []) + : []; + for (const email of [...emails, ...recentEmails]) { + if (calendarPart(email)) candidates.set(email.id, email); + } pageCount += 1; if (ids.length === 0) break; @@ -752,10 +853,113 @@ export async function listCalendarCandidateEmails( } log.info( - { count: emails.length, pages: pageCount, batch_size: batchSize, duration_ms: Date.now() - t }, + { + count: candidates.size, + attachment_total: Number.isFinite(total) ? total : 0, + pages: pageCount, + batch_size: batchSize, + recent_limit: recentLimit, + duration_ms: Date.now() - t, + }, "jmap.list_calendar_candidates" ); - return emails; + return [...candidates.values()]; +} + +export async function hydrateCalendarBodyValues( + apiUrl: string, + accountId: string, + emails: readonly Email[], + batchSize = CALENDAR_BODY_BATCH_SIZE, +): Promise { + const t = Date.now(); + const hydratedById = new Map(); + const pending: Email[] = []; + let cachedCount = 0; + + for (const email of emails) { + const part = calendarPart(email); + const existingValue = part?.partId ? email.bodyValues?.[part.partId] : undefined; + if (!part?.partId || (existingValue?.value && existingValue.isTruncated !== true)) { + hydratedById.set(email.id, email); + continue; + } + + const cachedValue = part.blobId + ? getCachedCalendarBody(accountId, part.blobId) + : undefined; + if (cachedValue !== undefined) { + hydratedById.set( + email.id, + withCalendarBodyValue(email, part.partId, cachedValue), + ); + cachedCount += 1; + continue; + } + + pending.push(email); + } + + let requestCount = 0; + for (let offset = 0; offset < pending.length; offset += batchSize) { + const batch = pending.slice(offset, offset + batchSize); + const data = await jmapCall( + apiUrl, + [ + [ + "Email/get", + { + accountId, + ids: batch.map((email) => email.id), + properties: ["id", "bodyValues"], + fetchAllBodyValues: true, + maxBodyValueBytes: MAX_CALENDAR_BODY_BYTES, + }, + "g", + ], + ], + { logSuccess: false }, + ); + requestCount += 1; + + const list = (data.methodResponses.find(([, , id]) => id === "g")?.[1] as + | { list?: Email[] } + | undefined)?.list ?? []; + const bodyValuesById = new Map( + list.map((email) => [email.id, email.bodyValues] as const), + ); + + for (const email of batch) { + const part = calendarPart(email); + const bodyValues = bodyValuesById.get(email.id); + const bodyValue = part?.partId ? bodyValues?.[part.partId] : undefined; + const hydrated = bodyValues + ? { ...email, bodyValues: { ...(email.bodyValues ?? {}), ...bodyValues } } + : email; + hydratedById.set(email.id, hydrated); + + if ( + part?.blobId && + bodyValue?.value && + bodyValue.isTruncated !== true + ) { + cacheCalendarBody(accountId, part.blobId, bodyValue.value); + } + } + } + + log.info( + { + count: emails.length, + fetched_count: pending.length, + cached_count: cachedCount, + request_count: requestCount, + duration_ms: Date.now() - t, + }, + "jmap.calendar_body_values", + ); + + return emails.map((email) => hydratedById.get(email.id) ?? email); } export async function saveDraft(