diff --git a/system/netlify/functions/og-preview.mjs b/system/netlify/functions/og-preview.mjs index 207426326b..dadb12b881 100644 --- a/system/netlify/functions/og-preview.mjs +++ b/system/netlify/functions/og-preview.mjs @@ -9,18 +9,21 @@ const isDev = process.env.NETLIFY_DEV === "true" || process.env.CONTEXT === "dev // Simple in-memory cache with TTL (1 hour) const cache = new Map(); const CACHE_TTL = 60 * 60 * 1000; // 1 hour +const UNAVAILABLE_TTL = 5 * 60 * 1000; +const MAX_HTML_BYTES = 50 * 1024; +const FETCH_TIMEOUT_MS = 8000; function getCached(url) { const entry = cache.get(url); if (!entry) return null; - if (Date.now() - entry.timestamp > CACHE_TTL) { + if (Date.now() - entry.timestamp >= entry.ttl) { cache.delete(url); return null; } - return entry.data; + return entry; } -function setCache(url, data) { +function setCache(url, data, ttl = CACHE_TTL) { // Limit cache size to prevent memory issues if (cache.size > 1000) { // Delete oldest entries @@ -29,7 +32,42 @@ function setCache(url, data) { cache.delete(entries[i][0]); } } - cache.set(url, { data, timestamp: Date.now() }); + cache.set(url, { data, timestamp: Date.now(), ttl }); +} + +function previewResponse(data, ttl) { + return Response.json(data, { headers: { + "Access-Control-Allow-Origin": "*", + "Cache-Control": `public, max-age=${Math.max(0, Math.floor(ttl / 1000))}`, + } }); +} + +function unavailablePreview(url, reason, upstreamStatus) { + return { + url: url.href, title: url.hostname, siteName: url.hostname, + description: null, image: null, favicon: null, + unavailable: true, reason, upstreamStatus, + }; +} + +async function readHtml(response) { + if (!response.body) return ""; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let html = "", bytes = 0; + try { + while (bytes < MAX_HTML_BYTES) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = value.subarray(0, MAX_HTML_BYTES - bytes); + bytes += chunk.byteLength; + html += decoder.decode(chunk, { stream: true }); + } + return html + decoder.decode(); + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } } export default async function handler(req) { @@ -86,14 +124,7 @@ export default async function handler(req) { // Check cache const cached = getCached(targetUrl); if (cached) { - return new Response(JSON.stringify(cached), { - status: 200, - headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - "Cache-Control": "public, max-age=3600", - }, - }); + return previewResponse(cached.data, cached.ttl - (Date.now() - cached.timestamp)); } // In dev mode, Netlify Dev intercepts all outbound HTTP from functions, @@ -120,10 +151,9 @@ export default async function handler(req) { }); } + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); try { - // Use fetch to get the page HTML - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 8000); const response = await fetch(targetUrl, { headers: { @@ -134,9 +164,15 @@ export default async function handler(req) { redirect: 'follow', }); - clearTimeout(timeout); - if (!response.ok) { + await response.body?.cancel().catch(() => {}); + // A remote site can decline previews while the original link remains + // clickable. Return site-only metadata and avoid retrying each view. + if ([401, 403, 404, 410].includes(response.status)) { + const data = unavailablePreview(parsedUrl, "upstream_unavailable", response.status); + setCache(targetUrl, data, UNAVAILABLE_TTL); + return previewResponse(data, UNAVAILABLE_TTL); + } return new Response(JSON.stringify({ error: `HTTP ${response.status}` }), { status: 502, headers: { @@ -146,30 +182,32 @@ export default async function handler(req) { }); } - const html = await response.text(); + const contentType = (response.headers.get("content-type") || "").split(";")[0].trim().toLowerCase(); + if (contentType && !["text/html", "application/xhtml+xml"].includes(contentType)) { + await response.body?.cancel().catch(() => {}); + const data = unavailablePreview(parsedUrl, "not_html", response.status); + setCache(targetUrl, data); + return previewResponse(data, CACHE_TTL); + } + const html = await readHtml(response); // Parse Open Graph and other meta tags - const resultData = parseMetaTags(html.slice(0, 50 * 1024), targetUrl); + const resultData = parseMetaTags(html, targetUrl); setCache(targetUrl, resultData); - return new Response(JSON.stringify(resultData), { - status: 200, - headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - "Cache-Control": "public, max-age=3600", - }, - }); + return previewResponse(resultData, CACHE_TTL); } catch (err) { console.error(`[og-preview] Error fetching ${targetUrl}:`, err.message, err.code || ''); const errorMessage = err.name === "AbortError" ? "Request timed out" : err.message; return new Response(JSON.stringify({ error: errorMessage, code: err.code }), { - status: 500, + status: err.name === "AbortError" ? 504 : 502, headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*", }, }); + } finally { + clearTimeout(timeout); } } diff --git a/system/public/aesthetic.computer/disks/chat.mjs b/system/public/aesthetic.computer/disks/chat.mjs index 5914ff00fa..eca754dc49 100644 --- a/system/public/aesthetic.computer/disks/chat.mjs +++ b/system/public/aesthetic.computer/disks/chat.mjs @@ -5213,7 +5213,7 @@ async function loadOgPreview(url, preload) { } // If still no image, try common favicon paths - if (!imageData && !faviconData) { + if (!imageData && !faviconData && !data.unavailable) { try { const urlObj = new URL(url); const defaultFavicon = `${urlObj.origin}/favicon.ico`; diff --git a/system/tests/og-preview.test.mjs b/system/tests/og-preview.test.mjs new file mode 100644 index 0000000000..fad917c275 --- /dev/null +++ b/system/tests/og-preview.test.mjs @@ -0,0 +1,83 @@ +// node --experimental-vm-modules --test system/tests/og-preview.test.mjs +import test from 'node:test'; +import assert from 'node:assert/strict'; +import vm from 'node:vm'; +import {readFile} from 'node:fs/promises'; + +const source=await readFile(new URL('../netlify/functions/og-preview.mjs',import.meta.url),'utf8'); +async function fixture(fetch) { + let now=1000000; + const timers=new Map();let nextTimer=0; + const context=vm.createContext({fetch,Response,URL,AbortController,TextDecoder, + Date:{now:()=>now},process:{env:{}},console:{log(){},error(){}}, + setTimeout:fn=>{const id=++nextTimer;timers.set(id,fn);return id;}, + clearTimeout:id=>timers.delete(id)}); + const module=new vm.SourceTextModule(source,{context}); + await module.link(()=>{throw new Error('Unexpected import');});await module.evaluate(); + return {request:(url='https://example.org/article')=>module.namespace.default(new Request('https://aesthetic.computer/api/og-preview?url='+encodeURIComponent(url))), + advance:ms=>{now+=ms;},timers}; +} + +test('HTML previews retain metadata and relative image URLs',async()=>{ + const f=await fixture(async()=>new Response('Fallback',{headers:{'Content-Type':'text/html; charset=utf-8'}})); + const r=await f.request();const d=await r.json(); + assert.equal(r.status,200);assert.equal(d.title,'A & B');assert.equal(d.image,'https://example.org/cover.png');assert.equal(d.unavailable,undefined);assert.equal(f.timers.size,0); +}); + +test('denied previews return cached site cards without claiming rich metadata',async()=>{ + let calls=0;const f=await fixture(async()=>{calls++;return new Response('Forbidden',{status:403});}); + const r=await f.request();const d=await r.json(); + assert.equal(r.status,200);assert.equal(d.title,'example.org');assert.equal(d.url,'https://example.org/article');assert.equal(d.unavailable,true);assert.equal(d.upstreamStatus,403);assert.equal(d.image,null);assert.equal(d.favicon,null); + assert.equal(r.headers.get('Cache-Control'),'public, max-age=300'); + f.advance(120000);const cached=await f.request();assert.equal(cached.headers.get('Cache-Control'),'public, max-age=180');assert.equal(calls,1); + f.advance(180000);await f.request();assert.equal(calls,2);assert.equal(f.timers.size,0); +}); + +test('other unavailable links get site cards while upstream server failures remain errors',async()=>{ + for(const status of [401,404,410]){ + const f=await fixture(async()=>new Response('',{status}));const r=await f.request();assert.equal(r.status,200);assert.equal((await r.json()).upstreamStatus,status); + } + let calls=0;const f=await fixture(async()=>{calls++;return new Response('',{status:500});}); + assert.equal((await f.request()).status,502);assert.equal((await f.request()).status,502);assert.equal(calls,2); +}); + +test('video links cancel the response without reading its body as HTML',async()=>{ + let cancelled=0; + const f=await fixture(async()=>({ok:true,status:200,headers:new Headers({'Content-Type':'video/mp4'}),body:{cancel:async()=>{cancelled++;},getReader(){throw new Error('Video body must not be read');}}})); + const d=await(await f.request('https://example.org/movie.mp4')).json(); + assert.equal(d.reason,'not_html');assert.equal(d.unavailable,true);assert.equal(cancelled,1); +}); + +test('oversized HTML is bounded and the remaining response is cancelled',async()=>{ + let emitted=0,cancelled=false; + const body=new ReadableStream({pull(controller){emitted+=4096;controller.enqueue(new TextEncoder().encode('Bounded'+ ' '.repeat(4096-22)));},cancel(){cancelled=true;}}); + const f=await fixture(async()=>new Response(body,{headers:{'Content-Type':'text/html'}})); + const d=await(await f.request()).json();assert.equal(d.title,'Bounded');assert.equal(cancelled,true);assert.ok(emitted<100000,'must not drain an unbounded body'); +}); + +test('the fetch deadline remains active while reading a stalled HTML body',async()=>{ + let bodyController; + const f=await fixture(async(_url,{signal})=>{ + const body=new ReadableStream({start(controller){bodyController=controller;}}); + signal.addEventListener('abort',()=>bodyController.error(new DOMException('Aborted','AbortError')),{once:true}); + return new Response(body,{headers:{'Content-Type':'text/html'}}); + }); + const pending=f.request();await new Promise(r=>setImmediate(r)); + assert.equal(f.timers.size,1);[...f.timers.values()][0](); + const response=await pending;assert.equal(response.status,504);assert.equal((await response.json()).error,'Request timed out');assert.equal(f.timers.size,0); +}); + +test('invalid links stay client errors without an upstream request',async()=>{ + const f=await fixture(()=>{throw new Error('No network expected');}); + assert.equal((await f.request('file:///etc/passwd')).status,400); +}); + +const chat=await readFile(new URL('../public/aesthetic.computer/disks/chat.mjs',import.meta.url),'utf8'); +const clientSource=chat.slice(chat.indexOf('async function loadOgPreview('),chat.indexOf('async function loadYoutubePreview(')); +for(const unavailable of [true,false])test(`chat ${unavailable?'skips unavailable-site icons':'preserves normal favicon fallback'}`,async()=>{ + const context=vm.createContext({URL,Response,ogPreviewCache:new Map(),globalOgPreviewCache:new Map(),ogLoadQueue:new Set(),console, + fetch:async()=>Response.json({url:'https://example.org/article',title:'example.org',image:null,favicon:null,unavailable})}); + vm.runInContext(clientSource,context); + const loaded=[];const result=await context.loadOgPreview('https://example.org/article',async url=>{loaded.push(url);return {img:'icon'};}); + assert.equal(result.url,'https://example.org/article');assert.equal(result.title,'example.org');assert.equal(loaded.length,unavailable?0:1);assert.equal(context.ogLoadQueue.size,0); +});