Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275import * as Effect from "effect/Effect";import type { BrowserBinding } from "agents/browser";import type { WebResult } from "../shared/web";import { WebDeadline, withResearchBrowser, type BrowserLeaseHooks,} from "./browser-session";import { WebFailure, webValidation } from "./web-errors";import { cleanText, publicWebUrl, webSource } from "./web-source";
export const BROWSER_TIMEOUT = 30_000;export const browserProgress = [ "Starting browser", "Loading webpage", "Waiting for page content", "Reading rendered page", "Closing browser",] as const;
// Only host-authored code runs. Selectors are serialized data, and extraction// uses an isolated world so page scripts cannot replace the host's DOM methods.export function pageExpression(selector?: string) { return `(() => { let matched = true; try { matched = !${JSON.stringify(selector ?? "")} || !!document.querySelector(${JSON.stringify(selector ?? "")}); } catch { return { invalidSelector: true }; } const root = document.querySelector('article, main') || document.body; const text = root?.innerText || ''; return { ready: document.readyState === 'complete', matched, title: document.title.slice(0, 240), content: text.slice(0, 16000), truncated: text.length > 16000, links: Array.from(document.querySelectorAll('a[href]')).slice(0, 100).map(a => ({ url: a.href.slice(0, 4097), title: (a.innerText || a.textContent || '').slice(0, 240) })) }; })()`;}type Page = { ready: boolean; matched: boolean; invalidSelector?: boolean; title: string; content: string; truncated: boolean; links: { url: string; title: string }[];};
export function readBrowserPage( browser: BrowserBinding, input: { url: string; waitForSelector?: string }, signal: AbortSignal | undefined, keepAlive: (promise: Promise<unknown>) => void, lease?: BrowserLeaseHooks, progress: (stage: number) => void = () => {},): Effect.Effect<WebResult, WebFailure> { return Effect.acquireUseRelease( Effect.sync(() => new WebDeadline(BROWSER_TIMEOUT, signal)), (deadline) => Effect.gen(function* () { const requestedUrl = yield* webValidation( () => publicWebUrl(input.url).href, ); progress(0); return yield* withResearchBrowser( browser, deadline, keepAlive, (cdp) => Effect.gen(function* () { const { targetId } = (yield* cdp.send("Target.createTarget", { url: "about:blank", })) as { targetId: string }; const { sessionId } = (yield* cdp.send("Target.attachToTarget", { targetId, flatten: true, })) as { sessionId: string }; const initial = (yield* cdp.send( "Page.getFrameTree", {}, sessionId, )) as { frameTree: { frame: { id: string } } }; const mainFrameId = initial.frameTree.frame.id; let status: number | undefined; let blockedNavigation = false; cdp.onEvent((event) => { if ( event.method === "Target.attachedToTarget" && event.params.targetInfo.targetId !== targetId ) { // Additional pages/workers stay paused and are closed. Research does // not need popups or a second uncontrolled script execution target. keepAlive( Effect.runPromise( cdp .send("Target.closeTarget", { targetId: event.params.targetInfo.targetId, }) .pipe(Effect.catchTag("WebFailure", () => Effect.void)), ), ); } if (event.sessionId !== sessionId) return; if ( event.method === "Network.responseReceived" && event.params.type === "Document" && event.params.frameId === mainFrameId ) status = event.params.response.status; if (event.method !== "Fetch.requestPaused") return; const { requestId, request, resourceType } = event.params; let allowed = true; try { publicWebUrl(request.url); } catch { allowed = false; } keepAlive( Effect.runPromise( cdp .send( allowed ? "Fetch.continueRequest" : "Fetch.failRequest", { requestId, ...(allowed ? {} : { errorReason: "BlockedByClient" }), }, sessionId, ) .pipe(Effect.catchTag("WebFailure", () => Effect.void)), ), ); if (!allowed && resourceType === "Document") blockedNavigation = true; }); yield* cdp.send("Target.setAutoAttach", { autoAttach: true, waitForDebuggerOnStart: true, flatten: true, }); yield* cdp.send("Page.enable", {}, sessionId); yield* cdp.send("Network.enable", {}, sessionId); yield* cdp.send( "Network.setBypassServiceWorker", { bypass: true }, sessionId, ); yield* cdp.send( "Fetch.enable", { patterns: [{ urlPattern: "*", requestStage: "Request" }] }, sessionId, ); progress(1); const navigation = (yield* cdp.send( "Page.navigate", { url: requestedUrl }, sessionId, )) as { errorText?: string }; if (blockedNavigation) return yield* Effect.fail(new WebFailure("blocked_url")); if (navigation.errorText) return yield* Effect.fail(new WebFailure("request_failed")); progress(2); let page: Page; let finalUrl: string; let readyAt: number | undefined; let stableAt = Date.now(); let previousContent = ""; while (true) { if (blockedNavigation) return yield* Effect.fail(new WebFailure("blocked_url")); const { frameTree } = (yield* cdp.send( "Page.getFrameTree", {}, sessionId, )) as { frameTree: { frame: { id: string; url: string } } }; // Protocol frame metadata is provenance. Canonical tags and page JS // cannot relabel retrieved content as another site's evidence. if (frameTree.frame.url !== "about:blank") yield* webValidation(() => publicWebUrl(frameTree.frame.url)); const { executionContextId } = (yield* cdp.send( "Page.createIsolatedWorld", { frameId: frameTree.frame.id, worldName: "flarebot-research", }, sessionId, )) as { executionContextId: number }; const evaluated = (yield* cdp.send( "Runtime.evaluate", { expression: pageExpression(input.waitForSelector), contextId: executionContextId, returnByValue: true, }, sessionId, )) as { result?: { value?: Page }; exceptionDetails?: unknown }; if (!evaluated.result?.value || evaluated.exceptionDetails) return yield* Effect.fail(new WebFailure("request_failed")); page = evaluated.result.value; const snapshot = JSON.stringify([ page.title, page.content, frameTree.frame.url, ]); if (snapshot !== previousContent) { previousContent = snapshot; stableAt = Date.now(); } if (page.ready) readyAt ??= Date.now(); const settled = input.waitForSelector ? page.matched : readyAt !== undefined && Date.now() - readyAt >= 1000 && Date.now() - stableAt >= 500; if (page.invalidSelector) return yield* Effect.fail(new WebFailure("invalid_selector")); const after = (yield* cdp.send( "Page.getFrameTree", {}, sessionId, )) as { frameTree: typeof frameTree }; if ( page.ready && settled && frameTree.frame.url !== "about:blank" && after.frameTree.frame.url === frameTree.frame.url ) { finalUrl = yield* webValidation( () => publicWebUrl(after.frameTree.frame.url).href, ); break; } yield* Effect.sleep(100); } if (status !== undefined && status >= 400) return yield* Effect.fail(new WebFailure("http_error", status)); progress(3); const links: { url: string; title: string }[] = []; const seen = new Set<string>(); for (const link of page.links) { try { const url = publicWebUrl(link.url).href; if (seen.has(url)) continue; seen.add(url); links.push({ url, title: cleanText(link.title, 240) || new URL(url).hostname, }); if (links.length === 20) break; } catch { /* Nonpublic destinations are not research follow-ups. */ } } const source = yield* webSource({ requestedUrl, finalUrl, sourceKind: "browser", title: cleanText(page.title, 240) || new URL(finalUrl).hostname, content: cleanText(page.content, 16000), truncated: page.truncated, links, }); progress(4); return { ok: true as const, sources: [source] }; }), lease, ); }), (deadline) => Effect.sync(() => deadline.dispose()), );}