Something went wrong. Try again.
source dump of claude code forked from oppi.li/claude-code
Something went wrong. Try again.
17 kB · 530 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531import axios, { type AxiosResponse } from 'axios'import { LRUCache } from 'lru-cache'import { type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, logEvent,} from '../../services/analytics/index.js'import { queryHaiku } from '../../services/api/claude.js'import { AbortError } from '../../utils/errors.js'import { getWebFetchUserAgent } from '../../utils/http.js'import { logError } from '../../utils/log.js'import { isBinaryContentType, persistBinaryContent,} from '../../utils/mcpOutputStorage.js'import { getSettings_DEPRECATED } from '../../utils/settings/settings.js'import { asSystemPrompt } from '../../utils/systemPromptType.js'import { isPreapprovedHost } from './preapproved.js'import { makeSecondaryModelPrompt } from './prompt.js'
// Custom error classes for domain blockingclass DomainBlockedError extends Error { constructor(domain: string) { super(`Claude Code is unable to fetch from ${domain}`) this.name = 'DomainBlockedError' }}
class DomainCheckFailedError extends Error { constructor(domain: string) { super( `Unable to verify if domain ${domain} is safe to fetch. This may be due to network restrictions or enterprise security policies blocking claude.ai.`, ) this.name = 'DomainCheckFailedError' }}
class EgressBlockedError extends Error { constructor(public readonly domain: string) { super( JSON.stringify({ error_type: 'EGRESS_BLOCKED', domain, message: `Access to ${domain} is blocked by the network egress proxy.`, }), ) this.name = 'EgressBlockedError' }}
// Cache for storing fetched URL contenttype CacheEntry = { bytes: number code: number codeText: string content: string contentType: string persistedPath?: string persistedSize?: number}
// Cache with 15-minute TTL and 50MB size limit// LRUCache handles automatic expiration and evictionconst CACHE_TTL_MS = 15 * 60 * 1000 // 15 minutesconst MAX_CACHE_SIZE_BYTES = 50 * 1024 * 1024 // 50MB
const URL_CACHE = new LRUCache<string, CacheEntry>({ maxSize: MAX_CACHE_SIZE_BYTES, ttl: CACHE_TTL_MS,})
// Separate cache for preflight domain checks. URL_CACHE is URL-keyed, so// fetching two paths on the same domain triggers two identical preflight// HTTP round-trips to api.anthropic.com. This hostname-keyed cache avoids// that. Only 'allowed' is cached — blocked/failed re-check on next attempt.const DOMAIN_CHECK_CACHE = new LRUCache<string, true>({ max: 128, ttl: 5 * 60 * 1000, // 5 minutes — shorter than URL_CACHE TTL})
export function clearWebFetchCache(): void { URL_CACHE.clear() DOMAIN_CHECK_CACHE.clear()}
// Lazy singleton — defers the turndown → @mixmark-io/domino import (~1.4MB// retained heap) until the first HTML fetch, and reuses one instance across// calls (construction builds 15 rule objects; .turndown() is stateless).// @types/turndown ships only `export =` (no .d.mts), so TS types the import// as the class itself while Bun wraps CJS in { default } — hence the cast.type TurndownCtor = typeof import('turndown')let turndownServicePromise: Promise<InstanceType<TurndownCtor>> | undefinedfunction getTurndownService(): Promise<InstanceType<TurndownCtor>> { return (turndownServicePromise ??= import('turndown').then(m => { const Turndown = (m as unknown as { default: TurndownCtor }).default return new Turndown() }))}
// PSR requested limiting the length of URLs to 250 to lower the potential// for a data exfiltration. However, this is too restrictive for some customers'// legitimate use cases, such as JWT-signed URLs (e.g., cloud service signed URLs)// that can be much longer. We already require user approval for each domain,// which provides a primary security boundary. In addition, Claude Code has// other data exfil channels, and this one does not seem relatively high risk,// so I'm removing that length restriction. -abconst MAX_URL_LENGTH = 2000
// Per PSR:// "Implement resource consumption controls because setting limits on CPU,// memory, and network usage for the Web Fetch tool can prevent a single// request or user from overwhelming the system."const MAX_HTTP_CONTENT_LENGTH = 10 * 1024 * 1024
// Timeout for the main HTTP fetch request (60 seconds).// Prevents hanging indefinitely on slow/unresponsive servers.const FETCH_TIMEOUT_MS = 60_000
// Timeout for the domain blocklist preflight check (10 seconds).const DOMAIN_CHECK_TIMEOUT_MS = 10_000
// Cap same-host redirect hops. Without this a malicious server can return// a redirect loop (/a → /b → /a …) and the per-request FETCH_TIMEOUT_MS// resets on every hop, hanging the tool until user interrupt. 10 matches// common client defaults (axios=5, follow-redirects=21, Chrome=20).const MAX_REDIRECTS = 10
// Truncate to not spend too many tokensexport const MAX_MARKDOWN_LENGTH = 100_000
export function isPreapprovedUrl(url: string): boolean { try { const parsedUrl = new URL(url) return isPreapprovedHost(parsedUrl.hostname, parsedUrl.pathname) } catch { return false }}
export function validateURL(url: string): boolean { if (url.length > MAX_URL_LENGTH) { return false }
let parsed try { parsed = new URL(url) } catch { return false }
// We don't need to check protocol here, as we'll upgrade http to https when making the request
// As long as we aren't supporting aiming to cookies or internal domains, // we should block URLs with usernames/passwords too, even though these // seem exceedingly unlikely. if (parsed.username || parsed.password) { return false }
// Initial filter that this isn't a privileged, company-internal URL // by checking that the hostname is publicly resolvable const hostname = parsed.hostname const parts = hostname.split('.') if (parts.length < 2) { return false }
return true}
type DomainCheckResult = | { status: 'allowed' } | { status: 'blocked' } | { status: 'check_failed'; error: Error }
export async function checkDomainBlocklist( domain: string,): Promise<DomainCheckResult> { if (DOMAIN_CHECK_CACHE.has(domain)) { return { status: 'allowed' } } try { const response = await axios.get( `https://api.anthropic.com/api/web/domain_info?domain=${encodeURIComponent(domain)}`, { timeout: DOMAIN_CHECK_TIMEOUT_MS }, ) if (response.status === 200) { if (response.data.can_fetch === true) { DOMAIN_CHECK_CACHE.set(domain, true) return { status: 'allowed' } } return { status: 'blocked' } } // Non-200 status but didn't throw return { status: 'check_failed', error: new Error(`Domain check returned status ${response.status}`), } } catch (e) { logError(e) return { status: 'check_failed', error: e as Error } }}
/** * Check if a redirect is safe to follow * Allows redirects that: * - Add or remove "www." in the hostname * - Keep the origin the same but change path/query params * - Or both of the above */export function isPermittedRedirect( originalUrl: string, redirectUrl: string,): boolean { try { const parsedOriginal = new URL(originalUrl) const parsedRedirect = new URL(redirectUrl)
if (parsedRedirect.protocol !== parsedOriginal.protocol) { return false }
if (parsedRedirect.port !== parsedOriginal.port) { return false }
if (parsedRedirect.username || parsedRedirect.password) { return false }
// Now check hostname conditions // 1. Adding www. is allowed: example.com -> www.example.com // 2. Removing www. is allowed: www.example.com -> example.com // 3. Same host (with or without www.) is allowed: paths can change const stripWww = (hostname: string) => hostname.replace(/^www\./, '') const originalHostWithoutWww = stripWww(parsedOriginal.hostname) const redirectHostWithoutWww = stripWww(parsedRedirect.hostname) return originalHostWithoutWww === redirectHostWithoutWww } catch (_error) { return false }}
/** * Helper function to handle fetching URLs with custom redirect handling * Recursively follows redirects if they pass the redirectChecker function * * Per PSR: * "Do not automatically follow redirects because following redirects could * allow for an attacker to exploit an open redirect vulnerability in a * trusted domain to force a user to make a request to a malicious domain * unknowingly" */type RedirectInfo = { type: 'redirect' originalUrl: string redirectUrl: string statusCode: number}
export async function getWithPermittedRedirects( url: string, signal: AbortSignal, redirectChecker: (originalUrl: string, redirectUrl: string) => boolean, depth = 0,): Promise<AxiosResponse<ArrayBuffer> | RedirectInfo> { if (depth > MAX_REDIRECTS) { throw new Error(`Too many redirects (exceeded ${MAX_REDIRECTS})`) } try { return await axios.get(url, { signal, timeout: FETCH_TIMEOUT_MS, maxRedirects: 0, responseType: 'arraybuffer', maxContentLength: MAX_HTTP_CONTENT_LENGTH, headers: { Accept: 'text/markdown, text/html, */*', 'User-Agent': getWebFetchUserAgent(), }, }) } catch (error) { if ( axios.isAxiosError(error) && error.response && [301, 302, 307, 308].includes(error.response.status) ) { const redirectLocation = error.response.headers.location if (!redirectLocation) { throw new Error('Redirect missing Location header') }
// Resolve relative URLs against the original URL const redirectUrl = new URL(redirectLocation, url).toString()
if (redirectChecker(url, redirectUrl)) { // Recursively follow the permitted redirect return getWithPermittedRedirects( redirectUrl, signal, redirectChecker, depth + 1, ) } else { // Return redirect information to the caller return { type: 'redirect', originalUrl: url, redirectUrl, statusCode: error.response.status, } } }
// Detect egress proxy blocks: the proxy returns 403 with // X-Proxy-Error: blocked-by-allowlist when egress is restricted if ( axios.isAxiosError(error) && error.response?.status === 403 && error.response.headers['x-proxy-error'] === 'blocked-by-allowlist' ) { const hostname = new URL(url).hostname throw new EgressBlockedError(hostname) }
throw error }}
function isRedirectInfo( response: AxiosResponse<ArrayBuffer> | RedirectInfo,): response is RedirectInfo { return 'type' in response && response.type === 'redirect'}
export type FetchedContent = { content: string bytes: number code: number codeText: string contentType: string persistedPath?: string persistedSize?: number}
export async function getURLMarkdownContent( url: string, abortController: AbortController,): Promise<FetchedContent | RedirectInfo> { if (!validateURL(url)) { throw new Error('Invalid URL') }
// Check cache (LRUCache handles TTL automatically) const cachedEntry = URL_CACHE.get(url) if (cachedEntry) { return { bytes: cachedEntry.bytes, code: cachedEntry.code, codeText: cachedEntry.codeText, content: cachedEntry.content, contentType: cachedEntry.contentType, persistedPath: cachedEntry.persistedPath, persistedSize: cachedEntry.persistedSize, } }
let parsedUrl: URL let upgradedUrl = url
try { parsedUrl = new URL(url)
// Upgrade http to https if needed if (parsedUrl.protocol === 'http:') { parsedUrl.protocol = 'https:' upgradedUrl = parsedUrl.toString() }
const hostname = parsedUrl.hostname
// Check if the user has opted to skip the blocklist check // This is for enterprise customers with restrictive security policies // that prevent outbound connections to claude.ai const settings = getSettings_DEPRECATED() if (!settings.skipWebFetchPreflight) { const checkResult = await checkDomainBlocklist(hostname) switch (checkResult.status) { case 'allowed': // Continue with the fetch break case 'blocked': throw new DomainBlockedError(hostname) case 'check_failed': throw new DomainCheckFailedError(hostname) } }
if (process.env.USER_TYPE === 'ant') { logEvent('tengu_web_fetch_host', { hostname: hostname as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, }) } } catch (e) { if ( e instanceof DomainBlockedError || e instanceof DomainCheckFailedError ) { // Expected user-facing failures - re-throw without logging as internal error throw e } logError(e) }
const response = await getWithPermittedRedirects( upgradedUrl, abortController.signal, isPermittedRedirect, )
// Check if we got a redirect response if (isRedirectInfo(response)) { return response }
const rawBuffer = Buffer.from(response.data) // Release the axios-held ArrayBuffer copy; rawBuffer owns the bytes now. // This lets GC reclaim up to MAX_HTTP_CONTENT_LENGTH (10MB) before Turndown // builds its DOM tree (which can be 3-5x the HTML size). ;(response as { data: unknown }).data = null const contentType = response.headers['content-type'] ?? ''
// Binary content: save raw bytes to disk with a proper extension so Claude // can inspect the file later. We still fall through to the utf-8 decode + // Haiku path below — for PDFs in particular the decoded string has enough // ASCII structure (/Title, text streams) that Haiku can summarize it, and // the saved file is a supplement rather than a replacement. let persistedPath: string | undefined let persistedSize: number | undefined if (isBinaryContentType(contentType)) { const persistId = `webfetch-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` const result = await persistBinaryContent(rawBuffer, contentType, persistId) if (!('error' in result)) { persistedPath = result.filepath persistedSize = result.size } }
const bytes = rawBuffer.length const htmlContent = rawBuffer.toString('utf-8')
let markdownContent: string let contentBytes: number if (contentType.includes('text/html')) { markdownContent = (await getTurndownService()).turndown(htmlContent) contentBytes = Buffer.byteLength(markdownContent) } else { // It's not HTML - just use it raw. The decoded string's UTF-8 byte // length equals rawBuffer.length (modulo U+FFFD replacement on invalid // bytes — negligible for cache eviction accounting), so skip the O(n) // Buffer.byteLength scan. markdownContent = htmlContent contentBytes = bytes }
// Store the fetched content in cache. Note that it's stored under // the original URL, not the upgraded or redirected URL. const entry: CacheEntry = { bytes, code: response.status, codeText: response.statusText, content: markdownContent, contentType, persistedPath, persistedSize, } // lru-cache requires positive integers; clamp to 1 for empty responses. URL_CACHE.set(url, entry, { size: Math.max(1, contentBytes) }) return entry}
export async function applyPromptToMarkdown( prompt: string, markdownContent: string, signal: AbortSignal, isNonInteractiveSession: boolean, isPreapprovedDomain: boolean,): Promise<string> { // Truncate content to avoid "Prompt is too long" errors from the secondary model const truncatedContent = markdownContent.length > MAX_MARKDOWN_LENGTH ? markdownContent.slice(0, MAX_MARKDOWN_LENGTH) + '\n\n[Content truncated due to length...]' : markdownContent
const modelPrompt = makeSecondaryModelPrompt( truncatedContent, prompt, isPreapprovedDomain, ) const assistantMessage = await queryHaiku({ systemPrompt: asSystemPrompt([]), userPrompt: modelPrompt, signal, options: { querySource: 'web_fetch_apply', agents: [], isNonInteractiveSession, hasAppendSystemPrompt: false, mcpTools: [], }, })
// We need to bubble this up, so that the tool call throws, causing us to return // an is_error tool_use block to the server, and render a red dot in the UI. if (signal.aborted) { throw new AbortError() }
const { content } = assistantMessage.message if (content.length > 0) { const contentBlock = content[0] if ('text' in contentBlock!) { return contentBlock.text } } return 'No response from model'}