Something went wrong. Try again.
opencode v2 mirror
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190export * as WebFetchTool from "./webfetch.js"
import type { Context } from "@opencode/plugin/effect/plugin"import { ToolFailure } from "@opencode/ai"import { Duration, Effect, Schema } from "effect"import { HttpClient, type HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"import { Parser } from "htmlparser2"import { Permission } from "../../permission.js"import { convertHTMLToMarkdown, MAX_MARKDOWN_BYTES } from "../html-markdown.js"import { collectBoundedResponseBody } from "../http-body.js"
export const name = "webfetch"export const MAX_RESPONSE_BYTES = MAX_MARKDOWN_BYTESexport const DEFAULT_TIMEOUT_SECONDS = 30export const MAX_TIMEOUT_SECONDS = 120
export const description = `Fetch content from an HTTP or HTTPS URL and return it as text, markdown, or HTML. Markdown is the default.
Use a more targeted tool when one is available. This tool is read-only. Large text results may be replaced with a preview while the complete output is retained in managed storage.`
const Timeout = Schema.Finite.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(MAX_TIMEOUT_SECONDS))
export const Input = Schema.Struct({ url: Schema.String.annotate({ description: "The HTTP or HTTPS URL to fetch content from" }), format: Schema.Literals(["text", "markdown", "html"]) .annotate({ description: "The format to return the content in. Defaults to markdown." }) .pipe(Schema.withDecodingDefaultKey(Effect.succeed("markdown" as const))), timeout: Schema.optionalKey(Timeout).annotate({ description: `Optional timeout in seconds (maximum: ${MAX_TIMEOUT_SECONDS})`, }),})
const Output = Schema.Struct({ url: Schema.String, contentType: Schema.String, format: Input.fields.format, output: Schema.String,})type Format = (typeof Input.Type)["format"]
const acceptHeader = (format: Format) => { switch (format) { case "markdown": return "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" case "text": return "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" case "html": return "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" }}
const headers = (format: Format, userAgent: string) => ({ "User-Agent": userAgent, Accept: acceptHeader(format), "Accept-Language": "en-US,en;q=0.9",})
const openCodeUserAgent = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OpenCode-User/1.0; +https://opencode.ai"
const isCloudflareChallenge = (error: HttpClientError.HttpClientError) => { if (error.reason._tag !== "StatusCodeError") return false const response = error.reason.response return response.status === 403 && response.headers["cf-mitigated"] === "challenge"}
const request = (url: string, format: Format, userAgent = openCodeUserAgent) => HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers(format, userAgent)))
const assertHttpUrl = (url: URL) => { if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("URL must use http:// or https://")}
const execute = (http: HttpClient.HttpClient, url: string, format: Format, userAgent = openCodeUserAgent) => http.execute(request(url, format, userAgent)).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk))
const collectBody = (response: HttpClientResponse.HttpClientResponse) => collectBoundedResponseBody( response, MAX_RESPONSE_BYTES, () => new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`), )
const mimeFrom = (contentType: string) => contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""const isImageAttachment = (mime: string) => mime.startsWith("image/") && mime !== "image/svg+xml" && mime !== "image/vnd.fastbidsheet"const isTextualMime = (mime: string) => !mime || mime.startsWith("text/") || mime === "application/json" || mime.endsWith("+json") || mime === "application/xml" || mime.endsWith("+xml") || mime === "application/javascript" || mime === "application/x-javascript"const convert = (content: string, contentType: string, format: Format) => { if (!contentType.includes("text/html")) return content if (format === "markdown") return convertHTMLToMarkdown(content) if (format === "text") return extractTextFromHTML(content) return content}
export const Plugin = { id: "opencode.tool.webfetch", effect: Effect.fn("WebFetchTool.Plugin")(function* (ctx: Context) { const http = yield* HttpClient.HttpClient const permission = yield* Permission.Service
yield* ctx.tool .transform((editor) => editor.add({ name, options: { codemode: false }, description, input: Input, output: Output, execute: (input, context) => Effect.gen(function* () { yield* Effect.try({ try: () => assertHttpUrl(new URL(input.url)), catch: (error) => error, })
yield* permission.assert({ action: name, resources: [input.url], save: ["*"], metadata: input, sessionID: context.sessionID, agent: context.agent, source: { type: "tool", messageID: context.messageID, id: context.id }, })
const { body, contentType } = yield* Effect.gen(function* () { const response = yield* execute(http, input.url, input.format).pipe( Effect.catchIf(isCloudflareChallenge, () => execute(http, input.url, input.format, "opencode")), ) const contentType = response.headers["content-type"] || "" const mime = mimeFrom(contentType) if (isImageAttachment(mime)) return yield* Effect.fail(new Error(`Unsupported fetched image content type: ${mime}`)) if (!isTextualMime(mime)) return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`)) return { body: yield* collectBody(response), contentType } }).pipe( Effect.timeoutOrElse({ duration: Duration.seconds(input.timeout ?? DEFAULT_TIMEOUT_SECONDS), orElse: () => Effect.fail(new Error("Request timed out")), }), ) const content = new TextDecoder().decode(body) const output = yield* Effect.try({ try: () => convert(content, contentType, input.format), catch: (error) => error, }) const result = { url: input.url, contentType, format: input.format, output, } return { output: result, content: result.output, metadata: { contentType: result.contentType } } }).pipe(Effect.mapError((error) => new ToolFailure({ message: `Unable to fetch ${input.url}`, error }))), }), ) .pipe(Effect.orDie) }),}
export function extractTextFromHTML(html: string) { let text = "" let skipDepth = 0 const parser = new Parser({ onopentag(name) { if (skipDepth > 0 || ["script", "style", "noscript", "iframe", "object", "embed"].includes(name)) skipDepth++ }, ontext(input) { if (skipDepth === 0) text += input }, onclosetag() { if (skipDepth > 0) skipDepth-- }, }) parser.write(html) parser.end() return text.trim()}
export { convertHTMLToMarkdown }