import { Parser } from "htmlparser2" const omitted = new Set(["script", "style", "noscript", "iframe", "object", "embed", "meta", "link", "template"]) const blocks = new Set([ "address", "article", "aside", "details", "dialog", "div", "dl", "fieldset", "figcaption", "figure", "footer", "form", "header", "main", "nav", "p", "section", "summary", ]) type Frame = { suppressed: boolean link?: { href: string; title?: string } suspendedLink?: Frame["link"] marker?: { index: number; block: number; leadingSpace?: boolean; previous?: Frame["marker"] } code?: { inline: boolean; text: string; language?: string } linkCode?: NonNullable resumedCode?: NonNullable list?: { ordered: boolean; next: number; previous?: Frame["list"] } item?: { indent: string; previous?: Frame["item"] } table?: { start: number rows: string[][] row?: string[] caption?: string fallback: boolean previous?: Frame["table"] } cell?: { start: number } caption?: { start: number } details?: { open: boolean; summary: boolean; previous?: Frame["details"] } } type Chunk = string | { raw: string } export const MAX_MARKDOWN_BYTES = 5 * 1024 * 1024 const CONTENT_BYTES = MAX_MARKDOWN_BYTES - 64 * 1024 export function convertHTMLToMarkdown(html: string) { const output: Chunk[] = [] const stack: Frame[] = [] const encoder = new TextEncoder() let pendingSpace = false let pendingIndent = "" let last = "" let quoteDepth = 0 let needsQuotePrefix = false let blockCount = 0 let depth = 0 let stopped = false let outputBytes = 0 let activeCode: NonNullable | undefined let activeLink: Frame["link"] | undefined let linkOpen = false let activeMarker: Frame["marker"] | undefined let activeList: Frame["list"] | undefined let activeItem: Frame["item"] | undefined let activeTable: NonNullable | undefined let activeCell: Frame["cell"] | undefined let tableDepth = 0 let fallbackSuppressedDepth = 0 let fallbackOmittedDepth = 0 let activeDetails: Frame["details"] | undefined const sliceBytes = (value: string, bytes: number) => { if (encoder.encode(value).byteLength <= bytes) return value const characters = Array.from(value) let low = 0 let high = characters.length while (low < high) { const middle = Math.ceil((low + high) / 2) if (encoder.encode(characters.slice(0, middle).join("")).byteLength <= bytes) low = middle else high = middle - 1 } return characters.slice(0, low).join("") } const append = (value: string, content = false) => { const limit = content ? CONTENT_BYTES : MAX_MARKDOWN_BYTES if (!value || outputBytes >= limit) return const bytes = encoder.encode(value) const remaining = limit - outputBytes const next = bytes.byteLength <= remaining ? value : sliceBytes(value, remaining) output.push(next) outputBytes += bytes.byteLength <= remaining ? bytes.byteLength : encoder.encode(next).byteLength last = next.at(-1) ?? last } const appendRaw = (value: string) => { const before = output.length append(value) if (output.length > before) output[output.length - 1] = { raw: output[output.length - 1] as string } } const take = (start: number) => { const chunks = output.splice(start) const value = chunks.map((chunk) => (typeof chunk === "string" ? chunk : chunk.raw)).join("") outputBytes -= encoder.encode(value).byteLength return value } const quotePrefix = () => "> ".repeat(Math.min(8, quoteDepth)) const prefixQuote = () => { if (!needsQuotePrefix || quoteDepth === 0 || activeCell) return append(quotePrefix()) needsQuotePrefix = false } const flushSpace = () => { if (!pendingSpace) return const marker = activeMarker if (marker && output.length === marker.index + 1 && last !== " " && last !== "\n") { const value = output[marker.index] if (typeof value === "string") output[marker.index] = ` ${value}` outputBytes++ marker.leadingSpace = true pendingSpace = false return } if (last && last !== "\n" && last !== " ") append(" ") pendingSpace = false } const inline = (value: string, open = false) => { if (open) flushSpace() prefixQuote() if (pendingIndent) { append(pendingIndent) pendingIndent = "" } if (activeLink && !linkOpen) { append("[") linkOpen = true } append(value) } const block = () => { if (activeLink && linkOpen) { append(`](${destination(activeLink.href)}${title(activeLink.title)})`) linkOpen = false } pendingSpace = false append("\n\n") blockCount++ needsQuotePrefix = quoteDepth > 0 pendingIndent = activeItem?.indent ?? "" } const suspendLink = (frame: Frame) => { if (!activeLink) return frame.suspendedLink = activeLink block() activeLink = undefined linkOpen = false } const text = (value: string) => { if (activeCode) { activeCode.text += value return } for (const part of value.split(/([\t\n\f\r ]+)/)) { if (!part) continue if (/^[\t\n\f\r ]+$/.test(part)) { pendingSpace = true continue } flushSpace() prefixQuote() if (pendingIndent) { append(pendingIndent) pendingIndent = "" } if (activeLink && !linkOpen) { append("[") linkOpen = true } const escaped = part .replace(/([\\`*_[\]<>|])/g, "\\$1") .replace(/~/g, "\\~") .replace(/^([#+-])/, "\\$1") .replace(/^(\d+)\./, "$1\\.") append(escaped, true) } } const destination = (value: string) => value.replace(/([\\()])/g, "\\$1").replace(/[\t\n\r ]+/g, "%20") const title = (value: string | undefined) => value ? ` "${value .replace(/[\t\n\r ]+/g, " ") .trim() .replace(/([\\"])/g, "\\$1")}"` : "" const inlineMarker = (name: string) => { if (name === "strong" || name === "b") return "**" if (name === "em" || name === "i") return "*" if (name === "s" || name === "strike" || name === "del") return "~~" return undefined } const finishCode = (code: NonNullable) => { if (code.inline && !code.text) return let backticks = 0 let tildes = 0 let currentBackticks = 0 let currentTildes = 0 for (const character of code.text) { currentBackticks = character === "`" ? currentBackticks + 1 : 0 currentTildes = character === "~" ? currentTildes + 1 : 0 backticks = Math.max(backticks, currentBackticks) tildes = Math.max(tildes, currentTildes) } if (code.inline) { const fence = "`".repeat(Math.max(1, backticks + 1)) flushSpace() prefixQuote() const available = Math.max(0, CONTENT_BYTES - outputBytes - fence.length * 2) let payload = sliceBytes(code.text, available) while (payload) { const padding = /^[ `]|[ `]$/.test(payload) && !/^ +$/.test(payload) ? " " : "" const bytes = encoder.encode(payload).byteLength if (bytes + padding.length * 2 <= available) { appendRaw(`${fence}${padding}${payload}${padding}${fence}`) return } // Padding costs at most two bytes, so at most two whole-code-point trims are needed. payload = sliceBytes(payload, bytes - 1) } return } if (activeCell) { text(code.text) return } const marker = backticks <= tildes ? "`" : "~" const length = Math.max(3, (marker === "`" ? backticks : tildes) + 1) const fence = marker.repeat(length) block() const prefix = `${fence}${code.language ?? ""}\n` const quote = quoteDepth > 0 ? quotePrefix() : "" let payload = code.text for (;;) { const candidate = `${prefix}${payload}${payload.endsWith("\n") ? "" : "\n"}${fence}` const value = quote ? candidate.replace(/^/gm, quote) : candidate const valueBytes = encoder.encode(value).byteLength if (outputBytes + valueBytes <= CONTENT_BYTES) { appendRaw(value) block() return } // No amount of trimming can make the empty fenced block fit. if (!payload) return const excess = valueBytes - Math.max(0, CONTENT_BYTES - outputBytes) payload = sliceBytes(payload, Math.max(0, encoder.encode(payload).byteLength - Math.ceil(excess))) } } const parser = new Parser({ onopentag(name, attributes) { depth++ if (depth > 10_000) { if (stack.at(-1)?.suppressed) { const visibleParent = stack.findLastIndex((frame) => !frame.suppressed) fallbackSuppressedDepth = visibleParent + 2 } activeCode = undefined stopped = true } if (stopped) { if (omitted.has(name)) fallbackOmittedDepth++ else pendingSpace = true return } const suppressed = (stack.at(-1)?.suppressed ?? false) || omitted.has(name) const frame: Frame = { suppressed } const hidden = "hidden" in attributes || attributes["aria-hidden"]?.toLowerCase() === "true" || name === "head" const details = activeDetails if (hidden || (details && !details.open && !details.summary && name !== "summary")) frame.suppressed = true stack.push(frame) if (frame.suppressed) return if (activeCode && !activeCode.inline) { if (name === "br") activeCode.text += "\n" if (name === "code" && attributes.class) activeCode.language = attributes.class.match(/(?:language-|lang-)([^\s]+)/)?.[1] return } if (name === "details") { frame.details = { open: "open" in attributes, summary: false, previous: activeDetails } activeDetails = frame.details block() return } if (name === "summary") { if (details) details.summary = true block() return } if (name === "pre") { suspendLink(frame) frame.code = { inline: false, text: "" } activeCode = frame.code return } if (name === "code") { if (activeCode?.inline) return frame.code = { inline: true, text: "" } activeCode = frame.code return } if (/^h[1-6]$/.test(name)) { block() inline(`${"#".repeat(Number(name[1]))} `) return } if (blocks.has(name)) { suspendLink(frame) if (name === "p" && last === " ") return block() return } if (name === "br") { pendingSpace = false inline(" \n") needsQuotePrefix = quoteDepth > 0 return } if (name === "hr") { block() inline("---") block() return } const marker = inlineMarker(name) if (marker) { inline(marker, true) frame.marker = { index: output.length - 1, block: blockCount, previous: activeMarker } activeMarker = frame.marker return } if (name === "a") { if (activeLink && activeCode?.inline) { const parent = stack.findLast( (candidate) => candidate.link === activeLink && candidate.linkCode === activeCode, ) if (parent) { finishCode(activeCode) if (linkOpen) append(`](${destination(activeLink.href)}${title(activeLink.title)})`) activeCode = parent.resumedCode parent.link = undefined parent.linkCode = undefined activeLink = undefined linkOpen = false } } if (activeCode?.inline) { frame.resumedCode = activeCode if (activeCode.text) finishCode(activeCode) activeCode.text = "" activeCode = undefined frame.link = { href: attributes.href ?? "", title: attributes.title } activeLink = frame.link linkOpen = true inline("[", true) frame.linkCode = { inline: true, text: "" } activeCode = frame.linkCode return } if (activeLink) { if (linkOpen) append(`](${destination(activeLink.href)}${title(activeLink.title)})`) const parent = stack.findLast((candidate) => candidate.link === activeLink) if (parent) parent.link = undefined activeLink = undefined linkOpen = false } frame.link = { href: attributes.href ?? "", title: attributes.title } activeLink = frame.link linkOpen = true return inline("[", true) } if (name === "img") { const alt = (attributes.alt ?? "").replace(/([\\\]])/g, "\\$1") const close = `](${destination(attributes.src ?? "")}${title(attributes.title)})` const open = "![" const available = CONTENT_BYTES - outputBytes - encoder.encode(open + close).byteLength inline(`${open}${sliceBytes(alt, Math.max(0, available))}${close}`, true) return } if (name === "blockquote") { suspendLink(frame) block() quoteDepth++ needsQuotePrefix = true return } if (name === "ul" || name === "ol") { suspendLink(frame) const start = Number.parseInt(attributes.start ?? "1") frame.list = { ordered: name === "ol", next: Number.isNaN(start) ? 1 : start, previous: activeList } activeList = frame.list block() return } if (name === "li") { block() const value = Number.parseInt(attributes.value ?? "") if (activeList?.ordered && !Number.isNaN(value)) activeList.next = value const marker = activeList?.ordered ? `${activeList.next++}.` : "-" const prefix = `${(activeItem?.indent ?? "").slice(0, 24)}${marker} ` frame.item = { indent: " ".repeat(prefix.length), previous: activeItem } activeItem = frame.item pendingIndent = "" inline(prefix) return } if (name === "table") { suspendLink(frame) tableDepth++ if (tableDepth === 1) { block() frame.table = { start: output.length, rows: [], fallback: false, previous: activeTable } activeTable = frame.table } else pendingSpace = true return } if (name === "tr") { if (tableDepth !== 1) { pendingSpace = true return } if (activeTable) activeTable.row = [] return } if (name === "th" || name === "td") { if (tableDepth !== 1) { pendingSpace = true return } if (attributes.colspan || attributes.rowspan) activeTable!.fallback = true frame.cell = { start: output.length } activeCell = frame.cell return } if (name === "caption") { frame.caption = { start: output.length } return } if (name === "dt") { block() inline("**") return } if (name === "dd") { inline("\n: ") return } }, ontext(value) { if (stopped) { if (fallbackSuppressedDepth === 0 && fallbackOmittedDepth === 0) text(value) return } if (stack.at(-1)?.suppressed) return text(value) }, onclosetag(name) { depth-- if (stopped) { if (fallbackOmittedDepth > 0 && omitted.has(name)) fallbackOmittedDepth-- if (fallbackSuppressedDepth > 0 && depth < fallbackSuppressedDepth) fallbackSuppressedDepth = 0 return } const frame = stack.pop() if (!frame || frame.suppressed) return if (frame.linkCode) { if (activeLink !== frame.link || !linkOpen) { frame.resumedCode!.text += frame.linkCode.text activeCode = frame.resumedCode if (activeLink === frame.link) activeLink = undefined linkOpen = false return } activeCode = undefined finishCode(frame.linkCode) if (frame.link && linkOpen) append(`](${destination(frame.link.href)}${title(frame.link.title)})`) activeLink = undefined linkOpen = false activeCode = frame.resumedCode return } if (activeCode && !activeCode.inline && !frame.code) return if (frame.code) { activeCode = undefined finishCode(frame.code) if (frame.suspendedLink) activeLink = frame.suspendedLink return } if (name === "summary") { if (activeDetails) activeDetails.summary = false return block() } if (name === "details") { activeDetails = frame.details?.previous return block() } if (name === "dt") { inline("**") return } if (name === "dd") return block() const marker = inlineMarker(name) if (marker) { const trailingSpace = pendingSpace pendingSpace = false if (frame.marker) activeMarker = frame.marker.previous if (frame.marker && (frame.marker.block !== blockCount || output.length === frame.marker.index + 1)) { output[frame.marker.index] = "" pendingSpace = trailingSpace || frame.marker.leadingSpace === true return } inline(marker) pendingSpace = trailingSpace || frame.marker?.leadingSpace === true return } if (name === "a") { if (frame.link && (activeLink === frame.link || !activeLink)) { activeLink = frame.link if (linkOpen || (last && last !== "\n")) append(`](${destination(frame.link.href)}${title(frame.link.title)})`) linkOpen = false activeLink = undefined } return } if (/^h[1-6]$/.test(name) || blocks.has(name)) { block() if (frame.suspendedLink) activeLink = frame.suspendedLink return } if (name === "blockquote") { quoteDepth-- block() if (frame.suspendedLink) activeLink = frame.suspendedLink return } if (name === "li") { activeItem = frame.item?.previous return block() } if (name === "ul" || name === "ol") { activeList = frame.list?.previous block() if (frame.suspendedLink) activeLink = frame.suspendedLink return } if ((name === "th" || name === "td") && tableDepth === 1) { activeCell = undefined if (frame.cell) { const value = take(frame.cell.start) .replace(/[\t\r\n ]+/g, " ") .trim() .replace(/(? 0 && table.rows.every((row) => row.length === width) if (loose) { append(loose) block() } if (table.caption) { append(table.caption) block() } if (!table.fallback && rectangular) { const prefix = `${quoteDepth > 0 ? quotePrefix() : ""}${pendingIndent}` pendingIndent = "" append(`${prefix}| ${table.rows[0].join(" | ")} |\n${prefix}|${" --- |".repeat(width)}`) for (const row of table.rows.slice(1)) append(`\n${prefix}| ${row.join(" | ")} |`) } else { for (const [index, row] of table.rows.entries()) { if (index > 0) block() append(row.join(" | ")) } } } block() if (frame.suspendedLink) activeLink = frame.suspendedLink return } pendingSpace = true } }, }) for (let index = 0; index < html.length; index += 64 * 1024) parser.write(html.slice(index, index + 64 * 1024)) parser.end() const normalized: string[] = [] let pendingText = "" const flushText = () => { if (!pendingText) return const lines = pendingText .replace(/[ \t]+\n/g, (space) => (space.startsWith(" ") ? " \n" : "\n")) .replace(/\n{3,}/g, "\n\n") .split("\n") normalized.push( lines .map((line, index) => { if (line) return line const before = lines[index - 1]?.match(/^(?:> )+/)?.[0] const after = lines[index + 1]?.match(/^(?:> )+/)?.[0] if (!before || !after || before.length === after.length) return line return "> ".repeat(Math.min(before.length, after.length) / 2).trimEnd() }) .join("\n"), ) pendingText = "" } for (const chunk of output) { if (typeof chunk !== "string") { flushText() normalized.push(chunk.raw) continue } pendingText += chunk } flushText() return sliceBytes(normalized.join("").trim(), MAX_MARKDOWN_BYTES) }