mirror of
https://github.com/anomalyco/opencode.git
synced 2026-08-13 13:29:00 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5cace7a01d |
@@ -331,7 +331,6 @@
|
||||
"minimatch": "10.2.5",
|
||||
"npm-package-arg": "13.0.2",
|
||||
"semver": "^7.6.3",
|
||||
"turndown": "7.2.0",
|
||||
"venice-ai-sdk-provider": "2.0.2",
|
||||
"which": "6.0.1",
|
||||
"xdg-basedir": "5.1.0",
|
||||
@@ -354,7 +353,6 @@
|
||||
"@types/npm-package-arg": "6.1.4",
|
||||
"@types/npmcli__arborist": "6.3.3",
|
||||
"@types/semver": "catalog:",
|
||||
"@types/turndown": "5.0.5",
|
||||
"@types/which": "3.0.4",
|
||||
"drizzle-kit": "catalog:",
|
||||
},
|
||||
|
||||
@@ -45,7 +45,6 @@
|
||||
"@types/npm-package-arg": "6.1.4",
|
||||
"@types/npmcli__arborist": "6.3.3",
|
||||
"@types/semver": "catalog:",
|
||||
"@types/turndown": "5.0.5",
|
||||
"@types/which": "3.0.4",
|
||||
"@parcel/watcher-darwin-arm64": "2.5.1",
|
||||
"@parcel/watcher-darwin-x64": "2.5.1",
|
||||
@@ -117,7 +116,6 @@
|
||||
"minimatch": "10.2.5",
|
||||
"npm-package-arg": "13.0.2",
|
||||
"semver": "^7.6.3",
|
||||
"turndown": "7.2.0",
|
||||
"venice-ai-sdk-provider": "2.0.2",
|
||||
"which": "6.0.1",
|
||||
"xdg-basedir": "5.1.0",
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
# HTML to Markdown renderer
|
||||
|
||||
## Goal
|
||||
|
||||
Replace Turndown and Domino in V2 Core only when an htmlparser2 event renderer preserves model-readable semantics and improves resource use and shipped size.
|
||||
|
||||
## Commands
|
||||
|
||||
- `bun run test tool-webfetch.test.ts` from `packages/core`
|
||||
- `bun build --entrypoints src/tool/html-markdown.ts --outdir <dir> --target node --format esm --minify`
|
||||
- `bun run build --single --skip-install` from `packages/cli`
|
||||
|
||||
## Metrics
|
||||
|
||||
- Primary: median conversion throughput after one warmup and nine measured runs.
|
||||
- Secondary: min/max spread, minified/gzip bundle size, CLI artifact size, and peak RSS where practical.
|
||||
|
||||
## Experiment Log
|
||||
|
||||
| Experiment | Hypothesis | Before | After | Decision |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Event renderer | Avoiding Domino's DOM lowers conversion cost while retaining semantics. | Turndown 4.23 MiB/s median (72.55 ms, 66.93-109.75) | Candidate 10.12 MiB/s median (30.32 ms, 22.29-83.55) | Keep: 2.39x throughput |
|
||||
| Safe fences | Fence length derived from code content prevents embedded backticks from closing blocks. | Turndown emitted triple fences around embedded triples | Candidate expands to four backticks | Keep |
|
||||
| Tables | Row/cell events retain tabular relationships better than flattened cell blocks. | Turndown flattened cells | Candidate emits GFM-readable tables | Keep |
|
||||
| Malformed inline blocks | Delimiters spanning implied block closes produce malformed Markdown. | Candidate left open emphasis | Candidate drops the delimiter and preserves visible text | Keep |
|
||||
|
||||
## Evaluation
|
||||
|
||||
Temporary snapshots from Example Domain, MDN's table reference, Python asyncio documentation, RFC 9110, and W3C's forms tutorial were evaluated on 2026-08-12. Candidate output retained the same heading counts on four sites and one additional visible MDN heading, the same link counts on three sites, two additional Python links, and the same fenced-code counts where Turndown recognized fences. Candidate output was 0-3.6% smaller; tables and preformatted code were more explicit. Snapshots and generated output are not committed.
|
||||
|
||||
The minified isolated evaluation bundle containing Turndown, Domino, htmlparser2, and both renderers was 311,557 bytes (98,680 gzip). The candidate renderer with htmlparser2 was 61,293 bytes (26,922 gzip). Installed Turndown plus Domino occupied 9,028 KiB; htmlparser2 was already required by Core.
|
||||
|
||||
The same-commit macOS arm64 CLI executable was 87,338,978 bytes with Turndown and 87,091,298 bytes with the candidate, a 247,680-byte reduction.
|
||||
|
||||
Real-site HTML is temporary evaluation data and is not committed.
|
||||
@@ -0,0 +1,361 @@
|
||||
import { Parser } from "htmlparser2"
|
||||
|
||||
const omitted = new Set(["script", "style", "noscript", "iframe", "object", "embed", "meta", "link", "template"])
|
||||
const blocks = new Set([
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"details",
|
||||
"dialog",
|
||||
"div",
|
||||
"dl",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"header",
|
||||
"main",
|
||||
"nav",
|
||||
"p",
|
||||
"section",
|
||||
"summary",
|
||||
])
|
||||
|
||||
type Frame = {
|
||||
tag: string
|
||||
suppressed: boolean
|
||||
link?: { href: string; title?: string }
|
||||
marker?: { index: number; block: number; value: string }
|
||||
code?: { inline: boolean; text: string; language?: string }
|
||||
list?: { ordered: boolean; next: number }
|
||||
table?: { cells: number; header: boolean; rows: number }
|
||||
cell?: { start: number }
|
||||
}
|
||||
|
||||
export function convertHTMLToMarkdown(html: string) {
|
||||
if (hasPathologicalDepth(html)) return extractPathologicalText(html)
|
||||
const output: string[] = []
|
||||
const stack: Frame[] = []
|
||||
let pendingSpace = false
|
||||
let last = ""
|
||||
let quoteDepth = 0
|
||||
let needsQuotePrefix = false
|
||||
let blockCount = 0
|
||||
let listDepth = 0
|
||||
let activeCode: NonNullable<Frame["code"]> | undefined
|
||||
let activeTable: NonNullable<Frame["table"]> | undefined
|
||||
let tableDepth = 0
|
||||
const raw: string[] = []
|
||||
|
||||
const append = (value: string) => {
|
||||
if (!value) return
|
||||
output.push(value)
|
||||
last = value.at(-1) ?? last
|
||||
}
|
||||
const prefixQuote = () => {
|
||||
if (!needsQuotePrefix || quoteDepth === 0) return
|
||||
append(`${"> ".repeat(Math.min(8, quoteDepth))}`)
|
||||
needsQuotePrefix = false
|
||||
}
|
||||
const flushSpace = () => {
|
||||
if (!pendingSpace) return
|
||||
const marker = stack.at(-1)?.marker
|
||||
if (marker && output.length === marker.index + 1 && last !== " " && last !== "\n") {
|
||||
output[marker.index] = ` ${output[marker.index]}`
|
||||
pendingSpace = false
|
||||
return
|
||||
}
|
||||
if (last && last !== "\n" && last !== " ") append(" ")
|
||||
pendingSpace = false
|
||||
}
|
||||
const inline = (value: string, open = false) => {
|
||||
if (open) flushSpace()
|
||||
prefixQuote()
|
||||
append(value)
|
||||
}
|
||||
const block = () => {
|
||||
pendingSpace = false
|
||||
append("\n\n")
|
||||
blockCount++
|
||||
needsQuotePrefix = quoteDepth > 0
|
||||
}
|
||||
const text = (value: string) => {
|
||||
if (activeCode) {
|
||||
activeCode.text += value
|
||||
return
|
||||
}
|
||||
for (const part of value.split(/([\t\n\f\r ]+)/)) {
|
||||
if (!part) continue
|
||||
if (/^[\t\n\f\r ]+$/.test(part)) {
|
||||
pendingSpace = true
|
||||
continue
|
||||
}
|
||||
flushSpace()
|
||||
prefixQuote()
|
||||
append(
|
||||
part
|
||||
.replace(/([\\`*_[\]<>|])/g, "\\$1")
|
||||
.replace(/~/g, "\\~")
|
||||
.replace(/^([#+-])/, "\\$1")
|
||||
.replace(/^(\d+)\./, "$1\\."),
|
||||
)
|
||||
}
|
||||
}
|
||||
const destination = (value: string) => value.replace(/([\\()])/g, "\\$1").replace(/[\t\n\r ]+/g, "%20")
|
||||
const title = (value: string | undefined) => (value ? ` "${value.replace(/([\\"])/g, "\\$1")}"` : "")
|
||||
const finishCode = (code: NonNullable<Frame["code"]>) => {
|
||||
let longest = 0
|
||||
let current = 0
|
||||
for (const character of code.text) {
|
||||
current = character === "`" ? current + 1 : 0
|
||||
longest = Math.max(longest, current)
|
||||
}
|
||||
const fence = "`".repeat(Math.max(code.inline ? 1 : 3, longest + 1))
|
||||
if (code.inline) {
|
||||
const padding = /^ | $/.test(code.text) && !/^ +$/.test(code.text) ? " " : ""
|
||||
flushSpace()
|
||||
inline(`${fence}${padding}${code.text}${padding}${fence}`)
|
||||
return
|
||||
}
|
||||
block()
|
||||
const value = `${fence}${code.language ?? ""}\n${code.text}${code.text.endsWith("\n") ? "" : "\n"}${fence}`
|
||||
const quoted = quoteDepth > 0 ? value.replace(/^/gm, `${"> ".repeat(Math.min(8, quoteDepth))}`) : value
|
||||
const placeholder = `\u0000${raw.length}\u0000`
|
||||
raw.push(quoted)
|
||||
append(placeholder)
|
||||
block()
|
||||
}
|
||||
|
||||
const parser = new Parser({
|
||||
onopentag(name, attributes) {
|
||||
const suppressed = (stack.at(-1)?.suppressed ?? false) || omitted.has(name)
|
||||
const frame: Frame = { tag: name, suppressed }
|
||||
stack.push(frame)
|
||||
if (suppressed) return
|
||||
|
||||
if (activeCode && !activeCode.inline) {
|
||||
if (name === "code" && attributes.class) activeCode.language = attributes.class.match(/(?:language-|lang-)([^\s]+)/)?.[1]
|
||||
return
|
||||
}
|
||||
if (name === "pre") {
|
||||
frame.code = { inline: false, text: "" }
|
||||
activeCode = frame.code
|
||||
return
|
||||
}
|
||||
if (name === "code") {
|
||||
frame.code = { inline: true, text: "" }
|
||||
activeCode = frame.code
|
||||
return
|
||||
}
|
||||
if (/^h[1-6]$/.test(name)) {
|
||||
block()
|
||||
inline(`${"#".repeat(Number(name[1]))} `)
|
||||
return
|
||||
}
|
||||
if (blocks.has(name)) {
|
||||
if (name === "p" && last === " ") return
|
||||
block()
|
||||
return
|
||||
}
|
||||
if (name === "br") {
|
||||
pendingSpace = false
|
||||
inline(" \n")
|
||||
needsQuotePrefix = quoteDepth > 0
|
||||
return
|
||||
}
|
||||
if (name === "hr") {
|
||||
block()
|
||||
inline("---")
|
||||
block()
|
||||
return
|
||||
}
|
||||
if (name === "strong" || name === "b") {
|
||||
inline("**", true)
|
||||
frame.marker = { index: output.length - 1, block: blockCount, value: "**" }
|
||||
return
|
||||
}
|
||||
if (name === "em" || name === "i") {
|
||||
inline("*", true)
|
||||
frame.marker = { index: output.length - 1, block: blockCount, value: "*" }
|
||||
return
|
||||
}
|
||||
if (name === "s" || name === "strike" || name === "del") {
|
||||
inline("~~", true)
|
||||
frame.marker = { index: output.length - 1, block: blockCount, value: "~~" }
|
||||
return
|
||||
}
|
||||
if (name === "a") {
|
||||
frame.link = { href: attributes.href ?? "", title: attributes.title }
|
||||
return inline(`[`, true)
|
||||
}
|
||||
if (name === "img") {
|
||||
inline(`![${(attributes.alt ?? "").replace(/([\\\]])/g, "\\$1")}](${destination(attributes.src ?? "")}${title(attributes.title)})`, true)
|
||||
return
|
||||
}
|
||||
if (name === "blockquote") {
|
||||
block()
|
||||
quoteDepth++
|
||||
needsQuotePrefix = true
|
||||
return
|
||||
}
|
||||
if (name === "ul" || name === "ol") {
|
||||
frame.list = { ordered: name === "ol", next: Number.parseInt(attributes.start ?? "1") || 1 }
|
||||
listDepth++
|
||||
block()
|
||||
return
|
||||
}
|
||||
if (name === "li") {
|
||||
block()
|
||||
const list = stack.findLast((item) => item.list)?.list
|
||||
const marker = list?.ordered ? `${list.next++}.` : "-"
|
||||
inline(`${" ".repeat(Math.min(8, Math.max(0, listDepth - 1)))}${marker} `)
|
||||
return
|
||||
}
|
||||
if (name === "table") {
|
||||
tableDepth++
|
||||
if (tableDepth === 1) {
|
||||
frame.table = { cells: 0, header: false, rows: 0 }
|
||||
activeTable = frame.table
|
||||
block()
|
||||
} else pendingSpace = true
|
||||
return
|
||||
}
|
||||
if (name === "tr") {
|
||||
if (tableDepth !== 1) {
|
||||
pendingSpace = true
|
||||
return
|
||||
}
|
||||
const table = activeTable
|
||||
pendingSpace = false
|
||||
if (table && table.rows > 0) {
|
||||
append("\n")
|
||||
needsQuotePrefix = quoteDepth > 0
|
||||
}
|
||||
inline("|")
|
||||
return
|
||||
}
|
||||
if (name === "th" || name === "td") {
|
||||
if (tableDepth !== 1) {
|
||||
pendingSpace = true
|
||||
return
|
||||
}
|
||||
const table = activeTable
|
||||
if (table) {
|
||||
table.cells++
|
||||
table.header ||= name === "th"
|
||||
}
|
||||
inline(" ")
|
||||
frame.cell = { start: output.length }
|
||||
}
|
||||
},
|
||||
ontext(value) {
|
||||
if (stack.at(-1)?.suppressed) return
|
||||
text(value)
|
||||
},
|
||||
onclosetag(name) {
|
||||
const frame = stack.pop()
|
||||
if (!frame || frame.suppressed) return
|
||||
if (frame.code) {
|
||||
activeCode = undefined
|
||||
return finishCode(frame.code)
|
||||
}
|
||||
if (name === "strong" || name === "b" || name === "em" || name === "i" || name === "s" || name === "strike" || name === "del") {
|
||||
const value = name === "strong" || name === "b" ? "**" : name === "em" || name === "i" ? "*" : "~~"
|
||||
if (frame.marker && frame.marker.block !== blockCount) {
|
||||
output[frame.marker.index] = ""
|
||||
return
|
||||
}
|
||||
if (frame.marker && output.length === frame.marker.index + 1) {
|
||||
output[frame.marker.index] = ""
|
||||
return
|
||||
}
|
||||
return inline(value)
|
||||
}
|
||||
if (name === "a") {
|
||||
return inline(`](${destination(frame.link?.href ?? "")}${title(frame.link?.title)})`)
|
||||
}
|
||||
if (/^h[1-6]$/.test(name) || blocks.has(name)) return block()
|
||||
if (name === "blockquote") {
|
||||
quoteDepth--
|
||||
return block()
|
||||
}
|
||||
if (name === "li") return block()
|
||||
if (name === "ul" || name === "ol") {
|
||||
listDepth--
|
||||
return block()
|
||||
}
|
||||
if ((name === "th" || name === "td") && tableDepth === 1) {
|
||||
if (frame.cell) {
|
||||
const value = output
|
||||
.splice(frame.cell.start)
|
||||
.join("")
|
||||
.replace(/[\t\r\n ]+/g, " ")
|
||||
.trim()
|
||||
.replace(/(?<!\\)\|/g, "\\|")
|
||||
append(value)
|
||||
}
|
||||
return inline(" |")
|
||||
}
|
||||
if (name === "tr") {
|
||||
if (tableDepth !== 1) return
|
||||
const table = activeTable
|
||||
if (table && table.rows === 0) {
|
||||
inline("\n")
|
||||
needsQuotePrefix = quoteDepth > 0
|
||||
inline(`|${" --- |".repeat(table.cells)}`)
|
||||
}
|
||||
if (table) {
|
||||
table.rows++
|
||||
table.cells = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
if (name === "table") {
|
||||
tableDepth--
|
||||
if (tableDepth === 0) {
|
||||
activeTable = undefined
|
||||
return block()
|
||||
}
|
||||
pendingSpace = true
|
||||
}
|
||||
},
|
||||
})
|
||||
parser.write(html)
|
||||
parser.end()
|
||||
return output
|
||||
.join("")
|
||||
.replace(/[ \t]+\n/g, (value) => (value.startsWith(" ") ? " \n" : "\n"))
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim()
|
||||
.replace(/\u0000(\d+)\u0000/g, (_, index) => raw[Number(index)] ?? "")
|
||||
}
|
||||
|
||||
function hasPathologicalDepth(html: string) {
|
||||
let depth = 0
|
||||
for (const match of html.matchAll(/<\s*(\/)?\s*([a-z][\w:-]*)\b[^>]*>/gi)) {
|
||||
if (match[1]) depth = Math.max(0, depth - 1)
|
||||
else if (!/\/$/.test(match[0].slice(0, -1).trim()) && !["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"].includes(match[2].toLowerCase())) depth++
|
||||
if (depth > 10_000) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function extractPathologicalText(html: string) {
|
||||
let output = ""
|
||||
let suppressed = 0
|
||||
const parser = new Parser({
|
||||
onopentag(name) {
|
||||
if (suppressed > 0 || omitted.has(name)) suppressed++
|
||||
},
|
||||
ontext(value) {
|
||||
if (suppressed === 0) output += value
|
||||
},
|
||||
onclosetag() {
|
||||
if (suppressed > 0) suppressed--
|
||||
},
|
||||
})
|
||||
parser.write(html.replace(/<\/?(?:[^>]+)>/g, (tag) => (omitted.has(tag.match(/^<\/?\s*([^\s/>]+)/)?.[1]?.toLowerCase() ?? "") ? tag : " ")))
|
||||
parser.end()
|
||||
return output.replace(/[\t\n\f\r ]+/g, " ").trim()
|
||||
}
|
||||
@@ -4,8 +4,8 @@ import { ToolFailure } from "@opencode-ai/llm"
|
||||
import { Duration, Effect, Layer, Schema } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Parser } from "htmlparser2"
|
||||
import TurndownService from "turndown"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { convertHTMLToMarkdown } from "./html-markdown"
|
||||
import { collectBoundedResponseBody } from "./http-body"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
@@ -196,14 +196,4 @@ export function extractTextFromHTML(html: string) {
|
||||
return text.trim()
|
||||
}
|
||||
|
||||
export function convertHTMLToMarkdown(html: string) {
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: "atx",
|
||||
hr: "---",
|
||||
bulletListMarker: "-",
|
||||
codeBlockStyle: "fenced",
|
||||
emDelimiter: "*",
|
||||
})
|
||||
turndown.remove(["script", "style", "meta", "link"])
|
||||
return turndown.turndown(html)
|
||||
}
|
||||
export { convertHTMLToMarkdown }
|
||||
|
||||
@@ -66,9 +66,109 @@ describe("WebFetchTool helpers", () => {
|
||||
})
|
||||
|
||||
test("ports HTML text and markdown conversions without active content", () => {
|
||||
const html = "<h1>Hello</h1><script>bad()</script><p>world <strong>wide</strong></p><style>.bad {}</style>"
|
||||
expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide**")
|
||||
const html =
|
||||
"<h1>Hello</h1><script>bad()</script><p>world <strong>wide</strong> <product-name>today</product-name></p><style>.bad {}</style>"
|
||||
expect(WebFetchTool.extractTextFromHTML(html)).toBe("Helloworld wide today")
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("# Hello\n\nworld **wide** today")
|
||||
})
|
||||
|
||||
test("renders headings, inline semantics, links, images, breaks, and thematic breaks", () => {
|
||||
const html = `<h2>Read <em>this</em></h2><p><a href="https://example.com/a (b)" title="Example">docs</a><br><img src="diagram.png" alt="a ] b"></p><hr><p><del>old</del></p>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`## Read *this*\n\n[docs](https://example.com/a%20\\(b\\) "Example") \n![a \\] b](diagram.png)\n\n---\n\n~~old~~`,
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves inline and preformatted code verbatim with safe fences", () => {
|
||||
const html = `<p>Use <code>say(\`hello\`)</code> now.</p><pre><code class="language-ts">const fence = \`\`\`\n& stays decoded</code></pre>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`Use \`\`say(\`hello\`)\`\` now.\n\n\`\`\`\`ts\nconst fence = \`\`\`\n& stays decoded\n\`\`\`\``,
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps nested ordered and unordered lists structurally readable", () => {
|
||||
const html = `<ol start="3"><li>alpha<ul><li>nested <strong>item</strong></li></ul></li><li><p>beta first</p><p>beta second</p></li></ol>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`3. alpha\n\n - nested **item**\n\n4. beta first\n\nbeta second`,
|
||||
)
|
||||
})
|
||||
|
||||
test("renders blockquotes and tables as readable Markdown", () => {
|
||||
const html = `<blockquote><p>quoted <em>text</em></p><ul><li>point</li></ul></blockquote><table><thead><tr><th>Name</th><th>Value</th></tr></thead><tbody><tr><td>one</td><td><code>1</code></td></tr></tbody></table>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`> quoted *text*\n\n> - point\n\n| Name | Value |\n| --- | --- |\n| one | \`1\` |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("decodes entities and normalizes prose whitespace without joining words", () => {
|
||||
const html = `<p>alpha\n <span>& beta</span> <unknown>café</unknown> gamma 😀</p><p>delta</p>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`alpha & beta café gamma 😀\n\ndelta`)
|
||||
})
|
||||
|
||||
test("omits active and fallback content while retaining surrounding prose", () => {
|
||||
const html = `<p>before <script><b>bad</b></script><style>bad</style><noscript>bad</noscript><iframe>bad</iframe><object>bad</object><embed src="bad"><meta content="bad"><link href="bad"><template>bad</template> after</p>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after")
|
||||
})
|
||||
|
||||
test("is deterministic and bounded for malformed maximum-size input", () => {
|
||||
const html = `<main><p>${"visible & text ".repeat(250_000)}</main></p></unknown>`
|
||||
const first = WebFetchTool.convertHTMLToMarkdown(html)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first)
|
||||
expect(first.startsWith("visible & text visible & text")).toBe(true)
|
||||
expect(first.length).toBeLessThanOrEqual(html.length)
|
||||
})
|
||||
|
||||
test("bounds deeply nested list output and fragmented code fences", () => {
|
||||
const lists = `${"<ul><li>item".repeat(2_000)}${"</li></ul>".repeat(2_000)}`
|
||||
const quotes = `${"<blockquote><p>item".repeat(2_000)}${"</p></blockquote>".repeat(2_000)}`
|
||||
const code = `<pre>${"` x ".repeat(250_000)}</pre>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4)
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4)
|
||||
expect(() => WebFetchTool.convertHTMLToMarkdown(code)).not.toThrow()
|
||||
expect(
|
||||
WebFetchTool.convertHTMLToMarkdown(
|
||||
"<div>".repeat(20_000) + "safe<script><b>bad</b>&</script><p>tail &</p>",
|
||||
),
|
||||
).toBe("safe tail &")
|
||||
})
|
||||
|
||||
test("escapes prose that would otherwise become Markdown structure", () => {
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p># heading</p><p>1. item</p><p>---</p><p>a | b</p>`)).toBe(
|
||||
`\\# heading\n\n1\\. item\n\n\\---\n\na \\| b`,
|
||||
)
|
||||
})
|
||||
|
||||
test("preserves code whitespace and quotes every line of multiline blocks", () => {
|
||||
const html = `<blockquote><pre>line \n\n\nnext</pre><table><tr><td>a|b</td><td>c</td></tr></table></blockquote>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`> \`\`\`\n> line \n> \n> \n> next\n> \`\`\`\n\n> | a\\|b | c |\n> | --- | --- |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("keeps visible whitespace around inline emphasis", () => {
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p>a<strong> b</strong> c a <em>b </em>c</p>`)).toBe(
|
||||
`a **b** c a *b* c`,
|
||||
)
|
||||
})
|
||||
|
||||
test("normalizes multiline table cells without changing their columns", () => {
|
||||
const html = `<table><tr><td>x<br>y</td><td><code>a|b</code></td><td><p>first</p><p>second</p></td></tr></table>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`| x y | \`a\\|b\` | first second |\n| --- | --- | --- |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("flattens nested tables without corrupting the outer table", () => {
|
||||
const html = `<table><tr><th>Parent</th><th>Sibling</th></tr><tr><td>Before<table><tr><th>Key</th><th>Value</th></tr><tr><td>A</td><td>1</td></tr></table>After</td><td>Tail</td></tr></table>`
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(
|
||||
`| Parent | Sibling |\n| --- | --- |\n| Before Key Value A 1 After | Tail |`,
|
||||
)
|
||||
})
|
||||
|
||||
test("escapes tilde fences and removes empty emphasis markers", () => {
|
||||
expect(WebFetchTool.convertHTMLToMarkdown(`<p>~~~</p><p><strong></strong>content</p><p>~~~</p>`)).toBe(
|
||||
`\\~\\~\\~\n\ncontent\n\n\\~\\~\\~`,
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -176,7 +276,7 @@ describe("WebFetchTool registration", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("returns an error result when HTML-to-Markdown conversion throws", () =>
|
||||
it.effect("converts deeply nested HTML without overflowing", () =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
respond = () =>
|
||||
@@ -189,8 +289,8 @@ describe("WebFetchTool registration", () => {
|
||||
const url = "https://1.1.1.1/deep-html"
|
||||
|
||||
expect(yield* executeTool(registry, call({ url, format: "markdown" }))).toEqual({
|
||||
type: "error",
|
||||
value: `Unable to fetch ${url}`,
|
||||
type: "text",
|
||||
value: "content",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user