refactor(util): reuse BOM-stripped text

packages/core/src/tool/html-markdown.ts

@@ -95,7 +95,7 @@ export function convertHTMLToMarkdown(html: string) {
     const remaining = limit - outputBytes
     const next = bytes.byteLength <= remaining ? value : sliceBytes(value, remaining)
     output.push(next)
-    outputBytes += bytes.byteLength <= remaining ? bytes.byteLength : encoder.encode(next).byteLength
+    outputBytes += encoder.encode(next).byteLength
     last = next.at(-1) ?? last
   }
   const appendRaw = (value: string) => {

packages/core/test/tool-webfetch.test.ts

@@ -128,15 +128,6 @@ describe("WebFetchTool helpers", () => {
     expect(output).toHaveLength(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024)
   })
 
-  test.each(["x", "\u00e9", "\u{1f600}"])("preserves UTF-8 boundaries at the content limit for %s", (character) => {
-    const budget = WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024
-    const fitting = "aa" + character.repeat(Math.floor((budget - 2) / Buffer.byteLength(character)))
-    expect(WebFetchTool.convertHTMLToMarkdown(fitting)).toBe(fitting)
-    const truncated = WebFetchTool.convertHTMLToMarkdown(fitting + character)
-    expect(truncated).toBe(fitting)
-    expect(Buffer.byteLength(truncated)).toBe(Buffer.byteLength(fitting))
-  })
-
   test("bounds deeply nested list output and fragmented code fences", () => {
     const lists = `${"<ul><li>item".repeat(2_000)}${"</li></ul>".repeat(2_000)}`
     const quotes = `${"<blockquote><p>item".repeat(2_000)}${"</p></blockquote>".repeat(2_000)}`

packages/util/src/bom.test.ts

@@ -0,0 +1,21 @@
+import { expect, test } from "bun:test"
+import { Bom } from "./bom.js"
+
+test.each([
+  { prefix: "", bom: false, expected: undefined },
+  { prefix: "", bom: true, expected: "\uFEFF" },
+  { prefix: "\uFEFF", bom: false, expected: "" },
+  { prefix: "\uFEFF", bom: true, expected: undefined },
+  { prefix: "\uFEFF\uFEFF", bom: false, expected: "" },
+  { prefix: "\uFEFF\uFEFF", bom: true, expected: "\uFEFF" },
+])("syncBytes(%j)", (row) => {
+  const encoder = new TextEncoder()
+  const text = "a\uFEFF\u00e9"
+  const input = encoder.encode(row.prefix + text)
+
+  expect(Bom.syncBytes(input, row.bom)).toEqual({
+    text,
+    bytes: row.expected === undefined ? undefined : encoder.encode(row.expected + text),
+  })
+  expect(input).toEqual(encoder.encode(row.prefix + text))
+})

packages/util/src/bom.ts

@@ -27,7 +27,7 @@ export function decodeBytes(content: Uint8Array) {
 export function syncBytes(content: Uint8Array, bom: boolean) {
   const decoded = decode(content)
   const current = split(decoded)
-  const canonical = join(current.text, bom)
+  const canonical = bom ? value + current.text : current.text
   return { text: current.text, bytes: decoded === canonical ? undefined : new TextEncoder().encode(canonical) }
 }