refactor(core): reuse Markdown chunk byte counts

packages/codemode/src/openapi/index.ts

@@ -53,6 +53,7 @@ export const fromSpec = (options: Options): Result => {
     if (!isRecord(pathValue)) continue
     for (const [method, operationValue] of Object.entries(pathValue)) {
       if (!methods.has(method) || !isRecord(operationValue)) continue
+      const segments = operationPath(method, path, operationValue, used, namespaces)
       const operation: Operation = {
         operationId: nonEmptyString(operationValue.operationId),
         method: method.toUpperCase(),
@@ -98,7 +99,6 @@ export const fromSpec = (options: Options): Result => {
         auth: options.auth,
         headers: options.headers ?? {},
       }
-      const segments = operationPath(method, path, operationValue, used, namespaces)
       used.add(segments.join("."))
       for (const index of segments.slice(0, -1).keys()) namespaces.add(segments.slice(0, index + 1).join("."))
       setTool(

packages/codemode/test/openapi.test.ts

@@ -278,30 +278,6 @@ describe("OpenAPI.fromSpec", () => {
     expect(Tool.isTool(toolAt(result.tools, "group.operation.other"))).toBe(true)
   })
 
-  test("does not reserve names for unsupported operations between duplicate operation IDs", () => {
-    const operation = { operationId: "group.item", responses: { 200: { description: "Success" } } }
-    for (const unsupported of [false, true]) {
-      const result = OpenAPI.fromSpec({
-        baseUrl,
-        spec: {
-          openapi: "3.1.0",
-          paths: {
-            "/first": { get: operation },
-            ...(unsupported ? { "/unsupported": { get: { ...operation, "x-websocket": true } } } : {}),
-            "/last": { get: operation },
-          },
-        },
-      })
-
-      expect(Object.keys(result.tools)).toEqual(["group", "group_item_2"])
-      expect(toolAt(result.tools, "group.item")).toMatchObject({ _tag: "CodeModeTool", description: "GET /first" })
-      expect(toolAt(result.tools, "group_item_2")).toMatchObject({ _tag: "CodeModeTool", description: "GET /last" })
-      expect(result.skipped).toEqual(
-        unsupported ? [{ method: "GET", path: "/unsupported", reason: "WebSocket operations are not supported" }] : [],
-      )
-    }
-  })
-
   test("synthesizes flat operation IDs from methods and paths", () => {
     const response = { responses: { 200: { description: "Success" } } }
     const tools = OpenAPI.fromSpec({

packages/core/src/tool/html-markdown.ts

@@ -95,7 +95,7 @@ export function convertHTMLToMarkdown(html: string) {
     const remaining = limit - outputBytes
     const next = bytes.byteLength <= remaining ? value : sliceBytes(value, remaining)
     output.push(next)
-    outputBytes += encoder.encode(next).byteLength
+    outputBytes += bytes.byteLength <= remaining ? bytes.byteLength : encoder.encode(next).byteLength
     last = next.at(-1) ?? last
   }
   const appendRaw = (value: string) => {

packages/core/test/tool-webfetch.test.ts

@@ -128,6 +128,15 @@ describe("WebFetchTool helpers", () => {
     expect(output).toHaveLength(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024)
   })
 
+  test.each(["x", "\u00e9", "\u{1f600}"])("preserves UTF-8 boundaries at the content limit for %s", (character) => {
+    const budget = WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024
+    const fitting = "aa" + character.repeat(Math.floor((budget - 2) / Buffer.byteLength(character)))
+    expect(WebFetchTool.convertHTMLToMarkdown(fitting)).toBe(fitting)
+    const truncated = WebFetchTool.convertHTMLToMarkdown(fitting + character)
+    expect(truncated).toBe(fitting)
+    expect(Buffer.byteLength(truncated)).toBe(Buffer.byteLength(fitting))
+  })
+
   test("bounds deeply nested list output and fragmented code fences", () => {
     const lists = `${"<ul><li>item".repeat(2_000)}${"</li></ul>".repeat(2_000)}`
     const quotes = `${"<blockquote><p>item".repeat(2_000)}${"</p></blockquote>".repeat(2_000)}`