fix(agents): preserve UTF-16 boundaries in context pruning (#104404)

Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
mushuiyu886
2026-07-11 22:26:21 +08:00
committed by GitHub
co-authored by Vincent Koc
parent f73c5541fe
commit 13817a6108
2 changed files with 30 additions and 2 deletions
@@ -357,6 +357,33 @@ describe("pruneContextMessages", () => {
expect(result).toBe(messages);
});
it("keeps soft-trim head and tail valid at UTF-16 boundaries", () => {
const largeText = `${"h".repeat(99)}😀${"m".repeat(100)}😀${"t".repeat(49)}`;
const result = pruneContextMessages({
messages: [
makeUser("summarize this"),
makeToolResult([{ type: "text", text: largeText }]),
makeAssistant([{ type: "text", text: "done" }]),
],
settings: {
...DEFAULT_CONTEXT_PRUNING_SETTINGS,
keepLastAssistants: 1,
softTrimRatio: 0,
hardClear: { ...DEFAULT_CONTEXT_PRUNING_SETTINGS.hardClear, enabled: false },
softTrim: { maxChars: 200, headChars: 100, tailChars: 50 },
},
ctx: CONTEXT_WINDOW_1M,
isToolPrunable: () => true,
contextWindowTokensOverride: 1,
});
const toolResult = result[1] as Extract<AgentMessage, { role: "toolResult" }>;
const textBlock = toolResult.content[0] as { type: "text"; text: string };
expect(textBlock.text).toBe(
`${"h".repeat(99)}\n...\n${"t".repeat(49)}\n\n[Tool result trimmed: kept first 100 chars and last 50 chars of 252 chars.]`,
);
});
it("soft-trims image-containing tool results by replacing image blocks with placeholders", () => {
const messages: AgentMessage[] = [
makeUser("summarize this"),
@@ -1,4 +1,5 @@
/** Context-pruning planner that trims old assistant/tool content under token pressure. */
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import type { ImageContent, TextContent, ToolResultMessage } from "../../../llm/types.js";
import { CHARS_PER_TOKEN_ESTIMATE, estimateStringChars } from "../../../utils/cjk-chars.js";
import { dropThinkingBlocks } from "../../embedded-agent-runner/thinking.js";
@@ -100,7 +101,7 @@ function takeHeadFromJoinedText(parts: string[], maxChars: number): string {
out += p;
remaining -= p.length;
} else {
out += p.slice(0, remaining);
out += sliceUtf16Safe(p, 0, remaining);
remaining = 0;
}
}
@@ -119,7 +120,7 @@ function takeTailFromJoinedText(parts: string[], maxChars: number): string {
out.push(p);
remaining -= p.length;
} else {
out.push(p.slice(p.length - remaining));
out.push(sliceUtf16Safe(p, -remaining));
break;
}
if (remaining > 0 && i > 0) {