mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(agents): preserve UTF-16 boundaries in context pruning (#104404)
Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
co-authored by
Vincent Koc
parent
f73c5541fe
commit
13817a6108
@@ -357,6 +357,33 @@ describe("pruneContextMessages", () => {
|
||||
expect(result).toBe(messages);
|
||||
});
|
||||
|
||||
it("keeps soft-trim head and tail valid at UTF-16 boundaries", () => {
|
||||
const largeText = `${"h".repeat(99)}😀${"m".repeat(100)}😀${"t".repeat(49)}`;
|
||||
const result = pruneContextMessages({
|
||||
messages: [
|
||||
makeUser("summarize this"),
|
||||
makeToolResult([{ type: "text", text: largeText }]),
|
||||
makeAssistant([{ type: "text", text: "done" }]),
|
||||
],
|
||||
settings: {
|
||||
...DEFAULT_CONTEXT_PRUNING_SETTINGS,
|
||||
keepLastAssistants: 1,
|
||||
softTrimRatio: 0,
|
||||
hardClear: { ...DEFAULT_CONTEXT_PRUNING_SETTINGS.hardClear, enabled: false },
|
||||
softTrim: { maxChars: 200, headChars: 100, tailChars: 50 },
|
||||
},
|
||||
ctx: CONTEXT_WINDOW_1M,
|
||||
isToolPrunable: () => true,
|
||||
contextWindowTokensOverride: 1,
|
||||
});
|
||||
|
||||
const toolResult = result[1] as Extract<AgentMessage, { role: "toolResult" }>;
|
||||
const textBlock = toolResult.content[0] as { type: "text"; text: string };
|
||||
expect(textBlock.text).toBe(
|
||||
`${"h".repeat(99)}\n...\n${"t".repeat(49)}\n\n[Tool result trimmed: kept first 100 chars and last 50 chars of 252 chars.]`,
|
||||
);
|
||||
});
|
||||
|
||||
it("soft-trims image-containing tool results by replacing image blocks with placeholders", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
makeUser("summarize this"),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/** Context-pruning planner that trims old assistant/tool content under token pressure. */
|
||||
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import type { ImageContent, TextContent, ToolResultMessage } from "../../../llm/types.js";
|
||||
import { CHARS_PER_TOKEN_ESTIMATE, estimateStringChars } from "../../../utils/cjk-chars.js";
|
||||
import { dropThinkingBlocks } from "../../embedded-agent-runner/thinking.js";
|
||||
@@ -100,7 +101,7 @@ function takeHeadFromJoinedText(parts: string[], maxChars: number): string {
|
||||
out += p;
|
||||
remaining -= p.length;
|
||||
} else {
|
||||
out += p.slice(0, remaining);
|
||||
out += sliceUtf16Safe(p, 0, remaining);
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
@@ -119,7 +120,7 @@ function takeTailFromJoinedText(parts: string[], maxChars: number): string {
|
||||
out.push(p);
|
||||
remaining -= p.length;
|
||||
} else {
|
||||
out.push(p.slice(p.length - remaining));
|
||||
out.push(sliceUtf16Safe(p, -remaining));
|
||||
break;
|
||||
}
|
||||
if (remaining > 0 && i > 0) {
|
||||
|
||||
Reference in New Issue
Block a user