mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(agent-core): avoid splitting surrogate pairs in grep line truncation (#97559)
truncateLine could cut a surrogate pair when the maxChars boundary fell between a high surrogate and its paired low surrogate, producing a broken unpaired surrogate in grep tool output.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
// Agent Core tests cover truncate behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { truncateHead, truncateTail } from "./truncate.js";
|
||||
import { truncateHead, truncateLine, truncateTail } from "./truncate.js";
|
||||
|
||||
describe("truncate utilities", () => {
|
||||
it("does not count a trailing newline as an extra display line", () => {
|
||||
@@ -20,4 +20,44 @@ describe("truncate utilities", () => {
|
||||
expect(result.lastLinePartial).toBe(true);
|
||||
expect(result.outputBytes).toBe(4);
|
||||
});
|
||||
|
||||
describe("truncateLine", () => {
|
||||
it("returns text unchanged when within limit", () => {
|
||||
expect(truncateLine("short", 10)).toEqual({ text: "short", wasTruncated: false });
|
||||
});
|
||||
|
||||
it("truncates and appends suffix when over limit", () => {
|
||||
const result = truncateLine("this is a very long line", 10);
|
||||
expect(result.wasTruncated).toBe(true);
|
||||
expect(result.text).toBe("this is a ... [truncated]");
|
||||
});
|
||||
|
||||
it("uses GREP_MAX_LINE_LENGTH as the default limit", () => {
|
||||
const result = truncateLine("x");
|
||||
expect(result.wasTruncated).toBe(false);
|
||||
expect(result.text).toBe("x");
|
||||
});
|
||||
|
||||
it("does not split a surrogate pair at the cut point", () => {
|
||||
// Emoji at boundary: "AB" + 🤖(surrogate pair) + "CD" — cut at 3 splits the emoji.
|
||||
expect(truncateLine("AB🤖CD", 3).text).toBe("AB... [truncated]");
|
||||
// Three emoji, cut in the middle of the second emoji.
|
||||
expect(truncateLine("🤖🤖🤖", 5).text).toBe("🤖🤖... [truncated]");
|
||||
// CJK Extension B (surrogate pair) at boundary stays intact.
|
||||
expect(truncateLine("AB𠮷CD", 5).text).toBe("AB𠮷C... [truncated]");
|
||||
});
|
||||
|
||||
it("never produces unpaired surrogates in output", () => {
|
||||
const results = [
|
||||
truncateLine("AB🤖CD", 3).text,
|
||||
truncateLine("🤖🤖🤖", 5).text,
|
||||
truncateLine("AB𠮷CD", 5).text,
|
||||
];
|
||||
for (const text of results) {
|
||||
expect(text).not.toMatch(
|
||||
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -359,6 +359,10 @@ function truncateStringToBytesFromEnd(str: string, maxBytes: number): string {
|
||||
|
||||
/**
|
||||
* Trim a single display line and mark it with the grep-style truncation suffix.
|
||||
*
|
||||
* The cut point is backed off by one code unit when it would otherwise split a
|
||||
* surrogate pair, so emoji / CJK Extension B characters crossing the boundary
|
||||
* stay intact instead of rendering as replacement characters.
|
||||
*/
|
||||
export function truncateLine(
|
||||
line: string,
|
||||
@@ -367,5 +371,16 @@ export function truncateLine(
|
||||
if (line.length <= maxChars) {
|
||||
return { text: line, wasTruncated: false };
|
||||
}
|
||||
return { text: `${line.slice(0, maxChars)}... [truncated]`, wasTruncated: true };
|
||||
let cut = maxChars;
|
||||
// Avoid splitting a surrogate pair at the truncation boundary.
|
||||
if (cut < line.length) {
|
||||
const lastCode = line.charCodeAt(cut - 1);
|
||||
if (lastCode >= 0xd800 && lastCode <= 0xdbff) {
|
||||
const nextCode = line.charCodeAt(cut);
|
||||
if (nextCode >= 0xdc00 && nextCode <= 0xdfff) {
|
||||
cut -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { text: `${line.slice(0, cut)}... [truncated]`, wasTruncated: true };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user