fix(agents): normalize surrogate cache fingerprints (#101009)

* fix(agents): normalize surrogate cache fingerprints

* fix(agents): reuse provider surrogate sanitizer

* chore(agents): keep transport imports grouped

* fix(agents): avoid runtime import for surrogate sanitizer

* fix(agents): scope surrogate cache normalization

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>

packages/ai/src/internal/shared.ts

@@ -2,4 +2,5 @@ export * from "../providers/simple-options.js";
 export * from "../providers/tool-result-text.js";
 export * from "../providers/transform-messages.js";
 export * from "../utils/prompt-cache-stability.js";
+export * from "../utils/sanitize-unicode.js";
 export * from "../utils/system-prompt-cache-boundary.js";

packages/ai/src/utils/prompt-cache-stability.ts

@@ -4,10 +4,11 @@
  * ordering.
  */
 import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
+import { sanitizeSurrogates } from "./sanitize-unicode.js";
 
 /** Normalize structured prompt text before hashing or snapshot comparison. */
 export function normalizeStructuredPromptSection(text: string): string {
-  return text
+  return sanitizeSurrogates(text)
     .replace(/\r\n?/g, "\n")
     .replace(/[ \t]+$/gm, "")
     .trim();

packages/ai/src/utils/system-prompt-cache-boundary.test.ts

@@ -44,6 +44,17 @@ describe("system prompt cache boundary helpers", () => {
       `Stable prefix${SYSTEM_PROMPT_CACHE_BOUNDARY}Per-turn lab context\nSecond line\n\nDynamic suffix\n\nMore detail`,
     );
   });
+
+  it("normalizes malformed surrogates in dynamic prompt sections", () => {
+    const high = String.fromCharCode(0xd83d);
+
+    expect(
+      prependSystemPromptAdditionAfterCacheBoundary({
+        systemPrompt: `Stable prefix${SYSTEM_PROMPT_CACHE_BOUNDARY}Dynamic${high} suffix`,
+        systemPromptAddition: `Per-turn${high} context`,
+      }),
+    ).toBe(`Stable prefix${SYSTEM_PROMPT_CACHE_BOUNDARY}Per-turn context\n\nDynamic suffix`);
+  });
 });
 
 describe("ensureSystemPromptCacheBoundary", () => {

src/agents/cache-trace.test.ts

@@ -355,4 +355,24 @@ describe("createCacheTrace", () => {
       messages: [{ role: "user", content: "hello", child: { ref: "[Circular]" } }],
     });
   });
+
+  it("fingerprints malformed and transport-normalized text identically", () => {
+    const { lines, trace } = createMemoryTraceForTest();
+    const high = String.fromCharCode(0xd83d);
+
+    trace?.recordStage("prompt:before", {
+      messages: [{ role: "user", content: `left${high}right`, timestamp: 1 }],
+    });
+    trace?.recordStage("prompt:images", {
+      messages: [{ role: "user", content: "leftright", timestamp: 1 }],
+    });
+
+    const malformedEvent = JSON.parse(lines[0]?.trim() ?? "{}") as {
+      messageFingerprints?: string[];
+    };
+    const normalizedEvent = JSON.parse(lines[1]?.trim() ?? "{}") as {
+      messageFingerprints?: string[];
+    };
+    expect(malformedEvent.messageFingerprints).toEqual(normalizedEvent.messageFingerprints);
+  });
 });

src/agents/cache-trace.ts

@@ -3,6 +3,7 @@
  */
 import crypto from "node:crypto";
 import path from "node:path";
+import { sanitizeSurrogates } from "@openclaw/ai/internal/shared";
 import { resolveStateDir } from "../config/paths.js";
 import type { OpenClawConfig } from "../config/types.openclaw.js";
 import { resolveUserPath } from "../utils.js";
@@ -114,7 +115,7 @@ function getWriter(filePath: string): CacheTraceWriter {
 }
 
 function digest(value: unknown): string {
-  const serialized = stableStringify(value);
+  const serialized = stableStringify(value, sanitizeSurrogates);
   return crypto.createHash("sha256").update(serialized).digest("hex");
 }
 

src/agents/stable-stringify.test.ts

@@ -2,6 +2,7 @@
  * Regression coverage for deterministic unknown-value stringification.
  * Verifies sorted keys, repeated references, cycles, binary data, and errors.
  */
+import { sanitizeSurrogates } from "@openclaw/ai/internal/shared";
 import { describe, expect, it } from "vitest";
 import { stableStringify } from "./stable-stringify.js";
 
@@ -28,6 +29,33 @@ describe("stableStringify", () => {
     expect(stableStringify(items)).toBe('[{"value":"same"},{"value":"same"},"[Circular]"]');
   });
 
+  it("opts into string normalization without changing the lossless default", () => {
+    const high = String.fromCharCode(0xd83d);
+    const low = String.fromCharCode(0xdc00);
+    const value = {
+      [`key${high}`]: "name",
+      high: `left${high}right`,
+      low: `left${low}right`,
+      valid: "emoji 🙈 ok",
+    };
+
+    expect(stableStringify(value)).toContain("\\ud83d");
+    expect(stableStringify(value, sanitizeSurrogates)).toBe(
+      '{"high":"leftright","key":"name","low":"leftright","valid":"emoji 🙈 ok"}',
+    );
+  });
+
+  it("sorts normalized keys before serializing them", () => {
+    const high = String.fromCharCode(0xd83d);
+    const malformed = { ba: 2, [`b${high}`]: 1 };
+    const normalized = { ba: 2, b: 1 };
+
+    expect(stableStringify(malformed, sanitizeSurrogates)).toBe(
+      stableStringify(normalized, sanitizeSurrogates),
+    );
+    expect(stableStringify(malformed, sanitizeSurrogates)).toBe('{"b":1,"ba":2}');
+  });
+
   it("serializes cache-trace edge types deterministically", () => {
     const error = new Error("boom");
     error.stack = "Error: boom\n    at test";

src/agents/stable-stringify.ts

@@ -5,12 +5,23 @@
  */
 import { Buffer } from "node:buffer";
 
-/** Deterministically stringifies unknown values for cache keys and diagnostics. */
-export function stableStringify(value: unknown): string {
-  return stringifyStableValue(value, new WeakSet());
+type StableStringNormalizer = (value: string) => string;
+
+const preserveString = (value: string) => value;
+
+/** Deterministically stringifies values, optionally normalizing strings before key ordering. */
+export function stableStringify(
+  value: unknown,
+  normalizeString: StableStringNormalizer = preserveString,
+): string {
+  return stringifyStableValue(value, new WeakSet(), normalizeString);
 }
 
-function stringifyStableValue(value: unknown, stack: WeakSet<object>): string {
+function stringifyStableValue(
+  value: unknown,
+  stack: WeakSet<object>,
+  normalizeString: StableStringNormalizer,
+): string {
   if (value === null || value === undefined) {
     return String(value);
   }
@@ -20,6 +31,9 @@ function stringifyStableValue(value: unknown, stack: WeakSet<object>): string {
   if (typeof value === "bigint") {
     return JSON.stringify(value.toString());
   }
+  if (typeof value === "string") {
+    return JSON.stringify(normalizeString(value));
+  }
   if (typeof value !== "object") {
     return JSON.stringify(value) ?? "null";
   }
@@ -29,13 +43,17 @@ function stringifyStableValue(value: unknown, stack: WeakSet<object>): string {
 
   stack.add(value);
   try {
-    return stringifyObjectValue(value, stack);
+    return stringifyObjectValue(value, stack, normalizeString);
   } finally {
     stack.delete(value);
   }
 }
 
-function stringifyObjectValue(value: object, stack: WeakSet<object>): string {
+function stringifyObjectValue(
+  value: object,
+  stack: WeakSet<object>,
+  normalizeString: StableStringNormalizer,
+): string {
   if (value instanceof Error) {
     return stringifyStableValue(
       {
@@ -44,6 +62,7 @@ function stringifyObjectValue(value: object, stack: WeakSet<object>): string {
         stack: value.stack,
       },
       stack,
+      normalizeString,
     );
   }
   if (value instanceof Uint8Array) {
@@ -53,19 +72,33 @@ function stringifyObjectValue(value: object, stack: WeakSet<object>): string {
         data: Buffer.from(value).toString("base64"),
       },
       stack,
+      normalizeString,
     );
   }
   if (Array.isArray(value)) {
     const serializedEntries: string[] = [];
     for (const entry of value) {
-      serializedEntries.push(stringifyStableValue(entry, stack));
+      serializedEntries.push(stringifyStableValue(entry, stack, normalizeString));
     }
     return `[${serializedEntries.join(",")}]`;
   }
   const record = value as Record<string, unknown>;
+  const entries = Object.keys(record)
+    .map((key) => ({ key, normalizedKey: normalizeString(key) }))
+    .toSorted((left, right) => {
+      const normalizedOrder = compareStableStrings(left.normalizedKey, right.normalizedKey);
+      // Distinct source keys can normalize alike; preserve deterministic ordering without loss.
+      return normalizedOrder || compareStableStrings(left.key, right.key);
+    });
   const serializedFields: string[] = [];
-  for (const key of Object.keys(record).toSorted()) {
-    serializedFields.push(`${JSON.stringify(key)}:${stringifyStableValue(record[key], stack)}`);
+  for (const { key, normalizedKey } of entries) {
+    serializedFields.push(
+      `${JSON.stringify(normalizedKey)}:${stringifyStableValue(record[key], stack, normalizeString)}`,
+    );
   }
   return `{${serializedFields.join(",")}}`;
 }
+
+function compareStableStrings(left: string, right: string): number {
+  return left < right ? -1 : left > right ? 1 : 0;
+}

src/agents/transport-stream-shared.ts

@@ -3,6 +3,7 @@
  *
  * Sanitizes provider payloads, merges metadata, and formats streamed assistant events.
  */
+import { sanitizeSurrogates } from "@openclaw/ai/internal/shared";
 import { createAssistantMessageEventStream } from "../llm/utils/event-stream.js";
 import { redactSensitiveText } from "../logging/redact.js";
 import { truncateErrorDetail } from "./provider-http-errors.js";
@@ -48,10 +49,7 @@ export function sanitizeTransportPayloadText(text: string): string {
   if (typeof text !== "string") {
     return "";
   }
-  return text.replace(
-    /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
-    "",
-  );
+  return sanitizeSurrogates(text);
 }
 
 export function sanitizeNonEmptyTransportPayloadText(