fix(memory-core): use CJK-aware tokenizer for dreaming dedupe (#80613) (#86645)

Summary:
- The PR extracts the CJK-aware memory tokenizer into a shared helper, routes dreaming dedupe through it, preserves MMR re-exports, and adds regression coverage for CJK and empty-token cases.
- PR surface: Source +15, Tests +96. Total +111 across 5 files.
- Reproducibility: yes. Current main has an ASCII-only tokenizeSnippet path in dreaming dedupe, and the source ... ction source bytes for the CJK failure modes; I did not run tests locally because this review is read-only.

Automerge notes:
- PR branch already contained follow-up commit before automerge: fix(memory-core): use Array.toSorted for #80613 lint fix
- PR branch already contained follow-up commit before automerge: fix(memory-core): preserve dedupe identity when both snippets tokeniz…
- PR branch already contained follow-up commit before automerge: fix(memory-core): rename __testing to testing in CJK regression tests…
- PR branch already contained follow-up commit before automerge: fix(memory-core): use CJK-aware tokenizer for dreaming dedupe (#80613)

Validation:
- ClawSweeper review passed for head ca9c02734c.
- Required merge gates passed before the squash merge.

Prepared head SHA: ca9c02734c
Review: https://github.com/openclaw/openclaw/pull/86645#issuecomment-4537414471

Co-authored-by: MoerAI <friendnt@g.skku.edu>
Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>
Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com>

extensions/memory-core/src/dreaming-phases.test.ts

@@ -2995,3 +2995,74 @@ describe("previewRemHarness", () => {
     expect(preview.deep.candidates[0]?.snippet).toContain("Always check weather");
   });
 });
+
+describe("dedupeEntries — CJK-aware snippet similarity (#80613)", () => {
+  // Reuse the same makeEntry helper shape used elsewhere in this file, but
+  // local here so we can vary `snippet` while keeping the rest stable.
+  function makeRecall(key: string, snippet: string): ShortTermRecallEntry {
+    return {
+      key,
+      path: "memory/2026-04-12.md",
+      startLine: 1,
+      endLine: 5,
+      source: "memory",
+      snippet,
+      recallCount: 1,
+      dailyCount: 0,
+      groundedCount: 0,
+      totalScore: 1,
+      maxScore: 1,
+      firstRecalledAt: "2026-04-12T08:00:00.000Z",
+      lastRecalledAt: "2026-04-12T08:00:00.000Z",
+      queryHashes: [],
+      recallDays: ["2026-04-12"],
+      conceptTags: [],
+    };
+  }
+
+  it("merges similar pure-CJK snippets at the same path (was missed by the ASCII-only tokenizer)", () => {
+    // Two close paraphrases of the same Chinese fact. The previous tokenizer
+    // produced empty sets for both, falling back to exact-string match and
+    // returning similarity 0, so both ended up as separate candidates.
+    const a = makeRecall("cjk-a", "教训：配置中实验开关字段是叫做规则");
+    const b = makeRecall("cjk-b", "教训：配置里实验开关的字段叫做规则");
+
+    const deduped = testing.dedupeEntries([a, b], 0.5);
+    expect(deduped).toHaveLength(1);
+    // First entry survives; recall counts merge in.
+    expect(deduped[0]?.key).toBe("cjk-a");
+  });
+
+  it("keeps distinct CJK snippets that only share ASCII tokens (was wrongly merged by the ASCII-only tokenizer)", () => {
+    // Both snippets have ASCII tokens {plan, exrule} but talk about wholly
+    // different facts in the CJK content. The previous tokenizer only saw
+    // those ASCII tokens, returned similarity 1.0, and silently dropped one
+    // of the two distinct memories. The CJK-aware tokenizer keeps them apart.
+    const a = makeRecall("mixed-a", "Plan 实验开关字段叫做 exRule");
+    const b = makeRecall("mixed-b", "Plan 整个产品体系彻底重构 exRule");
+
+    const deduped = testing.dedupeEntries([a, b], 0.7);
+    expect(deduped).toHaveLength(2);
+    expect(deduped.map((entry) => entry.key).toSorted()).toStrictEqual(["mixed-a", "mixed-b"]);
+  });
+
+  it("preserves the existing ASCII paraphrase dedupe behavior", () => {
+    // Sanity check: close English paraphrases at the same path still dedupe,
+    // so the fix does not regress the Latin-script behavior the prior tests
+    // relied on.
+    const a = makeRecall("en-a", "Plan config experiment toggle field is named exRule");
+    const b = makeRecall("en-b", "Plan configuration uses experiment toggle field named exRule");
+
+    const deduped = testing.dedupeEntries([a, b], 0.4);
+    expect(deduped).toHaveLength(1);
+    expect(deduped[0]?.key).toBe("en-a");
+  });
+
+  it("keeps unrelated short snippets separate (does not over-collapse)", () => {
+    const a = makeRecall("short-a", "weather: sunny");
+    const b = makeRecall("short-b", "deploy: blocked");
+
+    const deduped = testing.dedupeEntries([a, b], 0.5);
+    expect(deduped).toHaveLength(2);
+  });
+});

extensions/memory-core/src/dreaming-phases.ts

@@ -27,6 +27,7 @@ import {
   runDetachedDreamNarrative,
 } from "./dreaming-narrative.js";
 import { asRecord, formatErrorMessage, normalizeTrimmedString } from "./dreaming-shared.js";
+import { textSimilarity as snippetSimilarity } from "./memory/tokenize.js";
 import {
   filterLiveShortTermRecallEntries,
   readLightStagedKeys,
@@ -1388,33 +1389,15 @@ function entryAverageScore(entry: ShortTermRecallEntry): number {
   return signalCount > 0 ? Math.max(0, Math.min(1, entry.totalScore / signalCount)) : 0;
 }
 
-function tokenizeSnippet(snippet: string): Set<string> {
-  return new Set(normalizeStringEntries(snippet.toLowerCase().split(/[^a-z0-9]+/i)));
-}
-
-function jaccardSimilarity(left: string, right: string): number {
-  const leftTokens = tokenizeSnippet(left);
-  const rightTokens = tokenizeSnippet(right);
-  if (leftTokens.size === 0 || rightTokens.size === 0) {
-    return left.trim().toLowerCase() === right.trim().toLowerCase() ? 1 : 0;
-  }
-  let intersection = 0;
-  for (const token of leftTokens) {
-    if (rightTokens.has(token)) {
-      intersection += 1;
-    }
-  }
-  const union = new Set([...leftTokens, ...rightTokens]).size;
-  return union > 0 ? intersection / union : 0;
-}
-
+// Use the shared CJK-aware similarity helper so close-but-not-identical CJK
+// snippets do not slip past the dedupe threshold via the old ASCII-only path.
 function dedupeEntries(entries: ShortTermRecallEntry[], threshold: number): ShortTermRecallEntry[] {
   const deduped: ShortTermRecallEntry[] = [];
   for (const entry of entries) {
     const duplicate = deduped.find(
       (candidate) =>
         candidate.path === entry.path &&
-        jaccardSimilarity(candidate.snippet, entry.snippet) >= threshold,
+        snippetSimilarity(candidate.snippet, entry.snippet) >= threshold,
     );
     if (duplicate) {
       if (entry.recallCount > duplicate.recallCount) {
@@ -1905,6 +1888,8 @@ async function runPhaseIfTriggered(
 export const testing = {
   runPhaseIfTriggered,
   previewRemDreaming,
+  // Exposed for the #80613 regression test that exercises CJK-aware dedupe.
+  dedupeEntries,
   constants: {
     LIGHT_SLEEP_EVENT_TEXT,
     REM_SLEEP_EVENT_TEXT,

extensions/memory-core/src/memory/mmr.test.ts

@@ -150,6 +150,31 @@ describe("textSimilarity", () => {
       }
     }
   });
+
+  // Regression: dreaming dedupe (and any caller comparing arbitrary content
+  // strings via Jaccard tokens) must NOT merge distinct snippets when both
+  // sides tokenize to the empty set. The shared `tokenize` only emits ASCII
+  // word-tokens and CJK uni-/bigrams, so inputs in other scripts (Cyrillic,
+  // Arabic, emoji-only, punctuation-only) all tokenize to `{}` and would
+  // otherwise return Jaccard=1 → false dedupe.
+  it("falls back to literal equality when both inputs tokenize to empty sets (non-CJK/non-ASCII)", () => {
+    // Distinct Cyrillic snippets — must NOT merge.
+    expect(textSimilarity("Привет мир", "Доброе утро")).toBe(0);
+    // Distinct Arabic snippets — must NOT merge.
+    expect(textSimilarity("مرحبا بالعالم", "صباح الخير")).toBe(0);
+    // Emoji-only — distinct must NOT merge, identical must merge.
+    expect(textSimilarity("🦞🦞🦞", "🚀🚀🚀")).toBe(0);
+    expect(textSimilarity("🦞🦞🦞", "🦞🦞🦞")).toBe(1);
+    // Punctuation-only — distinct must NOT merge, identical must merge.
+    expect(textSimilarity("!!!", "???")).toBe(0);
+    expect(textSimilarity("!!!", "!!!")).toBe(1);
+    // Two identical Cyrillic snippets — same normalized string → merge.
+    expect(textSimilarity("Привет мир", "Привет мир")).toBe(1);
+    // Empty vs empty — preserve identity (both literally equal).
+    expect(textSimilarity("", "")).toBe(1);
+    // Normalized equality is case-insensitive for non-tokenized text.
+    expect(textSimilarity("Привет МИР", "привет мир")).toBe(1);
+  });
 });
 
 describe("computeMMRScore", () => {

extensions/memory-core/src/memory/mmr.ts

@@ -1,4 +1,4 @@
-import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
+import { jaccardSimilarity, textSimilarity, tokenize } from "./tokenize.js";
 
 /**
  * Maximal Marginal Relevance (MMR) re-ranking algorithm.
@@ -27,81 +27,10 @@ export const DEFAULT_MMR_CONFIG: MMRConfig = {
   lambda: 0.7,
 };
 
-/**
- * Regex matching CJK-family characters that lack whitespace word boundaries:
- * - CJK Unified Ideographs (Chinese hanzi, Japanese kanji, Korean hanja)
- * - CJK Extension A
- * - Hiragana & Katakana (Japanese)
- * - Hangul Syllables & Jamo (Korean)
- */
-const CJK_RE = /[\u3040-\u309f\u30a0-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\u1100-\u11ff]/;
-
-/**
- * Tokenize text for Jaccard similarity computation.
- * Extracts alphanumeric tokens, CJK-family characters (unigrams),
- * and consecutive CJK character pairs (bigrams).
- *
- * Bigrams are only created from characters that are adjacent in the
- * original text, so mixed content like "我喜欢hello你好" will NOT
- * produce the spurious bigram "欢你".
- */
-export function tokenize(text: string): Set<string> {
-  const lower = normalizeLowercaseStringOrEmpty(text);
-  const ascii = lower.match(/[a-z0-9_]+/g) ?? [];
-
-  // Track CJK characters with their original positions
-  const chars = Array.from(lower);
-  const cjkData: { char: string; index: number }[] = [];
-  for (let i = 0; i < chars.length; i++) {
-    if (CJK_RE.test(chars[i])) {
-      cjkData.push({ char: chars[i], index: i });
-    }
-  }
-
-  // Build bigrams only from originally adjacent CJK characters
-  const bigrams: string[] = [];
-  for (let i = 0; i < cjkData.length - 1; i++) {
-    if (cjkData[i + 1].index === cjkData[i].index + 1) {
-      bigrams.push(cjkData[i].char + cjkData[i + 1].char);
-    }
-  }
-
-  const unigrams = cjkData.map((d) => d.char);
-  return new Set([...ascii, ...bigrams, ...unigrams]);
-}
-
-/**
- * Compute Jaccard similarity between two token sets.
- * Returns a value in [0, 1] where 1 means identical sets.
- */
-export function jaccardSimilarity(setA: Set<string>, setB: Set<string>): number {
-  if (setA.size === 0 && setB.size === 0) {
-    return 1;
-  }
-  if (setA.size === 0 || setB.size === 0) {
-    return 0;
-  }
-
-  let intersectionSize = 0;
-  const smaller = setA.size <= setB.size ? setA : setB;
-  const larger = setA.size <= setB.size ? setB : setA;
-
-  for (const token of smaller) {
-    if (larger.has(token)) {
-      intersectionSize++;
-    }
-  }
-
-  const unionSize = setA.size + setB.size - intersectionSize;
-  return unionSize === 0 ? 0 : intersectionSize / unionSize;
-}
-
-/**
- * Compute text similarity between two content strings using Jaccard on tokens.
- */
-export function textSimilarity(contentA: string, contentB: string): number {
-  return jaccardSimilarity(tokenize(contentA), tokenize(contentB));
-}
+// Re-export the shared CJK-aware tokenizer + Jaccard helpers so existing
+// `import { tokenize, jaccardSimilarity, textSimilarity } from "./mmr.js"`
+// callers (including `mmr.test.ts`) continue to work without churn.
+export { jaccardSimilarity, textSimilarity, tokenize };
 
 /**
  * Compute the maximum similarity between an item and all selected items.

extensions/memory-core/src/memory/tokenize.ts

@@ -0,0 +1,101 @@
+import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
+
+/**
+ * Shared CJK-aware tokenizer + Jaccard similarity helpers.
+ *
+ * Originally introduced for memory MMR re-ranking; now also used by the dreaming
+ * dedupe path so similar-but-not-identical CJK candidates do not slip past the
+ * Jaccard threshold (issue #80613).
+ */
+
+/**
+ * Regex matching CJK-family characters that lack whitespace word boundaries:
+ * - CJK Unified Ideographs (Chinese hanzi, Japanese kanji, Korean hanja)
+ * - CJK Extension A
+ * - Hiragana & Katakana (Japanese)
+ * - Hangul Syllables & Jamo (Korean)
+ */
+const CJK_RE = /[\u3040-\u309f\u30a0-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\u1100-\u11ff]/;
+
+/**
+ * Tokenize text for Jaccard similarity computation.
+ * Extracts alphanumeric tokens, CJK-family characters (unigrams),
+ * and consecutive CJK character pairs (bigrams).
+ *
+ * Bigrams are only created from characters that are adjacent in the
+ * original text, so mixed content like "我喜欢hello你好" will NOT
+ * produce the spurious bigram "欢你".
+ */
+export function tokenize(text: string): Set<string> {
+  const lower = normalizeLowercaseStringOrEmpty(text);
+  const ascii = lower.match(/[a-z0-9_]+/g) ?? [];
+
+  // Track CJK characters with their original positions
+  const chars = Array.from(lower);
+  const cjkData: { char: string; index: number }[] = [];
+  for (let i = 0; i < chars.length; i++) {
+    if (CJK_RE.test(chars[i])) {
+      cjkData.push({ char: chars[i], index: i });
+    }
+  }
+
+  // Build bigrams only from originally adjacent CJK characters
+  const bigrams: string[] = [];
+  for (let i = 0; i < cjkData.length - 1; i++) {
+    if (cjkData[i + 1].index === cjkData[i].index + 1) {
+      bigrams.push(cjkData[i].char + cjkData[i + 1].char);
+    }
+  }
+
+  const unigrams = cjkData.map((d) => d.char);
+  return new Set([...ascii, ...bigrams, ...unigrams]);
+}
+
+/**
+ * Compute Jaccard similarity between two token sets.
+ * Returns a value in [0, 1] where 1 means identical sets.
+ */
+export function jaccardSimilarity(setA: Set<string>, setB: Set<string>): number {
+  if (setA.size === 0 && setB.size === 0) {
+    return 1;
+  }
+  if (setA.size === 0 || setB.size === 0) {
+    return 0;
+  }
+
+  let intersectionSize = 0;
+  const smaller = setA.size <= setB.size ? setA : setB;
+  const larger = setA.size <= setB.size ? setB : setA;
+
+  for (const token of smaller) {
+    if (larger.has(token)) {
+      intersectionSize++;
+    }
+  }
+
+  const unionSize = setA.size + setB.size - intersectionSize;
+  return unionSize === 0 ? 0 : intersectionSize / unionSize;
+}
+
+/**
+ * Compute text similarity between two content strings using Jaccard on tokens.
+ *
+ * When BOTH inputs tokenize to empty sets (e.g. Cyrillic/Arabic/emoji-only or
+ * punctuation-only snippets that contain no ASCII or CJK tokens), the raw
+ * `jaccardSimilarity` returns `1` for two empty sets. To prevent the dreaming
+ * dedupe path (and other callers that compare distinct strings via Jaccard)
+ * from collapsing distinct non-tokenized snippets into one, we fall back to
+ * exact normalized-string equality for that empty/empty case. Non-empty cases
+ * continue to use Jaccard unchanged.
+ */
+export function textSimilarity(contentA: string, contentB: string): number {
+  const tokensA = tokenize(contentA);
+  const tokensB = tokenize(contentB);
+  if (tokensA.size === 0 && tokensB.size === 0) {
+    return normalizeLowercaseStringOrEmpty(contentA) ===
+      normalizeLowercaseStringOrEmpty(contentB)
+      ? 1
+      : 0;
+  }
+  return jaccardSimilarity(tokensA, tokensB);
+}