fix(memory-core): use CJK-aware tokenizer for dreaming dedupe (#80613)

This commit is contained in:
clawsweeper
2026-05-25 21:35:18 +00:00
parent 6d65749a82
commit ca9c02734c
@@ -172,8 +172,7 @@ describe("textSimilarity", () => {
expect(textSimilarity("Привет мир", "Привет мир")).toBe(1);
// Empty vs empty — preserve identity (both literally equal).
expect(textSimilarity("", "")).toBe(1);
// Whitespace-only normalization is case-folded only, not trimmed — keep
// distinct strings distinct.
// Normalized equality is case-insensitive for non-tokenized text.
expect(textSimilarity("Привет МИР", "привет мир")).toBe(1);
});
});