mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-21 10:15:47 +00:00
fix(frontend): strip <memory> tags in Streamdown to avoid React console error (#4209)
* fix(frontend): use code-aware stripLeakedSystemTags for all internal marker tags Replace the ad-hoc /<\/?memory>/g in ClipboardSafeStreamdown with a dedicated stripLeakedSystemTags() function in preprocess.ts that: - Covers all INTERNAL_MARKER_TAGS (<memory>, <system-reminder>, <current_date>, <uploaded_files>, <slash_skill_activation>) instead of only <memory> - Is code-aware: skips fenced code blocks (```) and indented code blocks (4-space indent), so user-written meta-discussions about the memory system are not silently stripped - Handles opening tags, closing tags, self-closing tags, and tags with attributes - Adds 15 unit tests covering all the above cases * fix(frontend): use marker-aware fence tracking in stripLeakedSystemTags The boolean insideFence toggle incorrectly closes a fence on any line matching 3+ backticks or tildes, regardless of the actual delimiter. This causes tags inside a tilde-fenced block containing a backtick sub-fence, or a 4-backtick block containing a 3-backtick sub-fence, to be silently stripped. Track the opening fence marker (character and run length) so that only a matching marker with at least the same length closes the fence. Adds 4 new test cases: - Tilde fence with inner backtick fence - 4-backtick fence with inner 3-backtick fence - Shorter tilde closing inside longer tilde fence - Tags stripped after real closing fence * fix(frontend): fix TypeScript strict errors in fence marker tracking - Use non-null assertion (!) on fenceMatch[1] (guaranteed by regex) - Use charAt(0) instead of [0] to avoid string | undefined type
This commit is contained in:
qin-chenghan
@@ -3,6 +3,7 @@
|
||||
import { Component, type ComponentProps, type ReactNode } from "react";
|
||||
import { Streamdown } from "streamdown";
|
||||
|
||||
import { stripLeakedSystemTags } from "@/core/streamdown/preprocess";
|
||||
import { installClipboardFallback } from "@/core/clipboard";
|
||||
|
||||
export type ClipboardSafeStreamdownProps = ComponentProps<typeof Streamdown>;
|
||||
@@ -57,9 +58,15 @@ export function ClipboardSafeStreamdown({
|
||||
children,
|
||||
...props
|
||||
}: ClipboardSafeStreamdownProps) {
|
||||
// Strip leaked system-internal tags (<memory>, <system-reminder>, etc.)
|
||||
// that would cause React to log "unrecognized tag" console errors when
|
||||
// the markdown renderer passes them through as raw HTML.
|
||||
const sanitizedChildren =
|
||||
typeof children === "string" ? stripLeakedSystemTags(children) : children;
|
||||
|
||||
return (
|
||||
<StreamdownFallbackBoundary raw={children}>
|
||||
<Streamdown {...props}>{children}</Streamdown>
|
||||
<StreamdownFallbackBoundary raw={sanitizedChildren}>
|
||||
<Streamdown {...props}>{sanitizedChildren}</Streamdown>
|
||||
</StreamdownFallbackBoundary>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { INTERNAL_MARKER_TAGS } from "@/core/messages/utils";
|
||||
|
||||
import { normalizeMermaidMarkdown } from "./mermaid";
|
||||
|
||||
const MERMAID_BLOCK_HINT_RE = /mermaid/i;
|
||||
@@ -317,6 +319,67 @@ export function normalizeStreamdownMathMarkdown(markdown: string): string {
|
||||
return compactDisplayMathBlocks(normalizeLatexMathDelimiters(markdown));
|
||||
}
|
||||
|
||||
// Regex matching any opening, closing, or self-closing internal marker tag.
|
||||
// e.g. <memory>, </memory>, <memory attr="x">, <memory/>
|
||||
const _INTERNAL_TAG_RE = new RegExp(
|
||||
`</?(?:${INTERNAL_MARKER_TAGS.join("|")})(?:\\s[^>]*)?/?>`,
|
||||
"g",
|
||||
);
|
||||
|
||||
// Regex matching the start/end of a fenced code block (3+ backticks or tildes).
|
||||
// Captures the marker string so we can compare character and length.
|
||||
const FENCE_MARKER_RE = /^ {0,3}(`{3,}|~{3,})/;
|
||||
|
||||
/**
|
||||
* Strip leaked system-internal HTML tags from markdown content.
|
||||
*
|
||||
* Backend-injected markers like ``<memory>…</memory>`` can occasionally
|
||||
* reach the UI renderer (e.g. when a ``hide_from_ui`` reminder leaks through
|
||||
* the filter). React's DOM renderer logs "unrecognized tag" console errors
|
||||
* for unknown HTML elements. This function strips the tag markers while
|
||||
* preserving the inner content — unlike {@link stripInternalMarkers} in
|
||||
* ``utils.ts``, which removes the entire block.
|
||||
*
|
||||
* Code-aware: tags inside fenced code blocks (````` `````) and indented code
|
||||
* blocks (4-space indent) are left untouched, so user-written meta-discussions
|
||||
* about the memory system are not silently stripped.
|
||||
*
|
||||
* Fence tracking is marker-aware (tracking the opening character and run
|
||||
* length) so that a tilde-fenced block containing a shorter backtick run, or a
|
||||
* 4-backtick block containing a 3-backtick run, does not prematurely close the
|
||||
* fence.
|
||||
*/
|
||||
export function stripLeakedSystemTags(markdown: string): string {
|
||||
const lines = markdown.split("\n");
|
||||
let fenceMarker: string | null = null;
|
||||
|
||||
return lines
|
||||
.map((line) => {
|
||||
const fenceMatch = FENCE_MARKER_RE.exec(line);
|
||||
if (fenceMatch) {
|
||||
const marker = fenceMatch[1]!;
|
||||
if (fenceMarker === null) {
|
||||
// Opening a fenced code block
|
||||
fenceMarker = marker;
|
||||
} else if (
|
||||
marker.startsWith(fenceMarker.charAt(0)) &&
|
||||
marker.length >= fenceMarker.length
|
||||
) {
|
||||
// Closing fence: same character and at least as long as opener
|
||||
fenceMarker = null;
|
||||
}
|
||||
// Otherwise: different fence type or shorter run inside a fence
|
||||
// (e.g. ``` inside ~~~~, or `` inside `````) — stay inside.
|
||||
return line;
|
||||
}
|
||||
if (fenceMarker !== null || INDENTED_CODE_RE.test(line)) {
|
||||
return line;
|
||||
}
|
||||
return line.replace(_INTERNAL_TAG_RE, "");
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
export function preprocessStreamdownMarkdown(markdown: string): string {
|
||||
if (!MERMAID_BLOCK_HINT_RE.test(markdown) || !markdown.includes("-.->")) {
|
||||
return markdown;
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
compactDisplayMathBlocks,
|
||||
normalizeStreamdownMathMarkdown,
|
||||
preprocessStreamdownMarkdown,
|
||||
stripLeakedSystemTags,
|
||||
} from "@/core/streamdown/preprocess";
|
||||
|
||||
test("capBlockquoteNesting returns normal content unchanged", () => {
|
||||
@@ -247,3 +248,200 @@ test("normalizeStreamdownMathMarkdown requires matching backtick run to close co
|
||||
const expected = "Use ``\\(literal\\)` and still code`` then $x$";
|
||||
expect(normalizeStreamdownMathMarkdown(input)).toBe(expected);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// stripLeakedSystemTags
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("stripLeakedSystemTags strips <memory> tags preserving content", () => {
|
||||
expect(stripLeakedSystemTags("<memory>hello</memory>")).toBe("hello");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips all internal marker tags", () => {
|
||||
expect(
|
||||
stripLeakedSystemTags(
|
||||
"<system-reminder>reminder</system-reminder> <current_date>2024</current_date>",
|
||||
),
|
||||
).toBe("reminder 2024");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips self-closing tags", () => {
|
||||
expect(stripLeakedSystemTags("text<memory/>more")).toBe("textmore");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips tags with attributes", () => {
|
||||
expect(stripLeakedSystemTags('<memory class="x">text</memory>')).toBe("text");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags handles multiple occurrences", () => {
|
||||
expect(
|
||||
stripLeakedSystemTags(
|
||||
"<memory>a</memory> <memory>b</memory> <memory>c</memory>",
|
||||
),
|
||||
).toBe("a b c");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags leaves fenced code content untouched", () => {
|
||||
const input = [
|
||||
"<memory>outside</memory>",
|
||||
"```text",
|
||||
"<memory>inside code</memory>",
|
||||
"```",
|
||||
"<memory>after</memory>",
|
||||
].join("\n");
|
||||
const expected = [
|
||||
"outside",
|
||||
"```text",
|
||||
"<memory>inside code</memory>",
|
||||
"```",
|
||||
"after",
|
||||
].join("\n");
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags leaves indented code content untouched", () => {
|
||||
const input = [
|
||||
"<memory>outside</memory>",
|
||||
" <memory>indented code</memory>",
|
||||
].join("\n");
|
||||
const expected = ["outside", " <memory>indented code</memory>"].join("\n");
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags passes plain text unchanged", () => {
|
||||
expect(stripLeakedSystemTags("plain text")).toBe("plain text");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags returns empty string unchanged", () => {
|
||||
expect(stripLeakedSystemTags("")).toBe("");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags handles no tags present", () => {
|
||||
const input = "normal text with **bold** and `code`";
|
||||
expect(stripLeakedSystemTags(input)).toBe(input);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips <uploaded_files> tag", () => {
|
||||
expect(
|
||||
stripLeakedSystemTags("<uploaded_files>file.pdf</uploaded_files>"),
|
||||
).toBe("file.pdf");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips <slash_skill_activation> tag", () => {
|
||||
expect(
|
||||
stripLeakedSystemTags(
|
||||
"<slash_skill_activation>skill</slash_skill_activation>",
|
||||
),
|
||||
).toBe("skill");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags handles mixed tags on same line", () => {
|
||||
expect(
|
||||
stripLeakedSystemTags(
|
||||
"<memory>a</memory><system-reminder>b</system-reminder>",
|
||||
),
|
||||
).toBe("ab");
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags handles multiple fences correctly", () => {
|
||||
const input = [
|
||||
"<memory>a</memory>",
|
||||
"```",
|
||||
"<memory>inside 1</memory>",
|
||||
"```",
|
||||
"<memory>b</memory>",
|
||||
"```",
|
||||
"<memory>inside 2</memory>",
|
||||
"```",
|
||||
].join("\n");
|
||||
const expected = [
|
||||
"a",
|
||||
"```",
|
||||
"<memory>inside 1</memory>",
|
||||
"```",
|
||||
"b",
|
||||
"```",
|
||||
"<memory>inside 2</memory>",
|
||||
"```",
|
||||
].join("\n");
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags preserves tags inside tilde fence with inner backtick fence", () => {
|
||||
const input = [
|
||||
"<memory>outside</memory>",
|
||||
"~~~~",
|
||||
"```",
|
||||
"<memory>inside tilde</memory>",
|
||||
"```",
|
||||
"~~~~",
|
||||
"<memory>after</memory>",
|
||||
].join("\n");
|
||||
const expected = [
|
||||
"outside",
|
||||
"~~~~",
|
||||
"```",
|
||||
"<memory>inside tilde</memory>",
|
||||
"```",
|
||||
"~~~~",
|
||||
"after",
|
||||
].join("\n");
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags preserves tags inside 4-backtick fence with inner 3-backtick fence", () => {
|
||||
const input = [
|
||||
"<memory>outside</memory>",
|
||||
"````",
|
||||
"```",
|
||||
"<memory>inside 4-backtick</memory>",
|
||||
"```",
|
||||
"````",
|
||||
"<memory>after</memory>",
|
||||
].join("\n");
|
||||
const expected = [
|
||||
"outside",
|
||||
"````",
|
||||
"```",
|
||||
"<memory>inside 4-backtick</memory>",
|
||||
"```",
|
||||
"````",
|
||||
"after",
|
||||
].join("\n");
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags handles backtick fence inside tilde fence with shorter tilde closing", () => {
|
||||
// A 4-tilde fence containing a 3-backtick sub-fence; the closing tilde run
|
||||
// is shorter (3 vs 4) so it should NOT close the fence.
|
||||
const input = [
|
||||
"<memory>outside</memory>",
|
||||
"~~~~",
|
||||
"```",
|
||||
"<memory>inside</memory>",
|
||||
"```",
|
||||
"~~~",
|
||||
].join("\n");
|
||||
const expected = [
|
||||
"outside",
|
||||
"~~~~",
|
||||
"```",
|
||||
"<memory>inside</memory>",
|
||||
"```",
|
||||
"~~~",
|
||||
].join("\n");
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test("stripLeakedSystemTags strips tags after real closing fence", () => {
|
||||
const input = [
|
||||
"~~~~",
|
||||
"<memory>inside</memory>",
|
||||
"~~~~",
|
||||
"<memory>after</memory>",
|
||||
].join("\n");
|
||||
const expected = ["~~~~", "<memory>inside</memory>", "~~~~", "after"].join(
|
||||
"\n",
|
||||
);
|
||||
expect(stripLeakedSystemTags(input)).toBe(expected);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user