diff --git a/packages/speech-core/src/speech-text.test.ts b/packages/speech-core/src/speech-text.test.ts new file mode 100644 index 00000000000..da1dbbb820c --- /dev/null +++ b/packages/speech-core/src/speech-text.test.ts @@ -0,0 +1,105 @@ +import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; +import { describe, expect, it } from "vitest"; +import { + CODE_HEAVY_SPOKEN_FALLBACK, + isCodeHeavySpeechText, + normalizeSpeechText, +} from "./speech-text.js"; + +const speechStripOptions = { linkStyle: "label", mode: "speech" } as const; + +describe("speech text normalization", () => { + it("strips speech-hostile Markdown and decorative punctuation", () => { + const input = `# Release notes + +- Read the [guide](https://example.com/guide) +- Keep the useful detail + +| Area | Status | +| --- | --- | +| Talk | Ready | + +\`\`\`ts +const concise = true; +\`\`\` + +✨ ✨ ✨ +Really!!!!!`; + + const result = stripMarkdown(input, speechStripOptions); + + expect(result).toContain("Release notes"); + expect(result).toContain("Read the guide"); + expect(result).not.toContain("•"); + expect(result).not.toContain("https://example.com/guide"); + expect(result).toContain("Talk"); + expect(result).toContain("Status: Ready"); + expect(result).toContain("const concise = true;"); + expect(result).not.toMatch(/[#|`✨]/u); + expect(result).toContain("Really!"); + }); + + it("keeps meaningful numeric prefixes while removing decorative bullets", () => { + expect(stripMarkdown("404. Not found", speechStripOptions)).toBe("404. Not found"); + expect(stripMarkdown("• item", speechStripOptions)).toBe("item"); + }); + + it("falls back when fenced code is at least half of the reply", () => { + const input = `Brief note. + +\`\`\`ts +export function renderAnswer() { + return "most of this reply is code"; +} +\`\`\``; + + expect(isCodeHeavySpeechText(input)).toBe(true); + expect(normalizeSpeechText(input)).not.toBe(CODE_HEAVY_SPOKEN_FALLBACK); + }); + + it("keeps prose when fenced code is less than half of the reply", () => { + const input = `This explanation is intentionally long enough to remain the main part of the response. It tells the listener what the example does and why it matters before showing one tiny snippet. + +\`\`\`ts +const ready = true; +\`\`\``; + + expect(isCodeHeavySpeechText(input)).toBe(false); + const result = normalizeSpeechText(input); + expect(result).toContain("This explanation is intentionally long enough"); + expect(result).toContain("const ready = true;"); + }); + + it.each([ + ["a longer closing fence", "```\nconst answer = 42;\n````"], + ["an unclosed fence", "```\nconst answer = 42;"], + ["tilde fences", "~~~ts\nconst answer = 42;\n~~~"], + ["a mismatched marker inside a fence", "```\n~~~\nconst answer = 42;\n```"], + ["an indented closing fence", "```\nconst answer = 42;\n ```"], + ["a blockquoted fence", "> ```\n> const answer = 42;\n> ```"], + [ + "a two-level list-nested fence", + '- Outer\n - Inner\n ```\n const detailedAnswer = "this body dominates the reply";\n ```', + ], + ])("detects code-heavy text with %s", (_description, input) => { + expect(isCodeHeavySpeechText(input)).toBe(true); + }); + + it("keeps blockquoted code and surrounding prose aligned with Markdown stripping", () => { + const input = `> This explanation is deliberately much longer than the code it introduces, so it remains the reply's main content for speech. +> +> \`\`\`ts +> const ready = true; +> \`\`\``; + + expect(isCodeHeavySpeechText(input)).toBe(false); + expect(normalizeSpeechText(input)).toContain("This explanation is deliberately much longer"); + expect(normalizeSpeechText(input)).toContain("const ready = true;"); + }); + + it("does not count prose after a tab-terminated closing fence as code", () => { + const input = "```\nx\n```\t\nThis prose follows the code fence and is much longer than it."; + + expect(isCodeHeavySpeechText(input)).toBe(false); + }); +}); diff --git a/packages/speech-core/src/speech-text.ts b/packages/speech-core/src/speech-text.ts new file mode 100644 index 00000000000..e5335924140 --- /dev/null +++ b/packages/speech-core/src/speech-text.ts @@ -0,0 +1,126 @@ +import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; + +export const CODE_HEAVY_SPOKEN_FALLBACK = "I've put the detailed response on screen."; + +// At least half fenced-code content is unlikely to produce useful speech after stripping. +// Keep the threshold deterministic so Talk clients and providers hear the same fallback. +const CODE_HEAVY_FENCED_CHAR_RATIO = 0.5; + +type CodeFence = { + marker: "`" | "~"; + length: number; + blockquoteDepth: number; + listIndent: number; +}; + +type FenceContainer = Pick; + +function unwrapFenceContainer(line: string): { content: string; container: FenceContainer } { + let content = line; + let blockquoteDepth = 0; + while (true) { + const match = /^(?: {0,3}>[ \t]?)/u.exec(content); + if (!match) { + break; + } + content = content.slice(match[0].length); + blockquoteDepth += 1; + } + + const indentation = /^ +/u.exec(content)?.[0].length ?? 0; + const listIndent = indentation > 3 && indentation <= 8 ? indentation : 0; + if (listIndent > 0) { + content = content.slice(listIndent); + } + return { content, container: { blockquoteDepth, listIndent } }; +} + +function parseFenceOpener(line: string): CodeFence | undefined { + const { content, container } = unwrapFenceContainer(line); + const match = /^(?: {0,3})(`{3,}|~{3,})/u.exec(content); + const fence = match?.[1]; + if (!fence) { + return undefined; + } + + const marker = fence[0]; + if (marker !== "`" && marker !== "~") { + return undefined; + } + return { marker, length: fence.length, ...container }; +} + +function isFenceCloser(line: string, opener: CodeFence): boolean { + const { content, container } = unwrapFenceContainer(line); + if ( + container.blockquoteDepth !== opener.blockquoteDepth || + container.listIndent !== opener.listIndent + ) { + return false; + } + const match = /^(?: {0,3})(`+|~+)([ \t]*)$/u.exec(content); + const fence = match?.[1]; + return fence !== undefined && fence[0] === opener.marker && fence.length >= opener.length; +} + +function unwrapFenceBodyLine(line: string, opener: CodeFence): string { + let content = line; + for (let index = 0; index < opener.blockquoteDepth; index += 1) { + const match = /^(?: {0,3}>[ \t]?)/u.exec(content); + if (!match) { + return line; + } + content = content.slice(match[0].length); + } + if (opener.listIndent > 0 && content.startsWith(" ".repeat(opener.listIndent))) { + return content.slice(opener.listIndent); + } + return content; +} + +function countFencedCodeChars(text: string): number { + // Match common fenced-block containers so fallback and stripping classify the same code. + // This shallow scanner stops at two list levels; deeply nested exotic containers may misclassify. + // That only makes speech routing suboptimal, never loses message data. + const lines = text.split(/\r?\n/u); + let fencedCodeChars = 0; + let opener: CodeFence | undefined; + let bodyLines: string[] = []; + + for (const line of lines) { + if (!opener) { + opener = parseFenceOpener(line); + continue; + } + + if (isFenceCloser(line, opener)) { + fencedCodeChars += bodyLines.join("\n").length; + opener = undefined; + bodyLines = []; + continue; + } + + bodyLines.push(unwrapFenceBodyLine(line, opener)); + } + + if (opener) { + fencedCodeChars += bodyLines.join("\n").length; + } + return fencedCodeChars; +} + +export function isCodeHeavySpeechText(text: string): boolean { + const trimmed = text.trim(); + if (!trimmed) { + return false; + } + return countFencedCodeChars(trimmed) / trimmed.length >= CODE_HEAVY_FENCED_CHAR_RATIO; +} + +export function normalizeSpeechText(text: string): string { + const trimmed = text.trim(); + if (!trimmed) { + return ""; + } + return stripMarkdown(trimmed, { linkStyle: "label", mode: "speech" }).trim(); +} diff --git a/packages/speech-core/src/tts.test.ts b/packages/speech-core/src/tts.test.ts index 5d07e260a82..12d9dfa20d9 100644 --- a/packages/speech-core/src/tts.test.ts +++ b/packages/speech-core/src/tts.test.ts @@ -17,6 +17,7 @@ import type { SpeechTelephonySynthesisRequest, } from "openclaw/plugin-sdk/speech-core"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { CODE_HEAVY_SPOKEN_FALLBACK } from "./speech-text.js"; type MockSpeechSynthesisResult = Awaited>; @@ -123,6 +124,7 @@ const { setSummarizationEnabled, setTtsMaxLength, synthesizeSpeech, + textToSpeech, textToSpeechTelephony, } = await import("../runtime-api.js"); @@ -514,6 +516,38 @@ describe("speech-core native voice-note routing", () => { expect(request.timeoutMs).toBe(600_000); }); + it("normalizes non-streaming synthesis text before calling the provider", async () => { + const result = await synthesizeSpeech({ + text: "## Update\n\nRead the [guide](https://example.com/guide)!!!!!", + cfg: createTtsConfig("openclaw-speech-core-talk-markdown-test"), + disableFallback: true, + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("normalized talk synthesis request"); + expect(request.text).toBe("Update\n\nRead the guide!"); + }); + + it("speaks stripped code through the explicit textToSpeech conversion path", async () => { + let mediaDir: string | undefined; + try { + const result = await textToSpeech({ + text: "```ts\nconst answer = 42;\n```", + cfg: createTtsConfig("openclaw-speech-core-code-convert-test"), + }); + + expect(result.success).toBe(true); + const request = requireFirstSynthesisRequest("explicit code conversion request"); + expect(request.text).toBe("const answer = 42;"); + expect(request.text).not.toBe(CODE_HEAVY_SPOKEN_FALLBACK); + mediaDir = result.audioPath ? path.dirname(result.audioPath) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + it("resolves the configured timeout for voice listing", async () => { const listVoicesMock = vi.fn(async (_request: SpeechListVoicesRequest) => []); installSpeechProviders([ @@ -1028,6 +1062,70 @@ describe("speech-core native voice-note routing", () => { }); }); + it("normalizes voice-note Markdown once before synthesis", async () => { + const text = + 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\n```md\nconst literal = "[x](y)";\n```'; + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { text }, + cfg: createTtsConfig("openclaw-speech-core-once-normalized-markdown-test"), + channel: "telegram", + kind: "final", + }); + + const request = requireFirstSynthesisRequest("once-normalized voice-note synthesis request"); + expect(request.text).toBe( + 'This short explanation keeps the fenced literal below from becoming code-heavy.\n\nconst literal = "[x](y)";', + ); + expect(result.text).toBe(text); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + + it("skips channel auto-TTS audio for code-heavy replies", async () => { + const text = "```ts\nexport function answer() {\n return 42;\n}\n```"; + const result = await maybeApplyTtsToPayload({ + payload: { text }, + cfg: createTtsConfig("openclaw-speech-core-code-heavy-voice-note-test"), + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).not.toHaveBeenCalled(); + expect(result).toEqual({ text }); + }); + + it("synthesizes code-heavy explicitly tagged hidden TTS text", async () => { + const cfg = createTtsConfig("openclaw-speech-core-code-heavy-hidden-tts-test"); + let mediaDir: string | undefined; + try { + const result = await maybeApplyTtsToPayload({ + payload: { + text: '[[tts:text]]```ts\nconst detailedAnswer = "this code should still be spoken";\n```[[/tts:text]]', + audioAsVoice: true, + }, + cfg, + channel: "telegram", + kind: "final", + }); + + expect(synthesizeMock).toHaveBeenCalled(); + const request = requireFirstSynthesisRequest("code-heavy hidden TTS request"); + expect(request.text).toBe('const detailedAnswer = "this code should still be spoken";'); + expect(result.text).toBeUndefined(); + mediaDir = result.mediaUrl ? path.dirname(result.mediaUrl) : undefined; + } finally { + if (mediaDir) { + rmSync(mediaDir, { recursive: true, force: true }); + } + } + }); + it("synthesizes explicitly tagged short hidden TTS text", async () => { const cfg = createTtsConfig("openclaw-speech-core-short-hidden-tts-test"); let mediaDir: string | undefined; diff --git a/packages/speech-core/src/tts.ts b/packages/speech-core/src/tts.ts index e0243a067e4..68b4a8ac889 100644 --- a/packages/speech-core/src/tts.ts +++ b/packages/speech-core/src/tts.ts @@ -37,7 +37,6 @@ import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { stripMarkdown } from "openclaw/plugin-sdk/text-chunking"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { withSpeakerSelectionCompat } from "../speaker.js"; import { @@ -51,6 +50,7 @@ import { type VoiceProviderCandidate, } from "../voice-models.js"; import { assertSpeechRuntimeAvailable, isSpeechRuntimeAvailable } from "./runtime-availability.js"; +import { isCodeHeavySpeechText, normalizeSpeechText } from "./speech-text.js"; import { DEFAULT_TTS_TIMEOUT_MS, asProviderConfig, @@ -1074,6 +1074,7 @@ export async function synthesizeSpeech(params: { } const { cfg, config, persona, providers } = setup; + const textForSynthesis = normalizeSpeechText(params.text); const target = resolveTtsSynthesisTarget(params.channel); const errors: string[] = []; @@ -1122,7 +1123,7 @@ export async function synthesizeSpeech(params: { }); const prepared = await prepareSpeechSynthesis({ provider: resolvedProvider.provider, - text: params.text, + text: textForSynthesis, cfg, providerConfig: resolvedProvider.providerConfig, providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.provider.id], @@ -1660,6 +1661,12 @@ export async function maybeApplyTtsToPayload(params: { let textForAudio = ttsText.trim(); let wasSummarized = false; + if (!explicitTtsText && isCodeHeavySpeechText(textForAudio)) { + // The visible reply already carries code-heavy detail. Skip noisy voice-note audio instead of + // telling channel users to look at a screen they may not have. + return nextPayload; + } + if (textForAudio.length > maxLength) { if (!isSummarizationEnabled(prefsPath)) { logVerbose( @@ -1691,11 +1698,11 @@ export async function maybeApplyTtsToPayload(params: { } } - textForAudio = stripMarkdown(textForAudio).trim(); - if (!textForAudio) { + const normalizedTextForAudio = normalizeSpeechText(textForAudio); + if (!normalizedTextForAudio) { return nextPayload; } - if (!explicitTtsText && textForAudio.length < 10) { + if (!explicitTtsText && normalizedTextForAudio.length < 10) { return nextPayload; } diff --git a/src/gateway/server-methods/talk.ts b/src/gateway/server-methods/talk.ts index 6f085fcee95..776030286fe 100644 --- a/src/gateway/server-methods/talk.ts +++ b/src/gateway/server-methods/talk.ts @@ -20,6 +20,10 @@ import { withSpeakerSelectionCompat, withSpeakerSelectionFallbackCompat, } from "../../../packages/speech-core/speaker.js"; +import { + CODE_HEAVY_SPOKEN_FALLBACK, + isCodeHeavySpeechText, +} from "../../../packages/speech-core/src/speech-text.js"; import { getVoiceProviderConfig } from "../../../packages/speech-core/voice-models.js"; import { readConfigFileSnapshot } from "../../config/config.js"; import { redactConfigObject } from "../../config/redact-snapshot.js"; @@ -811,8 +815,9 @@ export const talkHandlers: GatewayRequestHandlers = { runtimeConfig, typedParams, ); + const speechText = isCodeHeavySpeechText(text) ? CODE_HEAVY_SPOKEN_FALLBACK : text; const result = await synthesizeSpeech({ - text, + text: speechText, cfg: setup.cfg, overrides, disableFallback: true, diff --git a/src/gateway/server.talk-runtime.test.ts b/src/gateway/server.talk-runtime.test.ts index 345e5a91ff4..bdfc20d0cc2 100644 --- a/src/gateway/server.talk-runtime.test.ts +++ b/src/gateway/server.talk-runtime.test.ts @@ -2,6 +2,7 @@ * Tests gateway talk runtime wiring for speech provider execution. */ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { CODE_HEAVY_SPOKEN_FALLBACK } from "../../packages/speech-core/src/speech-text.js"; import { invokeTalkSpeakDirect, type TalkSpeakTestPayload, @@ -186,6 +187,27 @@ describe("gateway talk runtime", () => { ); }); + it("uses the spoken fallback for code-heavy talk.speak replies", async () => { + await setAcmeTalkConfig(); + + await withAcmeSpeechProvider( + async () => ({ + audioBuffer: Buffer.from([7, 8, 9]), + outputFormat: "mp3", + fileExtension: ".mp3", + voiceCompatible: false, + }), + async () => { + const res = await invokeTalkSpeakDirect({ + text: "```ts\nexport function answer() {\n return 42;\n}\n```", + }); + + expect(res?.ok, JSON.stringify(res?.error)).toBe(true); + expect(expectSingleSynthesizeSpeechCall().text).toBe(CODE_HEAVY_SPOKEN_FALLBACK); + }, + ); + }); + it("resolves talk voice aliases case-insensitively and forwards provider overrides", async () => { await setElevenLabsTalkConfig(); diff --git a/src/shared/text/strip-markdown.ts b/src/shared/text/strip-markdown.ts index 6e2fd378b14..073155f17b6 100644 --- a/src/shared/text/strip-markdown.ts +++ b/src/shared/text/strip-markdown.ts @@ -8,6 +8,8 @@ type StripMarkdownOptions = { assistantTranscriptRolePrefix?: string; /** Link projection after formatting is removed. Default: label-and-url. */ linkStyle?: "label" | "label-and-url"; + /** Plain-text cleanup target. Speech removes decorative symbol and punctuation runs. */ + mode?: "plain-text" | "speech"; }; type PlainTextInsertion = { @@ -83,6 +85,26 @@ function applyPlainTextInsertions(text: string, insertions: PlainTextInsertion[] return output + text.slice(cursor); } +function cleanSpeechText(text: string): string { + return text + .split("\n") + .map((line) => { + if (/^[\p{P}\p{S}\s]+$/u.test(line)) { + return ""; + } + return line + .replace(/^[•◦▪‣⁃]\s+/u, "") + .replace(/(?:[\p{So}\p{Sk}]\s*){2,}/gu, " ") + .replace(/\.{4,}/g, "...") + .replace(/([!?,;:])\1+/g, "$1") + .replace(/[ \t]{2,}/g, " ") + .trim(); + }) + .join("\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + /** Parse Markdown, then protect role headers exposed by the final plain-text projection. */ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): string { // The IR parser preserves links when role annotations are enabled so this @@ -105,8 +127,9 @@ export function stripMarkdown(text: string, options: StripMarkdownOptions = {}): ...collectLinkInsertions(ir, options), ...collectParsedAssistantTranscriptRoleInsertions(ir, options), ]).trim(); - return applyPlainTextInsertions( + const projected = applyPlainTextInsertions( plainText, collectAssistantTranscriptRoleInsertions(plainText, options), ).trim(); + return options.mode === "speech" ? cleanSpeechText(projected) : projected; }