diff --git a/README.md b/README.md index 73ab4430a..b75be4049 100644 --- a/README.md +++ b/README.md @@ -635,6 +635,8 @@ Gateway-generated follow-up suggestions now normalize both plain-string model ou The Web UI composer can polish draft input before sending. The rewrite runs as a short Gateway LLM request using the `input_polish` model configuration, keeps slash skill prefixes such as `/data-analysis`, and only replaces the local draft after the user clicks the polish button; it does not create a thread run or persist a message. +The Web UI composer also supports browser-based voice dictation when the browser exposes the Web Speech API. The microphone button transcribes speech into the local draft only; DeerFlow receives only the transcribed text, while audio handling is delegated to the browser or operating system speech-recognition service according to that environment's policy. Users can review or edit the text before sending. + Interrupted first-turn runs still persist a fallback conversation title, so stopping a streaming response does not leave the thread as "Untitled" after refresh. In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation. diff --git a/README_zh.md b/README_zh.md index f6b522586..0c7bcf313 100644 --- a/README_zh.md +++ b/README_zh.md @@ -592,6 +592,8 @@ DEERFLOW_LANGGRAPH_URL=http://localhost:2026/api/langgraph # LangGraph API 完整 API 说明见 [`skills/public/claude-to-deerflow/SKILL.md`](skills/public/claude-to-deerflow/SKILL.md)。 +Web UI 输入框支持浏览器侧语音听写。浏览器提供 Web Speech API 时，麦克风按钮会把语音转写为本地草稿；DeerFlow 只接收转写后的文本，音频处理交由浏览器或操作系统语音识别服务按其环境策略完成。用户可以在发送前继续检查和编辑文本。 + ### Session Goals 用 `/goal <完成条件>` 为当前 thread 绑定一个激活态的完成条件。这个 goal 是 thread 维度的状态，而不是技能激活，所以它会跨轮次持续生效，直到 DeerFlow 判定它已被满足、或者你手动清除它。 diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index 8bb6ea196..62a833402 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -53,7 +53,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat - `workspace/` — Chat page components (messages, artifacts, settings) - `landing/` — Landing page sections - `docs/` — Docs / MDX rendering components -- **`core/`** — Business logic, the heart of the app. Domains include `threads/` (creation, streaming, state), `api/` (LangGraph client singleton), `agents/` (custom agents), `auth/` (authentication), `artifacts/`, `channels/` (IM connections), `i18n/` (en-US, zh-CN), `settings/`, `memory/`, `skills/`, `messages/`, `mcp/`, `models/`, `input-polish/` (pre-send draft rewrite API), `suggestions/`, `tasks/`, `todos/`, `tools/`, `workspace-changes/` (run-scoped changed-file summaries and diff fetching), `config/`, `notification/`, `blog/`, plus rendering helpers (`rehype/`, `streamdown/`) and `utils/`. +- **`core/`** — Business logic, the heart of the app. Domains include `threads/` (creation, streaming, state), `api/` (LangGraph client singleton), `agents/` (custom agents), `auth/` (authentication), `artifacts/`, `channels/` (IM connections), `i18n/` (en-US, zh-CN), `settings/`, `memory/`, `skills/`, `messages/`, `mcp/`, `models/`, `input-polish/` (pre-send draft rewrite API), `voice-input/` (browser speech-recognition helpers), `suggestions/`, `tasks/`, `todos/`, `tools/`, `workspace-changes/` (run-scoped changed-file summaries and diff fetching), `config/`, `notification/`, `blog/`, plus rendering helpers (`rehype/`, `streamdown/`) and `utils/`. - **`hooks/`** — Shared React hooks - **`lib/`** — Utilities (`cn()` from clsx + tailwind-merge) - **`content/`** — MDX content (blog posts, docs) rendered by the app @@ -63,7 +63,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat ### Data Flow -1. Optional composer helpers such as `core/input-polish` can rewrite the local draft before submission; confirmed user input then flows to thread hooks (`core/threads/hooks.ts`) → LangGraph SDK streaming +1. Optional composer helpers such as `core/input-polish` can rewrite the local draft before submission, and `core/voice-input` can transcribe browser microphone input into that same local draft; confirmed user input then flows to thread hooks (`core/threads/hooks.ts`) → LangGraph SDK streaming 2. Stream events update thread state (messages, artifacts, todos, goal) 3. Stop actions call the LangGraph SDK stream stop path; `core/threads/hooks.ts` invalidates current-thread, token-usage, and sidebar/search caches immediately and schedules one follow-up refetch because SDK stop may finish via abort + fire-and-forget cancel before backend title finalization commits 4. TanStack Query manages server state; localStorage stores user settings diff --git a/frontend/src/components/workspace/input-box.tsx b/frontend/src/components/workspace/input-box.tsx index 60e390355..b5cf5eee4 100644 --- a/frontend/src/components/workspace/input-box.tsx +++ b/frontend/src/components/workspace/input-box.tsx @@ -8,10 +8,12 @@ import { GraduationCapIcon, LightbulbIcon, Loader2Icon, + MicIcon, PaperclipIcon, PlusIcon, RocketIcon, SparklesIcon, + SquareIcon, TargetIcon, Undo2Icon, XIcon, @@ -89,6 +91,16 @@ import { type UploadLimits, type UploadLimitViolation, } from "@/core/uploads"; +import { + appendSpeechTranscript, + getSpeechRecognitionConstructor, + getSpeechRecognitionLanguage, + mapSpeechRecognitionError, + readSpeechRecognitionTranscript, + shouldRestartSpeechRecognition, + type BrowserSpeechRecognition, + type SpeechRecognitionErrorKind, +} from "@/core/voice-input/speech-recognition"; import { isIMEComposing } from "@/lib/ime"; import { cn } from "@/lib/utils"; @@ -200,6 +212,10 @@ export type InputBoxSubmitOptions = { onSent?: () => void; }; +type VoiceRecognitionStartOptions = { + focusAfterStart?: boolean; +}; + function buildHiddenConversationQuoteMessage({ contexts, }: { @@ -326,6 +342,17 @@ export function InputBox({ controller: null, sequence: 0, }); + const voiceRecognitionRef = useRef(null); + const voiceBaseTextRef = useRef(""); + const voiceLatestTextRef = useRef(""); + const voiceLastErrorKindRef = useRef(null); + const voiceStopRequestedRef = useRef(false); + const voiceRestartTimerRef = useRef | null>( + null, + ); + const startVoiceRecognitionRef = useRef< + ((options?: VoiceRecognitionStartOptions) => boolean) | null + >(null); const promptHistoryIndexRef = useRef(null); const promptHistoryDraftRef = useRef(""); @@ -336,6 +363,7 @@ export function InputBox({ const [followupsHidden, setFollowupsHidden] = useState(false); const [followupsLoading, setFollowupsLoading] = useState(false); const [polishingInput, setPolishingInput] = useState(false); + const [voiceListening, setVoiceListening] = useState(false); const [inputPolishUndo, setInputPolishUndo] = useState<{ originalText: string; rewrittenText: string; @@ -350,6 +378,58 @@ export function InputBox({ const wasStreamingRef = useRef(false); const messagesRef = useRef(thread.messages); + const clearVoiceRestartTimer = useCallback(() => { + if (voiceRestartTimerRef.current === null) { + return; + } + clearTimeout(voiceRestartTimerRef.current); + voiceRestartTimerRef.current = null; + }, []); + + const cleanupVoiceRecognition = useCallback( + ( + recognition: BrowserSpeechRecognition | null, + options: { keepListening?: boolean } = {}, + ) => { + clearVoiceRestartTimer(); + if (!recognition) { + if (!options.keepListening) { + voiceLastErrorKindRef.current = null; + voiceStopRequestedRef.current = false; + setVoiceListening(false); + } + return; + } + recognition.onend = null; + recognition.onerror = null; + recognition.onresult = null; + if (voiceRecognitionRef.current === recognition) { + voiceRecognitionRef.current = null; + } + if (!options.keepListening) { + voiceLastErrorKindRef.current = null; + voiceStopRequestedRef.current = false; + setVoiceListening(false); + } + }, + [clearVoiceRestartTimer], + ); + + const abortVoiceInput = useCallback(() => { + const recognition = voiceRecognitionRef.current; + voiceStopRequestedRef.current = true; + if (!recognition) { + cleanupVoiceRecognition(null); + return; + } + cleanupVoiceRecognition(recognition); + try { + recognition.abort(); + } catch { + // Browser implementations can throw when the recognizer already ended. + } + }, [cleanupVoiceRecognition]); + const [confirmOpen, setConfirmOpen] = useState(false); const [pendingSuggestion, setPendingSuggestion] = useState( null, @@ -852,6 +932,7 @@ export function InputBox({ toast.info(t.inputBox.pleaseWaitStreaming); return Promise.reject(new Error("streaming")); } + abortVoiceInput(); const messageWithSlashSkill = selectedSlashSkill ? { ...message, @@ -896,6 +977,7 @@ export function InputBox({ } }, [ + abortVoiceInput, handleCompactCommand, handleGoalCommand, onStop, @@ -1003,6 +1085,193 @@ export function InputBox({ (status === "streaming" || slashSkillQuery !== null || !canPolishInput(textInput.value ?? ""))); + const speechRecognitionConstructor = useMemo( + () => + typeof window === "undefined" + ? null + : getSpeechRecognitionConstructor(window), + [], + ); + const voiceInputSupported = speechRecognitionConstructor !== null; + + const getVoiceInputErrorMessage = useCallback( + (kind: SpeechRecognitionErrorKind) => { + switch (kind) { + case "permission_denied": + return t.inputBox.voiceInputPermissionDenied; + case "microphone_unavailable": + return t.inputBox.voiceInputMicrophoneUnavailable; + case "unsupported_language": + return t.inputBox.voiceInputUnsupportedLanguage; + case "network": + return t.inputBox.voiceInputNetworkError; + case "no_speech": + return t.inputBox.voiceInputNoSpeech; + case "cancelled": + return null; + default: + return t.inputBox.voiceInputFailed; + } + }, + [t], + ); + + const startVoiceRecognition = useCallback( + (options: VoiceRecognitionStartOptions = {}) => { + if (composerLocked || !speechRecognitionConstructor) { + return false; + } + + const recognition = new speechRecognitionConstructor(); + recognition.continuous = true; + recognition.interimResults = true; + recognition.lang = getSpeechRecognitionLanguage(locale); + recognition.maxAlternatives = 1; + voiceLastErrorKindRef.current = null; + voiceLatestTextRef.current = voiceBaseTextRef.current; + voiceRecognitionRef.current = recognition; + + recognition.onresult = (event) => { + if (voiceRecognitionRef.current !== recognition) { + return; + } + const transcript = readSpeechRecognitionTranscript(event.results).text; + const nextValue = appendSpeechTranscript( + voiceBaseTextRef.current, + transcript, + ); + voiceLatestTextRef.current = nextValue; + textInput.setInput(nextValue); + }; + recognition.onerror = (event) => { + const errorKind = mapSpeechRecognitionError(event.error); + voiceLastErrorKindRef.current = errorKind; + if ( + !voiceStopRequestedRef.current && + shouldRestartSpeechRecognition(errorKind) + ) { + return; + } + + const message = getVoiceInputErrorMessage(errorKind); + if (message) { + toast.error(message); + } + }; + recognition.onend = () => { + const shouldRestart = + voiceRecognitionRef.current === recognition && + !voiceStopRequestedRef.current && + shouldRestartSpeechRecognition(voiceLastErrorKindRef.current); + if (shouldRestart) { + voiceBaseTextRef.current = voiceLatestTextRef.current; + cleanupVoiceRecognition(recognition, { keepListening: true }); + voiceRestartTimerRef.current = setTimeout(() => { + voiceRestartTimerRef.current = null; + if (voiceStopRequestedRef.current) { + cleanupVoiceRecognition(null); + return; + } + const restarted = startVoiceRecognitionRef.current?.() ?? false; + if (!restarted) { + cleanupVoiceRecognition(null); + } + }, 150); + return; + } + cleanupVoiceRecognition(recognition); + }; + + setVoiceListening(true); + try { + recognition.start(); + if (options.focusAfterStart) { + requestAnimationFrame(() => { + if (selectedSlashSkill) { + focusContentEditableEnd(inlineSkillTextRef.current); + } else { + textareaRef.current?.focus(); + } + }); + } + return true; + } catch { + cleanupVoiceRecognition(recognition); + toast.error(t.inputBox.voiceInputFailed); + return false; + } + }, + [ + cleanupVoiceRecognition, + composerLocked, + getVoiceInputErrorMessage, + locale, + selectedSlashSkill, + speechRecognitionConstructor, + t.inputBox.voiceInputFailed, + textInput, + ], + ); + + useEffect(() => { + startVoiceRecognitionRef.current = startVoiceRecognition; + }, [startVoiceRecognition]); + + const stopVoiceInput = useCallback(() => { + const recognition = voiceRecognitionRef.current; + voiceStopRequestedRef.current = true; + if (!recognition) { + cleanupVoiceRecognition(null); + return; + } + try { + recognition.stop(); + } catch { + cleanupVoiceRecognition(recognition); + } + }, [cleanupVoiceRecognition]); + + const toggleVoiceInput = useCallback(() => { + if (voiceListening) { + stopVoiceInput(); + return; + } + if (composerLocked) { + return; + } + if (!speechRecognitionConstructor) { + toast.error(t.inputBox.voiceInputUnsupported); + return; + } + + abortInputPolishRequest(); + setInputPolishUndo(null); + promptHistoryIndexRef.current = null; + promptHistoryDraftRef.current = ""; + voiceStopRequestedRef.current = false; + voiceBaseTextRef.current = textInput.value ?? ""; + voiceLatestTextRef.current = voiceBaseTextRef.current; + startVoiceRecognition({ focusAfterStart: true }); + }, [ + abortInputPolishRequest, + composerLocked, + speechRecognitionConstructor, + startVoiceRecognition, + stopVoiceInput, + t.inputBox.voiceInputUnsupported, + textInput, + voiceListening, + ]); + + useEffect(() => { + if (composerLocked && voiceListening) { + stopVoiceInput(); + } + }, [composerLocked, stopVoiceInput, voiceListening]); + + useEffect(() => { + return () => abortVoiceInput(); + }, [abortVoiceInput, threadId]); useEffect(() => { setSkillSuggestionIndex(0); @@ -1282,19 +1551,25 @@ export function InputBox({ ); const handlePromptTextareaChange = useCallback(() => { + if (voiceListening) { + abortVoiceInput(); + } abortInputPolishRequest(); setInputPolishUndo(null); promptHistoryIndexRef.current = null; promptHistoryDraftRef.current = ""; - }, [abortInputPolishRequest]); + }, [abortInputPolishRequest, abortVoiceInput, voiceListening]); const updateInlineSkillTextInput = useCallback( (element: HTMLElement) => { + if (voiceListening) { + abortVoiceInput(); + } promptHistoryIndexRef.current = null; promptHistoryDraftRef.current = ""; textInput.setInput(element.textContent ?? ""); }, - [textInput], + [abortVoiceInput, textInput, voiceListening], ); useEffect(() => { @@ -1731,6 +2006,12 @@ export function InputBox({ disabled={composerLocked} uploadLimits={uploadLimits} /> + void; +}) { + const { t } = useI18n(); + const tooltipContent = !supported + ? t.inputBox.voiceInputUnsupported + : listening + ? t.inputBox.voiceInputListening + : t.inputBox.voiceInputStart; + const label = listening + ? t.inputBox.voiceInputStopLabel + : t.inputBox.voiceInputStartLabel; + + return ( + {tooltipContent}}> + + {listening ? ( + + ) : ( + + )} + + + ); +} + function SuggestionList({ onSelectPlaceholder, }: { diff --git a/frontend/src/core/i18n/locales/en-US.ts b/frontend/src/core/i18n/locales/en-US.ts index dcc49d697..fe3d94fc8 100644 --- a/frontend/src/core/i18n/locales/en-US.ts +++ b/frontend/src/core/i18n/locales/en-US.ts @@ -122,6 +122,24 @@ export const enUS: Translations = { inputPolishFailed: "Failed to polish input.", inputPolishUndo: "Undo polish", inputPolishCancel: "Cancel polishing", + voiceInputStartLabel: "Dictate with voice", + voiceInputStopLabel: "Stop voice input", + voiceInputStart: + "Dictate with voice. DeerFlow receives only transcribed text; audio is handled by your browser or system speech service.", + voiceInputStop: "Stop voice input", + voiceInputListening: "Listening... Click to stop voice input.", + voiceInputUnsupported: + "Voice input is not supported in this browser. Try Chrome or Edge.", + voiceInputPermissionDenied: + "Microphone access was denied. Allow microphone access and try again.", + voiceInputMicrophoneUnavailable: + "No microphone was detected. Check your device input and try again.", + voiceInputUnsupportedLanguage: + "Voice input does not support the current language in this browser.", + voiceInputNetworkError: + "Voice input could not reach the browser speech service.", + voiceInputNoSpeech: "No speech was detected. Please try again.", + voiceInputFailed: "Voice input failed. Please try again.", mode: "Mode", flashMode: "Flash", flashModeDescription: "Fast and efficient, but may not be accurate", diff --git a/frontend/src/core/i18n/locales/types.ts b/frontend/src/core/i18n/locales/types.ts index af8b04561..281b2929d 100644 --- a/frontend/src/core/i18n/locales/types.ts +++ b/frontend/src/core/i18n/locales/types.ts @@ -104,6 +104,18 @@ export interface Translations { inputPolishFailed: string; inputPolishUndo: string; inputPolishCancel: string; + voiceInputStartLabel: string; + voiceInputStopLabel: string; + voiceInputStart: string; + voiceInputStop: string; + voiceInputListening: string; + voiceInputUnsupported: string; + voiceInputPermissionDenied: string; + voiceInputMicrophoneUnavailable: string; + voiceInputUnsupportedLanguage: string; + voiceInputNetworkError: string; + voiceInputNoSpeech: string; + voiceInputFailed: string; mode: string; flashMode: string; flashModeDescription: string; diff --git a/frontend/src/core/i18n/locales/zh-CN.ts b/frontend/src/core/i18n/locales/zh-CN.ts index 9542be9ec..3a5e1984d 100644 --- a/frontend/src/core/i18n/locales/zh-CN.ts +++ b/frontend/src/core/i18n/locales/zh-CN.ts @@ -121,6 +121,20 @@ export const zhCN: Translations = { inputPolishFailed: "优化输入失败。", inputPolishUndo: "撤销优化", inputPolishCancel: "取消优化", + voiceInputStartLabel: "语音输入", + voiceInputStopLabel: "停止语音输入", + voiceInputStart: + "语音输入。DeerFlow 只接收转写文本，音频由浏览器或系统语音服务处理。", + voiceInputStop: "停止语音输入", + voiceInputListening: "正在聆听... 点击停止语音输入。", + voiceInputUnsupported: + "当前浏览器不支持语音输入。建议使用 Chrome 或 Edge。", + voiceInputPermissionDenied: "麦克风权限被拒绝。请允许麦克风访问后重试。", + voiceInputMicrophoneUnavailable: "未检测到麦克风。请检查设备输入后重试。", + voiceInputUnsupportedLanguage: "当前浏览器不支持该语言的语音输入。", + voiceInputNetworkError: "无法连接浏览器语音识别服务。", + voiceInputNoSpeech: "没有检测到语音，请重试。", + voiceInputFailed: "语音输入失败，请重试。", mode: "模式", flashMode: "闪速", flashModeDescription: "快速且高效的完成任务，但可能不够精准", diff --git a/frontend/src/core/voice-input/speech-recognition.ts b/frontend/src/core/voice-input/speech-recognition.ts new file mode 100644 index 000000000..ef0c28c9e --- /dev/null +++ b/frontend/src/core/voice-input/speech-recognition.ts @@ -0,0 +1,181 @@ +export type SpeechRecognitionErrorCode = + | "aborted" + | "audio-capture" + | "bad-grammar" + | "language-not-supported" + | "network" + | "no-speech" + | "not-allowed" + | "phrases-not-supported" + | "service-not-allowed"; + +export type SpeechRecognitionErrorKind = + | "cancelled" + | "microphone_unavailable" + | "permission_denied" + | "unsupported_language" + | "network" + | "no_speech" + | "unknown"; + +export type SpeechRecognitionConstructor = new () => BrowserSpeechRecognition; + +export type SpeechRecognitionEventLike = { + results: SpeechRecognitionResultListLike; +}; + +export type SpeechRecognitionErrorEventLike = { + error?: SpeechRecognitionErrorCode | string; +}; + +export type BrowserSpeechRecognition = { + continuous: boolean; + interimResults: boolean; + lang: string; + maxAlternatives: number; + onend: (() => void) | null; + onerror: ((event: SpeechRecognitionErrorEventLike) => void) | null; + onresult: ((event: SpeechRecognitionEventLike) => void) | null; + start: () => void; + stop: () => void; + abort: () => void; +}; + +type SpeechRecognitionWindow = Window & + typeof globalThis & { + SpeechRecognition?: SpeechRecognitionConstructor; + webkitSpeechRecognition?: SpeechRecognitionConstructor; + }; + +export type SpeechRecognitionAlternativeLike = { + transcript?: string; +}; + +export type SpeechRecognitionResultLike = { + 0?: SpeechRecognitionAlternativeLike; + isFinal: boolean; + length: number; +}; + +export type SpeechRecognitionResultListLike = { + [index: number]: SpeechRecognitionResultLike | undefined; + length: number; +}; + +const DEFAULT_SPEECH_RECOGNITION_LANGUAGE = "en-US"; +const SPEECH_RECOGNITION_LANGUAGE_ALLOWLIST = new Set([ + "de", + "en", + "es", + "fr", + "it", + "ja", + "ko", + "pt", + "zh", +]); + +export function getSpeechRecognitionConstructor( + value: unknown = globalThis, +): SpeechRecognitionConstructor | null { + const maybeWindow = value as Partial; + return ( + maybeWindow.SpeechRecognition ?? maybeWindow.webkitSpeechRecognition ?? null + ); +} + +export function getSpeechRecognitionLanguage(locale: string): string { + const normalized = normalizeBCP47Locale(locale); + const language = normalized.split("-")[0]?.toLowerCase(); + + if (language === "zh") { + return "zh-CN"; + } + if (language && SPEECH_RECOGNITION_LANGUAGE_ALLOWLIST.has(language)) { + return normalized; + } + return DEFAULT_SPEECH_RECOGNITION_LANGUAGE; +} + +export function shouldRestartSpeechRecognition( + lastError: SpeechRecognitionErrorKind | null, +): boolean { + return lastError === null || lastError === "no_speech"; +} + +export function readSpeechRecognitionTranscript( + results: SpeechRecognitionResultListLike, +): { finalText: string; interimText: string; text: string } { + let finalText = ""; + let interimText = ""; + + for (const result of Array.from( + { length: results.length }, + (_, index) => results[index], + )) { + const transcript = result?.[0]?.transcript ?? ""; + if (result?.isFinal) { + finalText += transcript; + } else { + interimText += transcript; + } + } + + return { + finalText: normalizeSpeechTranscript(finalText), + interimText: normalizeSpeechTranscript(interimText), + text: normalizeSpeechTranscript(`${finalText}${interimText}`), + }; +} + +export function appendSpeechTranscript(baseText: string, transcript: string) { + const cleanTranscript = normalizeSpeechTranscript(transcript); + if (!cleanTranscript) { + return baseText; + } + + const cleanBase = baseText.trimEnd(); + if (!cleanBase) { + return cleanTranscript; + } + + return `${cleanBase} ${cleanTranscript}`; +} + +export function normalizeSpeechTranscript(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +export function mapSpeechRecognitionError( + error: SpeechRecognitionErrorCode | string | undefined, +): SpeechRecognitionErrorKind { + switch (error) { + case "aborted": + return "cancelled"; + case "audio-capture": + return "microphone_unavailable"; + case "not-allowed": + case "service-not-allowed": + return "permission_denied"; + case "language-not-supported": + return "unsupported_language"; + case "network": + return "network"; + case "no-speech": + return "no_speech"; + default: + return "unknown"; + } +} + +function normalizeBCP47Locale(locale: string): string { + const trimmed = locale.trim(); + if (!trimmed) { + return DEFAULT_SPEECH_RECOGNITION_LANGUAGE; + } + try { + return Intl.getCanonicalLocales(trimmed)[0] ?? trimmed; + } catch { + return DEFAULT_SPEECH_RECOGNITION_LANGUAGE; + } +} diff --git a/frontend/tests/unit/core/voice-input/speech-recognition.test.ts b/frontend/tests/unit/core/voice-input/speech-recognition.test.ts new file mode 100644 index 000000000..ea3ea9d01 --- /dev/null +++ b/frontend/tests/unit/core/voice-input/speech-recognition.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "@rstest/core"; + +import { + appendSpeechTranscript, + getSpeechRecognitionConstructor, + getSpeechRecognitionLanguage, + mapSpeechRecognitionError, + readSpeechRecognitionTranscript, + shouldRestartSpeechRecognition, + type SpeechRecognitionConstructor, +} from "@/core/voice-input/speech-recognition"; + +describe("speech recognition helpers", () => { + it("prefers the standard constructor and falls back to webkit", () => { + const standard = makeSpeechRecognitionConstructor(); + const webkit = makeSpeechRecognitionConstructor(); + + expect( + getSpeechRecognitionConstructor({ + SpeechRecognition: standard as unknown as SpeechRecognitionConstructor, + webkitSpeechRecognition: + webkit as unknown as SpeechRecognitionConstructor, + }), + ).toBe(standard); + expect( + getSpeechRecognitionConstructor({ + webkitSpeechRecognition: + webkit as unknown as SpeechRecognitionConstructor, + }), + ).toBe(webkit); + expect(getSpeechRecognitionConstructor({})).toBeNull(); + }); + + it("maps DeerFlow locales to browser recognition locales", () => { + expect(getSpeechRecognitionLanguage("zh-CN")).toBe("zh-CN"); + expect(getSpeechRecognitionLanguage("zh-Hans")).toBe("zh-CN"); + expect(getSpeechRecognitionLanguage("en-US")).toBe("en-US"); + expect(getSpeechRecognitionLanguage("en-GB")).toBe("en-GB"); + expect(getSpeechRecognitionLanguage("fr-FR")).toBe("fr-FR"); + expect(getSpeechRecognitionLanguage("ja-JP")).toBe("ja-JP"); + expect(getSpeechRecognitionLanguage("xx-YY")).toBe("en-US"); + expect(getSpeechRecognitionLanguage("not a locale")).toBe("en-US"); + }); + + it("combines final and interim transcripts with whitespace cleanup", () => { + expect( + readSpeechRecognitionTranscript({ + length: 3, + 0: { isFinal: true, length: 1, 0: { transcript: " hello " } }, + 1: { isFinal: true, length: 1, 0: { transcript: " world" } }, + 2: { isFinal: false, length: 1, 0: { transcript: " again " } }, + }), + ).toEqual({ + finalText: "hello world", + interimText: "again", + text: "hello world again", + }); + }); + + it("appends transcript to an existing draft without duplicating whitespace", () => { + expect(appendSpeechTranscript("", " hello world ")).toBe("hello world"); + expect(appendSpeechTranscript("Draft", "voice text")).toBe( + "Draft voice text", + ); + expect(appendSpeechTranscript("Draft\n", "voice text")).toBe( + "Draft voice text", + ); + expect(appendSpeechTranscript("Draft", " ")).toBe("Draft"); + }); + + it("normalizes browser speech recognition errors", () => { + expect(mapSpeechRecognitionError("not-allowed")).toBe("permission_denied"); + expect(mapSpeechRecognitionError("service-not-allowed")).toBe( + "permission_denied", + ); + expect(mapSpeechRecognitionError("audio-capture")).toBe( + "microphone_unavailable", + ); + expect(mapSpeechRecognitionError("language-not-supported")).toBe( + "unsupported_language", + ); + expect(mapSpeechRecognitionError("network")).toBe("network"); + expect(mapSpeechRecognitionError("no-speech")).toBe("no_speech"); + expect(mapSpeechRecognitionError("aborted")).toBe("cancelled"); + expect(mapSpeechRecognitionError("bad-grammar")).toBe("unknown"); + }); + + it("restarts only after browser auto-end conditions", () => { + expect(shouldRestartSpeechRecognition(null)).toBe(true); + expect(shouldRestartSpeechRecognition("no_speech")).toBe(true); + expect(shouldRestartSpeechRecognition("cancelled")).toBe(false); + expect(shouldRestartSpeechRecognition("permission_denied")).toBe(false); + expect(shouldRestartSpeechRecognition("network")).toBe(false); + expect(shouldRestartSpeechRecognition("unknown")).toBe(false); + }); +}); + +function makeSpeechRecognitionConstructor(): SpeechRecognitionConstructor { + return class { + continuous = false; + interimResults = false; + lang = ""; + maxAlternatives = 1; + onend = null; + onerror = null; + onresult = null; + + start() { + return undefined; + } + + stop() { + return undefined; + } + + abort() { + return undefined; + } + }; +}