feat(frontend): add voice dictation input (#4036)

* feat(frontend): add voice dictation input

* fix(frontend): address voice input review feedback

* ci: rerun frontend checks
This commit is contained in:
Ryker_Feng
2026-07-10 23:13:06 +08:00
committed by GitHub
parent a2a949b178
commit be637163a6
9 changed files with 678 additions and 4 deletions
+2
View File
@@ -635,6 +635,8 @@ Gateway-generated follow-up suggestions now normalize both plain-string model ou
The Web UI composer can polish draft input before sending. The rewrite runs as a short Gateway LLM request using the `input_polish` model configuration, keeps slash skill prefixes such as `/data-analysis`, and only replaces the local draft after the user clicks the polish button; it does not create a thread run or persist a message.
The Web UI composer also supports browser-based voice dictation when the browser exposes the Web Speech API. The microphone button transcribes speech into the local draft only; DeerFlow receives only the transcribed text, while audio handling is delegated to the browser or operating system speech-recognition service according to that environment's policy. Users can review or edit the text before sending.
Interrupted first-turn runs still persist a fallback conversation title, so stopping a streaming response does not leave the thread as "Untitled" after refresh.
In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation.
+2
View File
@@ -592,6 +592,8 @@ DEERFLOW_LANGGRAPH_URL=http://localhost:2026/api/langgraph # LangGraph API
完整 API 说明见 [`skills/public/claude-to-deerflow/SKILL.md`](skills/public/claude-to-deerflow/SKILL.md)。
Web UI 输入框支持浏览器侧语音听写。浏览器提供 Web Speech API 时,麦克风按钮会把语音转写为本地草稿;DeerFlow 只接收转写后的文本,音频处理交由浏览器或操作系统语音识别服务按其环境策略完成。用户可以在发送前继续检查和编辑文本。
### Session Goals
用 `/goal <完成条件>` 为当前 thread 绑定一个激活态的完成条件。这个 goal 是 thread 维度的状态,而不是技能激活,所以它会跨轮次持续生效,直到 DeerFlow 判定它已被满足、或者你手动清除它。
+2 -2
View File
@@ -53,7 +53,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat
- `workspace/` — Chat page components (messages, artifacts, settings)
- `landing/` — Landing page sections
- `docs/` — Docs / MDX rendering components
- **`core/`** — Business logic, the heart of the app. Domains include `threads/` (creation, streaming, state), `api/` (LangGraph client singleton), `agents/` (custom agents), `auth/` (authentication), `artifacts/`, `channels/` (IM connections), `i18n/` (en-US, zh-CN), `settings/`, `memory/`, `skills/`, `messages/`, `mcp/`, `models/`, `input-polish/` (pre-send draft rewrite API), `suggestions/`, `tasks/`, `todos/`, `tools/`, `workspace-changes/` (run-scoped changed-file summaries and diff fetching), `config/`, `notification/`, `blog/`, plus rendering helpers (`rehype/`, `streamdown/`) and `utils/`.
- **`core/`** — Business logic, the heart of the app. Domains include `threads/` (creation, streaming, state), `api/` (LangGraph client singleton), `agents/` (custom agents), `auth/` (authentication), `artifacts/`, `channels/` (IM connections), `i18n/` (en-US, zh-CN), `settings/`, `memory/`, `skills/`, `messages/`, `mcp/`, `models/`, `input-polish/` (pre-send draft rewrite API), `voice-input/` (browser speech-recognition helpers), `suggestions/`, `tasks/`, `todos/`, `tools/`, `workspace-changes/` (run-scoped changed-file summaries and diff fetching), `config/`, `notification/`, `blog/`, plus rendering helpers (`rehype/`, `streamdown/`) and `utils/`.
- **`hooks/`** — Shared React hooks
- **`lib/`** — Utilities (`cn()` from clsx + tailwind-merge)
- **`content/`** — MDX content (blog posts, docs) rendered by the app
@@ -63,7 +63,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat
### Data Flow
1. Optional composer helpers such as `core/input-polish` can rewrite the local draft before submission; confirmed user input then flows to thread hooks (`core/threads/hooks.ts`) → LangGraph SDK streaming
1. Optional composer helpers such as `core/input-polish` can rewrite the local draft before submission, and `core/voice-input` can transcribe browser microphone input into that same local draft; confirmed user input then flows to thread hooks (`core/threads/hooks.ts`) → LangGraph SDK streaming
2. Stream events update thread state (messages, artifacts, todos, goal)
3. Stop actions call the LangGraph SDK stream stop path; `core/threads/hooks.ts` invalidates current-thread, token-usage, and sidebar/search caches immediately and schedules one follow-up refetch because SDK stop may finish via abort + fire-and-forget cancel before backend title finalization commits
4. TanStack Query manages server state; localStorage stores user settings
+327 -2
View File
@@ -8,10 +8,12 @@ import {
GraduationCapIcon,
LightbulbIcon,
Loader2Icon,
MicIcon,
PaperclipIcon,
PlusIcon,
RocketIcon,
SparklesIcon,
SquareIcon,
TargetIcon,
Undo2Icon,
XIcon,
@@ -89,6 +91,16 @@ import {
type UploadLimits,
type UploadLimitViolation,
} from "@/core/uploads";
import {
appendSpeechTranscript,
getSpeechRecognitionConstructor,
getSpeechRecognitionLanguage,
mapSpeechRecognitionError,
readSpeechRecognitionTranscript,
shouldRestartSpeechRecognition,
type BrowserSpeechRecognition,
type SpeechRecognitionErrorKind,
} from "@/core/voice-input/speech-recognition";
import { isIMEComposing } from "@/lib/ime";
import { cn } from "@/lib/utils";
@@ -200,6 +212,10 @@ export type InputBoxSubmitOptions = {
onSent?: () => void;
};
type VoiceRecognitionStartOptions = {
focusAfterStart?: boolean;
};
function buildHiddenConversationQuoteMessage({
contexts,
}: {
@@ -326,6 +342,17 @@ export function InputBox({
controller: null,
sequence: 0,
});
const voiceRecognitionRef = useRef<BrowserSpeechRecognition | null>(null);
const voiceBaseTextRef = useRef("");
const voiceLatestTextRef = useRef("");
const voiceLastErrorKindRef = useRef<SpeechRecognitionErrorKind | null>(null);
const voiceStopRequestedRef = useRef(false);
const voiceRestartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(
null,
);
const startVoiceRecognitionRef = useRef<
((options?: VoiceRecognitionStartOptions) => boolean) | null
>(null);
const promptHistoryIndexRef = useRef<number | null>(null);
const promptHistoryDraftRef = useRef("");
@@ -336,6 +363,7 @@ export function InputBox({
const [followupsHidden, setFollowupsHidden] = useState(false);
const [followupsLoading, setFollowupsLoading] = useState(false);
const [polishingInput, setPolishingInput] = useState(false);
const [voiceListening, setVoiceListening] = useState(false);
const [inputPolishUndo, setInputPolishUndo] = useState<{
originalText: string;
rewrittenText: string;
@@ -350,6 +378,58 @@ export function InputBox({
const wasStreamingRef = useRef(false);
const messagesRef = useRef(thread.messages);
const clearVoiceRestartTimer = useCallback(() => {
if (voiceRestartTimerRef.current === null) {
return;
}
clearTimeout(voiceRestartTimerRef.current);
voiceRestartTimerRef.current = null;
}, []);
const cleanupVoiceRecognition = useCallback(
(
recognition: BrowserSpeechRecognition | null,
options: { keepListening?: boolean } = {},
) => {
clearVoiceRestartTimer();
if (!recognition) {
if (!options.keepListening) {
voiceLastErrorKindRef.current = null;
voiceStopRequestedRef.current = false;
setVoiceListening(false);
}
return;
}
recognition.onend = null;
recognition.onerror = null;
recognition.onresult = null;
if (voiceRecognitionRef.current === recognition) {
voiceRecognitionRef.current = null;
}
if (!options.keepListening) {
voiceLastErrorKindRef.current = null;
voiceStopRequestedRef.current = false;
setVoiceListening(false);
}
},
[clearVoiceRestartTimer],
);
const abortVoiceInput = useCallback(() => {
const recognition = voiceRecognitionRef.current;
voiceStopRequestedRef.current = true;
if (!recognition) {
cleanupVoiceRecognition(null);
return;
}
cleanupVoiceRecognition(recognition);
try {
recognition.abort();
} catch {
// Browser implementations can throw when the recognizer already ended.
}
}, [cleanupVoiceRecognition]);
const [confirmOpen, setConfirmOpen] = useState(false);
const [pendingSuggestion, setPendingSuggestion] = useState<string | null>(
null,
@@ -852,6 +932,7 @@ export function InputBox({
toast.info(t.inputBox.pleaseWaitStreaming);
return Promise.reject(new Error("streaming"));
}
abortVoiceInput();
const messageWithSlashSkill = selectedSlashSkill
? {
...message,
@@ -896,6 +977,7 @@ export function InputBox({
}
},
[
abortVoiceInput,
handleCompactCommand,
handleGoalCommand,
onStop,
@@ -1003,6 +1085,193 @@ export function InputBox({
(status === "streaming" ||
slashSkillQuery !== null ||
!canPolishInput(textInput.value ?? "")));
const speechRecognitionConstructor = useMemo(
() =>
typeof window === "undefined"
? null
: getSpeechRecognitionConstructor(window),
[],
);
const voiceInputSupported = speechRecognitionConstructor !== null;
const getVoiceInputErrorMessage = useCallback(
(kind: SpeechRecognitionErrorKind) => {
switch (kind) {
case "permission_denied":
return t.inputBox.voiceInputPermissionDenied;
case "microphone_unavailable":
return t.inputBox.voiceInputMicrophoneUnavailable;
case "unsupported_language":
return t.inputBox.voiceInputUnsupportedLanguage;
case "network":
return t.inputBox.voiceInputNetworkError;
case "no_speech":
return t.inputBox.voiceInputNoSpeech;
case "cancelled":
return null;
default:
return t.inputBox.voiceInputFailed;
}
},
[t],
);
const startVoiceRecognition = useCallback(
(options: VoiceRecognitionStartOptions = {}) => {
if (composerLocked || !speechRecognitionConstructor) {
return false;
}
const recognition = new speechRecognitionConstructor();
recognition.continuous = true;
recognition.interimResults = true;
recognition.lang = getSpeechRecognitionLanguage(locale);
recognition.maxAlternatives = 1;
voiceLastErrorKindRef.current = null;
voiceLatestTextRef.current = voiceBaseTextRef.current;
voiceRecognitionRef.current = recognition;
recognition.onresult = (event) => {
if (voiceRecognitionRef.current !== recognition) {
return;
}
const transcript = readSpeechRecognitionTranscript(event.results).text;
const nextValue = appendSpeechTranscript(
voiceBaseTextRef.current,
transcript,
);
voiceLatestTextRef.current = nextValue;
textInput.setInput(nextValue);
};
recognition.onerror = (event) => {
const errorKind = mapSpeechRecognitionError(event.error);
voiceLastErrorKindRef.current = errorKind;
if (
!voiceStopRequestedRef.current &&
shouldRestartSpeechRecognition(errorKind)
) {
return;
}
const message = getVoiceInputErrorMessage(errorKind);
if (message) {
toast.error(message);
}
};
recognition.onend = () => {
const shouldRestart =
voiceRecognitionRef.current === recognition &&
!voiceStopRequestedRef.current &&
shouldRestartSpeechRecognition(voiceLastErrorKindRef.current);
if (shouldRestart) {
voiceBaseTextRef.current = voiceLatestTextRef.current;
cleanupVoiceRecognition(recognition, { keepListening: true });
voiceRestartTimerRef.current = setTimeout(() => {
voiceRestartTimerRef.current = null;
if (voiceStopRequestedRef.current) {
cleanupVoiceRecognition(null);
return;
}
const restarted = startVoiceRecognitionRef.current?.() ?? false;
if (!restarted) {
cleanupVoiceRecognition(null);
}
}, 150);
return;
}
cleanupVoiceRecognition(recognition);
};
setVoiceListening(true);
try {
recognition.start();
if (options.focusAfterStart) {
requestAnimationFrame(() => {
if (selectedSlashSkill) {
focusContentEditableEnd(inlineSkillTextRef.current);
} else {
textareaRef.current?.focus();
}
});
}
return true;
} catch {
cleanupVoiceRecognition(recognition);
toast.error(t.inputBox.voiceInputFailed);
return false;
}
},
[
cleanupVoiceRecognition,
composerLocked,
getVoiceInputErrorMessage,
locale,
selectedSlashSkill,
speechRecognitionConstructor,
t.inputBox.voiceInputFailed,
textInput,
],
);
useEffect(() => {
startVoiceRecognitionRef.current = startVoiceRecognition;
}, [startVoiceRecognition]);
const stopVoiceInput = useCallback(() => {
const recognition = voiceRecognitionRef.current;
voiceStopRequestedRef.current = true;
if (!recognition) {
cleanupVoiceRecognition(null);
return;
}
try {
recognition.stop();
} catch {
cleanupVoiceRecognition(recognition);
}
}, [cleanupVoiceRecognition]);
const toggleVoiceInput = useCallback(() => {
if (voiceListening) {
stopVoiceInput();
return;
}
if (composerLocked) {
return;
}
if (!speechRecognitionConstructor) {
toast.error(t.inputBox.voiceInputUnsupported);
return;
}
abortInputPolishRequest();
setInputPolishUndo(null);
promptHistoryIndexRef.current = null;
promptHistoryDraftRef.current = "";
voiceStopRequestedRef.current = false;
voiceBaseTextRef.current = textInput.value ?? "";
voiceLatestTextRef.current = voiceBaseTextRef.current;
startVoiceRecognition({ focusAfterStart: true });
}, [
abortInputPolishRequest,
composerLocked,
speechRecognitionConstructor,
startVoiceRecognition,
stopVoiceInput,
t.inputBox.voiceInputUnsupported,
textInput,
voiceListening,
]);
useEffect(() => {
if (composerLocked && voiceListening) {
stopVoiceInput();
}
}, [composerLocked, stopVoiceInput, voiceListening]);
useEffect(() => {
return () => abortVoiceInput();
}, [abortVoiceInput, threadId]);
useEffect(() => {
setSkillSuggestionIndex(0);
@@ -1282,19 +1551,25 @@ export function InputBox({
);
const handlePromptTextareaChange = useCallback(() => {
if (voiceListening) {
abortVoiceInput();
}
abortInputPolishRequest();
setInputPolishUndo(null);
promptHistoryIndexRef.current = null;
promptHistoryDraftRef.current = "";
}, [abortInputPolishRequest]);
}, [abortInputPolishRequest, abortVoiceInput, voiceListening]);
const updateInlineSkillTextInput = useCallback(
(element: HTMLElement) => {
if (voiceListening) {
abortVoiceInput();
}
promptHistoryIndexRef.current = null;
promptHistoryDraftRef.current = "";
textInput.setInput(element.textContent ?? "");
},
[textInput],
[abortVoiceInput, textInput, voiceListening],
);
useEffect(() => {
@@ -1731,6 +2006,12 @@ export function InputBox({
disabled={composerLocked}
uploadLimits={uploadLimits}
/>
<VoiceInputButton
disabled={composerLocked}
listening={voiceListening}
supported={voiceInputSupported}
onToggle={toggleVoiceInput}
/>
<Tooltip
content={
polishingInput
@@ -2153,6 +2434,50 @@ export function InputBox({
);
}
function VoiceInputButton({
disabled,
listening,
supported,
onToggle,
}: {
disabled?: boolean;
listening: boolean;
supported: boolean;
onToggle: () => void;
}) {
const { t } = useI18n();
const tooltipContent = !supported
? t.inputBox.voiceInputUnsupported
: listening
? t.inputBox.voiceInputListening
: t.inputBox.voiceInputStart;
const label = listening
? t.inputBox.voiceInputStopLabel
: t.inputBox.voiceInputStartLabel;
return (
<Tooltip content={<span className="block max-w-72">{tooltipContent}</span>}>
<PromptInputButton
aria-label={label}
aria-pressed={listening}
className={cn(
"px-2!",
listening && "text-primary bg-primary/10 hover:bg-primary/15",
)}
data-testid="voice-input-button"
disabled={(disabled ?? false) || !supported}
onClick={onToggle}
>
{listening ? (
<SquareIcon className="size-3 fill-current" />
) : (
<MicIcon className="size-3" />
)}
</PromptInputButton>
</Tooltip>
);
}
function SuggestionList({
onSelectPlaceholder,
}: {
+18
View File
@@ -122,6 +122,24 @@ export const enUS: Translations = {
inputPolishFailed: "Failed to polish input.",
inputPolishUndo: "Undo polish",
inputPolishCancel: "Cancel polishing",
voiceInputStartLabel: "Dictate with voice",
voiceInputStopLabel: "Stop voice input",
voiceInputStart:
"Dictate with voice. DeerFlow receives only transcribed text; audio is handled by your browser or system speech service.",
voiceInputStop: "Stop voice input",
voiceInputListening: "Listening... Click to stop voice input.",
voiceInputUnsupported:
"Voice input is not supported in this browser. Try Chrome or Edge.",
voiceInputPermissionDenied:
"Microphone access was denied. Allow microphone access and try again.",
voiceInputMicrophoneUnavailable:
"No microphone was detected. Check your device input and try again.",
voiceInputUnsupportedLanguage:
"Voice input does not support the current language in this browser.",
voiceInputNetworkError:
"Voice input could not reach the browser speech service.",
voiceInputNoSpeech: "No speech was detected. Please try again.",
voiceInputFailed: "Voice input failed. Please try again.",
mode: "Mode",
flashMode: "Flash",
flashModeDescription: "Fast and efficient, but may not be accurate",
+12
View File
@@ -104,6 +104,18 @@ export interface Translations {
inputPolishFailed: string;
inputPolishUndo: string;
inputPolishCancel: string;
voiceInputStartLabel: string;
voiceInputStopLabel: string;
voiceInputStart: string;
voiceInputStop: string;
voiceInputListening: string;
voiceInputUnsupported: string;
voiceInputPermissionDenied: string;
voiceInputMicrophoneUnavailable: string;
voiceInputUnsupportedLanguage: string;
voiceInputNetworkError: string;
voiceInputNoSpeech: string;
voiceInputFailed: string;
mode: string;
flashMode: string;
flashModeDescription: string;
+14
View File
@@ -121,6 +121,20 @@ export const zhCN: Translations = {
inputPolishFailed: "优化输入失败。",
inputPolishUndo: "撤销优化",
inputPolishCancel: "取消优化",
voiceInputStartLabel: "语音输入",
voiceInputStopLabel: "停止语音输入",
voiceInputStart:
"语音输入。DeerFlow 只接收转写文本,音频由浏览器或系统语音服务处理。",
voiceInputStop: "停止语音输入",
voiceInputListening: "正在聆听... 点击停止语音输入。",
voiceInputUnsupported:
"当前浏览器不支持语音输入。建议使用 Chrome 或 Edge。",
voiceInputPermissionDenied: "麦克风权限被拒绝。请允许麦克风访问后重试。",
voiceInputMicrophoneUnavailable: "未检测到麦克风。请检查设备输入后重试。",
voiceInputUnsupportedLanguage: "当前浏览器不支持该语言的语音输入。",
voiceInputNetworkError: "无法连接浏览器语音识别服务。",
voiceInputNoSpeech: "没有检测到语音,请重试。",
voiceInputFailed: "语音输入失败,请重试。",
mode: "模式",
flashMode: "闪速",
flashModeDescription: "快速且高效的完成任务,但可能不够精准",
@@ -0,0 +1,181 @@
export type SpeechRecognitionErrorCode =
| "aborted"
| "audio-capture"
| "bad-grammar"
| "language-not-supported"
| "network"
| "no-speech"
| "not-allowed"
| "phrases-not-supported"
| "service-not-allowed";
export type SpeechRecognitionErrorKind =
| "cancelled"
| "microphone_unavailable"
| "permission_denied"
| "unsupported_language"
| "network"
| "no_speech"
| "unknown";
export type SpeechRecognitionConstructor = new () => BrowserSpeechRecognition;
export type SpeechRecognitionEventLike = {
results: SpeechRecognitionResultListLike;
};
export type SpeechRecognitionErrorEventLike = {
error?: SpeechRecognitionErrorCode | string;
};
export type BrowserSpeechRecognition = {
continuous: boolean;
interimResults: boolean;
lang: string;
maxAlternatives: number;
onend: (() => void) | null;
onerror: ((event: SpeechRecognitionErrorEventLike) => void) | null;
onresult: ((event: SpeechRecognitionEventLike) => void) | null;
start: () => void;
stop: () => void;
abort: () => void;
};
type SpeechRecognitionWindow = Window &
typeof globalThis & {
SpeechRecognition?: SpeechRecognitionConstructor;
webkitSpeechRecognition?: SpeechRecognitionConstructor;
};
export type SpeechRecognitionAlternativeLike = {
transcript?: string;
};
export type SpeechRecognitionResultLike = {
0?: SpeechRecognitionAlternativeLike;
isFinal: boolean;
length: number;
};
export type SpeechRecognitionResultListLike = {
[index: number]: SpeechRecognitionResultLike | undefined;
length: number;
};
const DEFAULT_SPEECH_RECOGNITION_LANGUAGE = "en-US";
const SPEECH_RECOGNITION_LANGUAGE_ALLOWLIST = new Set([
"de",
"en",
"es",
"fr",
"it",
"ja",
"ko",
"pt",
"zh",
]);
export function getSpeechRecognitionConstructor(
value: unknown = globalThis,
): SpeechRecognitionConstructor | null {
const maybeWindow = value as Partial<SpeechRecognitionWindow>;
return (
maybeWindow.SpeechRecognition ?? maybeWindow.webkitSpeechRecognition ?? null
);
}
export function getSpeechRecognitionLanguage(locale: string): string {
const normalized = normalizeBCP47Locale(locale);
const language = normalized.split("-")[0]?.toLowerCase();
if (language === "zh") {
return "zh-CN";
}
if (language && SPEECH_RECOGNITION_LANGUAGE_ALLOWLIST.has(language)) {
return normalized;
}
return DEFAULT_SPEECH_RECOGNITION_LANGUAGE;
}
export function shouldRestartSpeechRecognition(
lastError: SpeechRecognitionErrorKind | null,
): boolean {
return lastError === null || lastError === "no_speech";
}
export function readSpeechRecognitionTranscript(
results: SpeechRecognitionResultListLike,
): { finalText: string; interimText: string; text: string } {
let finalText = "";
let interimText = "";
for (const result of Array.from(
{ length: results.length },
(_, index) => results[index],
)) {
const transcript = result?.[0]?.transcript ?? "";
if (result?.isFinal) {
finalText += transcript;
} else {
interimText += transcript;
}
}
return {
finalText: normalizeSpeechTranscript(finalText),
interimText: normalizeSpeechTranscript(interimText),
text: normalizeSpeechTranscript(`${finalText}${interimText}`),
};
}
export function appendSpeechTranscript(baseText: string, transcript: string) {
const cleanTranscript = normalizeSpeechTranscript(transcript);
if (!cleanTranscript) {
return baseText;
}
const cleanBase = baseText.trimEnd();
if (!cleanBase) {
return cleanTranscript;
}
return `${cleanBase} ${cleanTranscript}`;
}
export function normalizeSpeechTranscript(value: string) {
return value.replace(/\s+/g, " ").trim();
}
export function mapSpeechRecognitionError(
error: SpeechRecognitionErrorCode | string | undefined,
): SpeechRecognitionErrorKind {
switch (error) {
case "aborted":
return "cancelled";
case "audio-capture":
return "microphone_unavailable";
case "not-allowed":
case "service-not-allowed":
return "permission_denied";
case "language-not-supported":
return "unsupported_language";
case "network":
return "network";
case "no-speech":
return "no_speech";
default:
return "unknown";
}
}
function normalizeBCP47Locale(locale: string): string {
const trimmed = locale.trim();
if (!trimmed) {
return DEFAULT_SPEECH_RECOGNITION_LANGUAGE;
}
try {
return Intl.getCanonicalLocales(trimmed)[0] ?? trimmed;
} catch {
return DEFAULT_SPEECH_RECOGNITION_LANGUAGE;
}
}
@@ -0,0 +1,120 @@
import { describe, expect, it } from "@rstest/core";
import {
appendSpeechTranscript,
getSpeechRecognitionConstructor,
getSpeechRecognitionLanguage,
mapSpeechRecognitionError,
readSpeechRecognitionTranscript,
shouldRestartSpeechRecognition,
type SpeechRecognitionConstructor,
} from "@/core/voice-input/speech-recognition";
describe("speech recognition helpers", () => {
it("prefers the standard constructor and falls back to webkit", () => {
const standard = makeSpeechRecognitionConstructor();
const webkit = makeSpeechRecognitionConstructor();
expect(
getSpeechRecognitionConstructor({
SpeechRecognition: standard as unknown as SpeechRecognitionConstructor,
webkitSpeechRecognition:
webkit as unknown as SpeechRecognitionConstructor,
}),
).toBe(standard);
expect(
getSpeechRecognitionConstructor({
webkitSpeechRecognition:
webkit as unknown as SpeechRecognitionConstructor,
}),
).toBe(webkit);
expect(getSpeechRecognitionConstructor({})).toBeNull();
});
it("maps DeerFlow locales to browser recognition locales", () => {
expect(getSpeechRecognitionLanguage("zh-CN")).toBe("zh-CN");
expect(getSpeechRecognitionLanguage("zh-Hans")).toBe("zh-CN");
expect(getSpeechRecognitionLanguage("en-US")).toBe("en-US");
expect(getSpeechRecognitionLanguage("en-GB")).toBe("en-GB");
expect(getSpeechRecognitionLanguage("fr-FR")).toBe("fr-FR");
expect(getSpeechRecognitionLanguage("ja-JP")).toBe("ja-JP");
expect(getSpeechRecognitionLanguage("xx-YY")).toBe("en-US");
expect(getSpeechRecognitionLanguage("not a locale")).toBe("en-US");
});
it("combines final and interim transcripts with whitespace cleanup", () => {
expect(
readSpeechRecognitionTranscript({
length: 3,
0: { isFinal: true, length: 1, 0: { transcript: " hello " } },
1: { isFinal: true, length: 1, 0: { transcript: " world" } },
2: { isFinal: false, length: 1, 0: { transcript: " again " } },
}),
).toEqual({
finalText: "hello world",
interimText: "again",
text: "hello world again",
});
});
it("appends transcript to an existing draft without duplicating whitespace", () => {
expect(appendSpeechTranscript("", " hello world ")).toBe("hello world");
expect(appendSpeechTranscript("Draft", "voice text")).toBe(
"Draft voice text",
);
expect(appendSpeechTranscript("Draft\n", "voice text")).toBe(
"Draft voice text",
);
expect(appendSpeechTranscript("Draft", " ")).toBe("Draft");
});
it("normalizes browser speech recognition errors", () => {
expect(mapSpeechRecognitionError("not-allowed")).toBe("permission_denied");
expect(mapSpeechRecognitionError("service-not-allowed")).toBe(
"permission_denied",
);
expect(mapSpeechRecognitionError("audio-capture")).toBe(
"microphone_unavailable",
);
expect(mapSpeechRecognitionError("language-not-supported")).toBe(
"unsupported_language",
);
expect(mapSpeechRecognitionError("network")).toBe("network");
expect(mapSpeechRecognitionError("no-speech")).toBe("no_speech");
expect(mapSpeechRecognitionError("aborted")).toBe("cancelled");
expect(mapSpeechRecognitionError("bad-grammar")).toBe("unknown");
});
it("restarts only after browser auto-end conditions", () => {
expect(shouldRestartSpeechRecognition(null)).toBe(true);
expect(shouldRestartSpeechRecognition("no_speech")).toBe(true);
expect(shouldRestartSpeechRecognition("cancelled")).toBe(false);
expect(shouldRestartSpeechRecognition("permission_denied")).toBe(false);
expect(shouldRestartSpeechRecognition("network")).toBe(false);
expect(shouldRestartSpeechRecognition("unknown")).toBe(false);
});
});
function makeSpeechRecognitionConstructor(): SpeechRecognitionConstructor {
return class {
continuous = false;
interimResults = false;
lang = "";
maxAlternatives = 1;
onend = null;
onerror = null;
onresult = null;
start() {
return undefined;
}
stop() {
return undefined;
}
abort() {
return undefined;
}
};
}