mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-07-20 09:45:47 +00:00
feat(frontend): add voice dictation input (#4036)
* feat(frontend): add voice dictation input * fix(frontend): address voice input review feedback * ci: rerun frontend checks
This commit is contained in:
@@ -635,6 +635,8 @@ Gateway-generated follow-up suggestions now normalize both plain-string model ou
|
||||
|
||||
The Web UI composer can polish draft input before sending. The rewrite runs as a short Gateway LLM request using the `input_polish` model configuration, keeps slash skill prefixes such as `/data-analysis`, and only replaces the local draft after the user clicks the polish button; it does not create a thread run or persist a message.
|
||||
|
||||
The Web UI composer also supports browser-based voice dictation when the browser exposes the Web Speech API. The microphone button transcribes speech into the local draft only; DeerFlow receives only the transcribed text, while audio handling is delegated to the browser or operating system speech-recognition service according to that environment's policy. Users can review or edit the text before sending.
|
||||
|
||||
Interrupted first-turn runs still persist a fallback conversation title, so stopping a streaming response does not leave the thread as "Untitled" after refresh.
|
||||
|
||||
In the Web UI, completed assistant turns can be branched into a new main conversation. The new thread starts from that turn's checkpoint. Because workspace files are not checkpointed, the branch only receives a best-effort copy of the current workspace when you branch from the latest turn; branching from an older turn keeps just the restored message history so the branch never inherits files that were created in a later part of the conversation.
|
||||
|
||||
@@ -592,6 +592,8 @@ DEERFLOW_LANGGRAPH_URL=http://localhost:2026/api/langgraph # LangGraph API
|
||||
|
||||
完整 API 说明见 [`skills/public/claude-to-deerflow/SKILL.md`](skills/public/claude-to-deerflow/SKILL.md)。
|
||||
|
||||
Web UI 输入框支持浏览器侧语音听写。浏览器提供 Web Speech API 时,麦克风按钮会把语音转写为本地草稿;DeerFlow 只接收转写后的文本,音频处理交由浏览器或操作系统语音识别服务按其环境策略完成。用户可以在发送前继续检查和编辑文本。
|
||||
|
||||
### Session Goals
|
||||
|
||||
用 `/goal <完成条件>` 为当前 thread 绑定一个激活态的完成条件。这个 goal 是 thread 维度的状态,而不是技能激活,所以它会跨轮次持续生效,直到 DeerFlow 判定它已被满足、或者你手动清除它。
|
||||
|
||||
+2
-2
@@ -53,7 +53,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat
|
||||
- `workspace/` — Chat page components (messages, artifacts, settings)
|
||||
- `landing/` — Landing page sections
|
||||
- `docs/` — Docs / MDX rendering components
|
||||
- **`core/`** — Business logic, the heart of the app. Domains include `threads/` (creation, streaming, state), `api/` (LangGraph client singleton), `agents/` (custom agents), `auth/` (authentication), `artifacts/`, `channels/` (IM connections), `i18n/` (en-US, zh-CN), `settings/`, `memory/`, `skills/`, `messages/`, `mcp/`, `models/`, `input-polish/` (pre-send draft rewrite API), `suggestions/`, `tasks/`, `todos/`, `tools/`, `workspace-changes/` (run-scoped changed-file summaries and diff fetching), `config/`, `notification/`, `blog/`, plus rendering helpers (`rehype/`, `streamdown/`) and `utils/`.
|
||||
- **`core/`** — Business logic, the heart of the app. Domains include `threads/` (creation, streaming, state), `api/` (LangGraph client singleton), `agents/` (custom agents), `auth/` (authentication), `artifacts/`, `channels/` (IM connections), `i18n/` (en-US, zh-CN), `settings/`, `memory/`, `skills/`, `messages/`, `mcp/`, `models/`, `input-polish/` (pre-send draft rewrite API), `voice-input/` (browser speech-recognition helpers), `suggestions/`, `tasks/`, `todos/`, `tools/`, `workspace-changes/` (run-scoped changed-file summaries and diff fetching), `config/`, `notification/`, `blog/`, plus rendering helpers (`rehype/`, `streamdown/`) and `utils/`.
|
||||
- **`hooks/`** — Shared React hooks
|
||||
- **`lib/`** — Utilities (`cn()` from clsx + tailwind-merge)
|
||||
- **`content/`** — MDX content (blog posts, docs) rendered by the app
|
||||
@@ -63,7 +63,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat
|
||||
|
||||
### Data Flow
|
||||
|
||||
1. Optional composer helpers such as `core/input-polish` can rewrite the local draft before submission; confirmed user input then flows to thread hooks (`core/threads/hooks.ts`) → LangGraph SDK streaming
|
||||
1. Optional composer helpers such as `core/input-polish` can rewrite the local draft before submission, and `core/voice-input` can transcribe browser microphone input into that same local draft; confirmed user input then flows to thread hooks (`core/threads/hooks.ts`) → LangGraph SDK streaming
|
||||
2. Stream events update thread state (messages, artifacts, todos, goal)
|
||||
3. Stop actions call the LangGraph SDK stream stop path; `core/threads/hooks.ts` invalidates current-thread, token-usage, and sidebar/search caches immediately and schedules one follow-up refetch because SDK stop may finish via abort + fire-and-forget cancel before backend title finalization commits
|
||||
4. TanStack Query manages server state; localStorage stores user settings
|
||||
|
||||
@@ -8,10 +8,12 @@ import {
|
||||
GraduationCapIcon,
|
||||
LightbulbIcon,
|
||||
Loader2Icon,
|
||||
MicIcon,
|
||||
PaperclipIcon,
|
||||
PlusIcon,
|
||||
RocketIcon,
|
||||
SparklesIcon,
|
||||
SquareIcon,
|
||||
TargetIcon,
|
||||
Undo2Icon,
|
||||
XIcon,
|
||||
@@ -89,6 +91,16 @@ import {
|
||||
type UploadLimits,
|
||||
type UploadLimitViolation,
|
||||
} from "@/core/uploads";
|
||||
import {
|
||||
appendSpeechTranscript,
|
||||
getSpeechRecognitionConstructor,
|
||||
getSpeechRecognitionLanguage,
|
||||
mapSpeechRecognitionError,
|
||||
readSpeechRecognitionTranscript,
|
||||
shouldRestartSpeechRecognition,
|
||||
type BrowserSpeechRecognition,
|
||||
type SpeechRecognitionErrorKind,
|
||||
} from "@/core/voice-input/speech-recognition";
|
||||
import { isIMEComposing } from "@/lib/ime";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
@@ -200,6 +212,10 @@ export type InputBoxSubmitOptions = {
|
||||
onSent?: () => void;
|
||||
};
|
||||
|
||||
type VoiceRecognitionStartOptions = {
|
||||
focusAfterStart?: boolean;
|
||||
};
|
||||
|
||||
function buildHiddenConversationQuoteMessage({
|
||||
contexts,
|
||||
}: {
|
||||
@@ -326,6 +342,17 @@ export function InputBox({
|
||||
controller: null,
|
||||
sequence: 0,
|
||||
});
|
||||
const voiceRecognitionRef = useRef<BrowserSpeechRecognition | null>(null);
|
||||
const voiceBaseTextRef = useRef("");
|
||||
const voiceLatestTextRef = useRef("");
|
||||
const voiceLastErrorKindRef = useRef<SpeechRecognitionErrorKind | null>(null);
|
||||
const voiceStopRequestedRef = useRef(false);
|
||||
const voiceRestartTimerRef = useRef<ReturnType<typeof setTimeout> | null>(
|
||||
null,
|
||||
);
|
||||
const startVoiceRecognitionRef = useRef<
|
||||
((options?: VoiceRecognitionStartOptions) => boolean) | null
|
||||
>(null);
|
||||
const promptHistoryIndexRef = useRef<number | null>(null);
|
||||
const promptHistoryDraftRef = useRef("");
|
||||
|
||||
@@ -336,6 +363,7 @@ export function InputBox({
|
||||
const [followupsHidden, setFollowupsHidden] = useState(false);
|
||||
const [followupsLoading, setFollowupsLoading] = useState(false);
|
||||
const [polishingInput, setPolishingInput] = useState(false);
|
||||
const [voiceListening, setVoiceListening] = useState(false);
|
||||
const [inputPolishUndo, setInputPolishUndo] = useState<{
|
||||
originalText: string;
|
||||
rewrittenText: string;
|
||||
@@ -350,6 +378,58 @@ export function InputBox({
|
||||
const wasStreamingRef = useRef(false);
|
||||
const messagesRef = useRef(thread.messages);
|
||||
|
||||
const clearVoiceRestartTimer = useCallback(() => {
|
||||
if (voiceRestartTimerRef.current === null) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(voiceRestartTimerRef.current);
|
||||
voiceRestartTimerRef.current = null;
|
||||
}, []);
|
||||
|
||||
const cleanupVoiceRecognition = useCallback(
|
||||
(
|
||||
recognition: BrowserSpeechRecognition | null,
|
||||
options: { keepListening?: boolean } = {},
|
||||
) => {
|
||||
clearVoiceRestartTimer();
|
||||
if (!recognition) {
|
||||
if (!options.keepListening) {
|
||||
voiceLastErrorKindRef.current = null;
|
||||
voiceStopRequestedRef.current = false;
|
||||
setVoiceListening(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
recognition.onend = null;
|
||||
recognition.onerror = null;
|
||||
recognition.onresult = null;
|
||||
if (voiceRecognitionRef.current === recognition) {
|
||||
voiceRecognitionRef.current = null;
|
||||
}
|
||||
if (!options.keepListening) {
|
||||
voiceLastErrorKindRef.current = null;
|
||||
voiceStopRequestedRef.current = false;
|
||||
setVoiceListening(false);
|
||||
}
|
||||
},
|
||||
[clearVoiceRestartTimer],
|
||||
);
|
||||
|
||||
const abortVoiceInput = useCallback(() => {
|
||||
const recognition = voiceRecognitionRef.current;
|
||||
voiceStopRequestedRef.current = true;
|
||||
if (!recognition) {
|
||||
cleanupVoiceRecognition(null);
|
||||
return;
|
||||
}
|
||||
cleanupVoiceRecognition(recognition);
|
||||
try {
|
||||
recognition.abort();
|
||||
} catch {
|
||||
// Browser implementations can throw when the recognizer already ended.
|
||||
}
|
||||
}, [cleanupVoiceRecognition]);
|
||||
|
||||
const [confirmOpen, setConfirmOpen] = useState(false);
|
||||
const [pendingSuggestion, setPendingSuggestion] = useState<string | null>(
|
||||
null,
|
||||
@@ -852,6 +932,7 @@ export function InputBox({
|
||||
toast.info(t.inputBox.pleaseWaitStreaming);
|
||||
return Promise.reject(new Error("streaming"));
|
||||
}
|
||||
abortVoiceInput();
|
||||
const messageWithSlashSkill = selectedSlashSkill
|
||||
? {
|
||||
...message,
|
||||
@@ -896,6 +977,7 @@ export function InputBox({
|
||||
}
|
||||
},
|
||||
[
|
||||
abortVoiceInput,
|
||||
handleCompactCommand,
|
||||
handleGoalCommand,
|
||||
onStop,
|
||||
@@ -1003,6 +1085,193 @@ export function InputBox({
|
||||
(status === "streaming" ||
|
||||
slashSkillQuery !== null ||
|
||||
!canPolishInput(textInput.value ?? "")));
|
||||
const speechRecognitionConstructor = useMemo(
|
||||
() =>
|
||||
typeof window === "undefined"
|
||||
? null
|
||||
: getSpeechRecognitionConstructor(window),
|
||||
[],
|
||||
);
|
||||
const voiceInputSupported = speechRecognitionConstructor !== null;
|
||||
|
||||
const getVoiceInputErrorMessage = useCallback(
|
||||
(kind: SpeechRecognitionErrorKind) => {
|
||||
switch (kind) {
|
||||
case "permission_denied":
|
||||
return t.inputBox.voiceInputPermissionDenied;
|
||||
case "microphone_unavailable":
|
||||
return t.inputBox.voiceInputMicrophoneUnavailable;
|
||||
case "unsupported_language":
|
||||
return t.inputBox.voiceInputUnsupportedLanguage;
|
||||
case "network":
|
||||
return t.inputBox.voiceInputNetworkError;
|
||||
case "no_speech":
|
||||
return t.inputBox.voiceInputNoSpeech;
|
||||
case "cancelled":
|
||||
return null;
|
||||
default:
|
||||
return t.inputBox.voiceInputFailed;
|
||||
}
|
||||
},
|
||||
[t],
|
||||
);
|
||||
|
||||
const startVoiceRecognition = useCallback(
|
||||
(options: VoiceRecognitionStartOptions = {}) => {
|
||||
if (composerLocked || !speechRecognitionConstructor) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const recognition = new speechRecognitionConstructor();
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = true;
|
||||
recognition.lang = getSpeechRecognitionLanguage(locale);
|
||||
recognition.maxAlternatives = 1;
|
||||
voiceLastErrorKindRef.current = null;
|
||||
voiceLatestTextRef.current = voiceBaseTextRef.current;
|
||||
voiceRecognitionRef.current = recognition;
|
||||
|
||||
recognition.onresult = (event) => {
|
||||
if (voiceRecognitionRef.current !== recognition) {
|
||||
return;
|
||||
}
|
||||
const transcript = readSpeechRecognitionTranscript(event.results).text;
|
||||
const nextValue = appendSpeechTranscript(
|
||||
voiceBaseTextRef.current,
|
||||
transcript,
|
||||
);
|
||||
voiceLatestTextRef.current = nextValue;
|
||||
textInput.setInput(nextValue);
|
||||
};
|
||||
recognition.onerror = (event) => {
|
||||
const errorKind = mapSpeechRecognitionError(event.error);
|
||||
voiceLastErrorKindRef.current = errorKind;
|
||||
if (
|
||||
!voiceStopRequestedRef.current &&
|
||||
shouldRestartSpeechRecognition(errorKind)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const message = getVoiceInputErrorMessage(errorKind);
|
||||
if (message) {
|
||||
toast.error(message);
|
||||
}
|
||||
};
|
||||
recognition.onend = () => {
|
||||
const shouldRestart =
|
||||
voiceRecognitionRef.current === recognition &&
|
||||
!voiceStopRequestedRef.current &&
|
||||
shouldRestartSpeechRecognition(voiceLastErrorKindRef.current);
|
||||
if (shouldRestart) {
|
||||
voiceBaseTextRef.current = voiceLatestTextRef.current;
|
||||
cleanupVoiceRecognition(recognition, { keepListening: true });
|
||||
voiceRestartTimerRef.current = setTimeout(() => {
|
||||
voiceRestartTimerRef.current = null;
|
||||
if (voiceStopRequestedRef.current) {
|
||||
cleanupVoiceRecognition(null);
|
||||
return;
|
||||
}
|
||||
const restarted = startVoiceRecognitionRef.current?.() ?? false;
|
||||
if (!restarted) {
|
||||
cleanupVoiceRecognition(null);
|
||||
}
|
||||
}, 150);
|
||||
return;
|
||||
}
|
||||
cleanupVoiceRecognition(recognition);
|
||||
};
|
||||
|
||||
setVoiceListening(true);
|
||||
try {
|
||||
recognition.start();
|
||||
if (options.focusAfterStart) {
|
||||
requestAnimationFrame(() => {
|
||||
if (selectedSlashSkill) {
|
||||
focusContentEditableEnd(inlineSkillTextRef.current);
|
||||
} else {
|
||||
textareaRef.current?.focus();
|
||||
}
|
||||
});
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
cleanupVoiceRecognition(recognition);
|
||||
toast.error(t.inputBox.voiceInputFailed);
|
||||
return false;
|
||||
}
|
||||
},
|
||||
[
|
||||
cleanupVoiceRecognition,
|
||||
composerLocked,
|
||||
getVoiceInputErrorMessage,
|
||||
locale,
|
||||
selectedSlashSkill,
|
||||
speechRecognitionConstructor,
|
||||
t.inputBox.voiceInputFailed,
|
||||
textInput,
|
||||
],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
startVoiceRecognitionRef.current = startVoiceRecognition;
|
||||
}, [startVoiceRecognition]);
|
||||
|
||||
const stopVoiceInput = useCallback(() => {
|
||||
const recognition = voiceRecognitionRef.current;
|
||||
voiceStopRequestedRef.current = true;
|
||||
if (!recognition) {
|
||||
cleanupVoiceRecognition(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
recognition.stop();
|
||||
} catch {
|
||||
cleanupVoiceRecognition(recognition);
|
||||
}
|
||||
}, [cleanupVoiceRecognition]);
|
||||
|
||||
const toggleVoiceInput = useCallback(() => {
|
||||
if (voiceListening) {
|
||||
stopVoiceInput();
|
||||
return;
|
||||
}
|
||||
if (composerLocked) {
|
||||
return;
|
||||
}
|
||||
if (!speechRecognitionConstructor) {
|
||||
toast.error(t.inputBox.voiceInputUnsupported);
|
||||
return;
|
||||
}
|
||||
|
||||
abortInputPolishRequest();
|
||||
setInputPolishUndo(null);
|
||||
promptHistoryIndexRef.current = null;
|
||||
promptHistoryDraftRef.current = "";
|
||||
voiceStopRequestedRef.current = false;
|
||||
voiceBaseTextRef.current = textInput.value ?? "";
|
||||
voiceLatestTextRef.current = voiceBaseTextRef.current;
|
||||
startVoiceRecognition({ focusAfterStart: true });
|
||||
}, [
|
||||
abortInputPolishRequest,
|
||||
composerLocked,
|
||||
speechRecognitionConstructor,
|
||||
startVoiceRecognition,
|
||||
stopVoiceInput,
|
||||
t.inputBox.voiceInputUnsupported,
|
||||
textInput,
|
||||
voiceListening,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (composerLocked && voiceListening) {
|
||||
stopVoiceInput();
|
||||
}
|
||||
}, [composerLocked, stopVoiceInput, voiceListening]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => abortVoiceInput();
|
||||
}, [abortVoiceInput, threadId]);
|
||||
|
||||
useEffect(() => {
|
||||
setSkillSuggestionIndex(0);
|
||||
@@ -1282,19 +1551,25 @@ export function InputBox({
|
||||
);
|
||||
|
||||
const handlePromptTextareaChange = useCallback(() => {
|
||||
if (voiceListening) {
|
||||
abortVoiceInput();
|
||||
}
|
||||
abortInputPolishRequest();
|
||||
setInputPolishUndo(null);
|
||||
promptHistoryIndexRef.current = null;
|
||||
promptHistoryDraftRef.current = "";
|
||||
}, [abortInputPolishRequest]);
|
||||
}, [abortInputPolishRequest, abortVoiceInput, voiceListening]);
|
||||
|
||||
const updateInlineSkillTextInput = useCallback(
|
||||
(element: HTMLElement) => {
|
||||
if (voiceListening) {
|
||||
abortVoiceInput();
|
||||
}
|
||||
promptHistoryIndexRef.current = null;
|
||||
promptHistoryDraftRef.current = "";
|
||||
textInput.setInput(element.textContent ?? "");
|
||||
},
|
||||
[textInput],
|
||||
[abortVoiceInput, textInput, voiceListening],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1731,6 +2006,12 @@ export function InputBox({
|
||||
disabled={composerLocked}
|
||||
uploadLimits={uploadLimits}
|
||||
/>
|
||||
<VoiceInputButton
|
||||
disabled={composerLocked}
|
||||
listening={voiceListening}
|
||||
supported={voiceInputSupported}
|
||||
onToggle={toggleVoiceInput}
|
||||
/>
|
||||
<Tooltip
|
||||
content={
|
||||
polishingInput
|
||||
@@ -2153,6 +2434,50 @@ export function InputBox({
|
||||
);
|
||||
}
|
||||
|
||||
function VoiceInputButton({
|
||||
disabled,
|
||||
listening,
|
||||
supported,
|
||||
onToggle,
|
||||
}: {
|
||||
disabled?: boolean;
|
||||
listening: boolean;
|
||||
supported: boolean;
|
||||
onToggle: () => void;
|
||||
}) {
|
||||
const { t } = useI18n();
|
||||
const tooltipContent = !supported
|
||||
? t.inputBox.voiceInputUnsupported
|
||||
: listening
|
||||
? t.inputBox.voiceInputListening
|
||||
: t.inputBox.voiceInputStart;
|
||||
const label = listening
|
||||
? t.inputBox.voiceInputStopLabel
|
||||
: t.inputBox.voiceInputStartLabel;
|
||||
|
||||
return (
|
||||
<Tooltip content={<span className="block max-w-72">{tooltipContent}</span>}>
|
||||
<PromptInputButton
|
||||
aria-label={label}
|
||||
aria-pressed={listening}
|
||||
className={cn(
|
||||
"px-2!",
|
||||
listening && "text-primary bg-primary/10 hover:bg-primary/15",
|
||||
)}
|
||||
data-testid="voice-input-button"
|
||||
disabled={(disabled ?? false) || !supported}
|
||||
onClick={onToggle}
|
||||
>
|
||||
{listening ? (
|
||||
<SquareIcon className="size-3 fill-current" />
|
||||
) : (
|
||||
<MicIcon className="size-3" />
|
||||
)}
|
||||
</PromptInputButton>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
function SuggestionList({
|
||||
onSelectPlaceholder,
|
||||
}: {
|
||||
|
||||
@@ -122,6 +122,24 @@ export const enUS: Translations = {
|
||||
inputPolishFailed: "Failed to polish input.",
|
||||
inputPolishUndo: "Undo polish",
|
||||
inputPolishCancel: "Cancel polishing",
|
||||
voiceInputStartLabel: "Dictate with voice",
|
||||
voiceInputStopLabel: "Stop voice input",
|
||||
voiceInputStart:
|
||||
"Dictate with voice. DeerFlow receives only transcribed text; audio is handled by your browser or system speech service.",
|
||||
voiceInputStop: "Stop voice input",
|
||||
voiceInputListening: "Listening... Click to stop voice input.",
|
||||
voiceInputUnsupported:
|
||||
"Voice input is not supported in this browser. Try Chrome or Edge.",
|
||||
voiceInputPermissionDenied:
|
||||
"Microphone access was denied. Allow microphone access and try again.",
|
||||
voiceInputMicrophoneUnavailable:
|
||||
"No microphone was detected. Check your device input and try again.",
|
||||
voiceInputUnsupportedLanguage:
|
||||
"Voice input does not support the current language in this browser.",
|
||||
voiceInputNetworkError:
|
||||
"Voice input could not reach the browser speech service.",
|
||||
voiceInputNoSpeech: "No speech was detected. Please try again.",
|
||||
voiceInputFailed: "Voice input failed. Please try again.",
|
||||
mode: "Mode",
|
||||
flashMode: "Flash",
|
||||
flashModeDescription: "Fast and efficient, but may not be accurate",
|
||||
|
||||
@@ -104,6 +104,18 @@ export interface Translations {
|
||||
inputPolishFailed: string;
|
||||
inputPolishUndo: string;
|
||||
inputPolishCancel: string;
|
||||
voiceInputStartLabel: string;
|
||||
voiceInputStopLabel: string;
|
||||
voiceInputStart: string;
|
||||
voiceInputStop: string;
|
||||
voiceInputListening: string;
|
||||
voiceInputUnsupported: string;
|
||||
voiceInputPermissionDenied: string;
|
||||
voiceInputMicrophoneUnavailable: string;
|
||||
voiceInputUnsupportedLanguage: string;
|
||||
voiceInputNetworkError: string;
|
||||
voiceInputNoSpeech: string;
|
||||
voiceInputFailed: string;
|
||||
mode: string;
|
||||
flashMode: string;
|
||||
flashModeDescription: string;
|
||||
|
||||
@@ -121,6 +121,20 @@ export const zhCN: Translations = {
|
||||
inputPolishFailed: "优化输入失败。",
|
||||
inputPolishUndo: "撤销优化",
|
||||
inputPolishCancel: "取消优化",
|
||||
voiceInputStartLabel: "语音输入",
|
||||
voiceInputStopLabel: "停止语音输入",
|
||||
voiceInputStart:
|
||||
"语音输入。DeerFlow 只接收转写文本,音频由浏览器或系统语音服务处理。",
|
||||
voiceInputStop: "停止语音输入",
|
||||
voiceInputListening: "正在聆听... 点击停止语音输入。",
|
||||
voiceInputUnsupported:
|
||||
"当前浏览器不支持语音输入。建议使用 Chrome 或 Edge。",
|
||||
voiceInputPermissionDenied: "麦克风权限被拒绝。请允许麦克风访问后重试。",
|
||||
voiceInputMicrophoneUnavailable: "未检测到麦克风。请检查设备输入后重试。",
|
||||
voiceInputUnsupportedLanguage: "当前浏览器不支持该语言的语音输入。",
|
||||
voiceInputNetworkError: "无法连接浏览器语音识别服务。",
|
||||
voiceInputNoSpeech: "没有检测到语音,请重试。",
|
||||
voiceInputFailed: "语音输入失败,请重试。",
|
||||
mode: "模式",
|
||||
flashMode: "闪速",
|
||||
flashModeDescription: "快速且高效的完成任务,但可能不够精准",
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
export type SpeechRecognitionErrorCode =
|
||||
| "aborted"
|
||||
| "audio-capture"
|
||||
| "bad-grammar"
|
||||
| "language-not-supported"
|
||||
| "network"
|
||||
| "no-speech"
|
||||
| "not-allowed"
|
||||
| "phrases-not-supported"
|
||||
| "service-not-allowed";
|
||||
|
||||
export type SpeechRecognitionErrorKind =
|
||||
| "cancelled"
|
||||
| "microphone_unavailable"
|
||||
| "permission_denied"
|
||||
| "unsupported_language"
|
||||
| "network"
|
||||
| "no_speech"
|
||||
| "unknown";
|
||||
|
||||
export type SpeechRecognitionConstructor = new () => BrowserSpeechRecognition;
|
||||
|
||||
export type SpeechRecognitionEventLike = {
|
||||
results: SpeechRecognitionResultListLike;
|
||||
};
|
||||
|
||||
export type SpeechRecognitionErrorEventLike = {
|
||||
error?: SpeechRecognitionErrorCode | string;
|
||||
};
|
||||
|
||||
export type BrowserSpeechRecognition = {
|
||||
continuous: boolean;
|
||||
interimResults: boolean;
|
||||
lang: string;
|
||||
maxAlternatives: number;
|
||||
onend: (() => void) | null;
|
||||
onerror: ((event: SpeechRecognitionErrorEventLike) => void) | null;
|
||||
onresult: ((event: SpeechRecognitionEventLike) => void) | null;
|
||||
start: () => void;
|
||||
stop: () => void;
|
||||
abort: () => void;
|
||||
};
|
||||
|
||||
type SpeechRecognitionWindow = Window &
|
||||
typeof globalThis & {
|
||||
SpeechRecognition?: SpeechRecognitionConstructor;
|
||||
webkitSpeechRecognition?: SpeechRecognitionConstructor;
|
||||
};
|
||||
|
||||
export type SpeechRecognitionAlternativeLike = {
|
||||
transcript?: string;
|
||||
};
|
||||
|
||||
export type SpeechRecognitionResultLike = {
|
||||
0?: SpeechRecognitionAlternativeLike;
|
||||
isFinal: boolean;
|
||||
length: number;
|
||||
};
|
||||
|
||||
export type SpeechRecognitionResultListLike = {
|
||||
[index: number]: SpeechRecognitionResultLike | undefined;
|
||||
length: number;
|
||||
};
|
||||
|
||||
const DEFAULT_SPEECH_RECOGNITION_LANGUAGE = "en-US";
|
||||
const SPEECH_RECOGNITION_LANGUAGE_ALLOWLIST = new Set([
|
||||
"de",
|
||||
"en",
|
||||
"es",
|
||||
"fr",
|
||||
"it",
|
||||
"ja",
|
||||
"ko",
|
||||
"pt",
|
||||
"zh",
|
||||
]);
|
||||
|
||||
export function getSpeechRecognitionConstructor(
|
||||
value: unknown = globalThis,
|
||||
): SpeechRecognitionConstructor | null {
|
||||
const maybeWindow = value as Partial<SpeechRecognitionWindow>;
|
||||
return (
|
||||
maybeWindow.SpeechRecognition ?? maybeWindow.webkitSpeechRecognition ?? null
|
||||
);
|
||||
}
|
||||
|
||||
export function getSpeechRecognitionLanguage(locale: string): string {
|
||||
const normalized = normalizeBCP47Locale(locale);
|
||||
const language = normalized.split("-")[0]?.toLowerCase();
|
||||
|
||||
if (language === "zh") {
|
||||
return "zh-CN";
|
||||
}
|
||||
if (language && SPEECH_RECOGNITION_LANGUAGE_ALLOWLIST.has(language)) {
|
||||
return normalized;
|
||||
}
|
||||
return DEFAULT_SPEECH_RECOGNITION_LANGUAGE;
|
||||
}
|
||||
|
||||
export function shouldRestartSpeechRecognition(
|
||||
lastError: SpeechRecognitionErrorKind | null,
|
||||
): boolean {
|
||||
return lastError === null || lastError === "no_speech";
|
||||
}
|
||||
|
||||
export function readSpeechRecognitionTranscript(
|
||||
results: SpeechRecognitionResultListLike,
|
||||
): { finalText: string; interimText: string; text: string } {
|
||||
let finalText = "";
|
||||
let interimText = "";
|
||||
|
||||
for (const result of Array.from(
|
||||
{ length: results.length },
|
||||
(_, index) => results[index],
|
||||
)) {
|
||||
const transcript = result?.[0]?.transcript ?? "";
|
||||
if (result?.isFinal) {
|
||||
finalText += transcript;
|
||||
} else {
|
||||
interimText += transcript;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
finalText: normalizeSpeechTranscript(finalText),
|
||||
interimText: normalizeSpeechTranscript(interimText),
|
||||
text: normalizeSpeechTranscript(`${finalText}${interimText}`),
|
||||
};
|
||||
}
|
||||
|
||||
export function appendSpeechTranscript(baseText: string, transcript: string) {
|
||||
const cleanTranscript = normalizeSpeechTranscript(transcript);
|
||||
if (!cleanTranscript) {
|
||||
return baseText;
|
||||
}
|
||||
|
||||
const cleanBase = baseText.trimEnd();
|
||||
if (!cleanBase) {
|
||||
return cleanTranscript;
|
||||
}
|
||||
|
||||
return `${cleanBase} ${cleanTranscript}`;
|
||||
}
|
||||
|
||||
export function normalizeSpeechTranscript(value: string) {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
export function mapSpeechRecognitionError(
|
||||
error: SpeechRecognitionErrorCode | string | undefined,
|
||||
): SpeechRecognitionErrorKind {
|
||||
switch (error) {
|
||||
case "aborted":
|
||||
return "cancelled";
|
||||
case "audio-capture":
|
||||
return "microphone_unavailable";
|
||||
case "not-allowed":
|
||||
case "service-not-allowed":
|
||||
return "permission_denied";
|
||||
case "language-not-supported":
|
||||
return "unsupported_language";
|
||||
case "network":
|
||||
return "network";
|
||||
case "no-speech":
|
||||
return "no_speech";
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeBCP47Locale(locale: string): string {
|
||||
const trimmed = locale.trim();
|
||||
if (!trimmed) {
|
||||
return DEFAULT_SPEECH_RECOGNITION_LANGUAGE;
|
||||
}
|
||||
try {
|
||||
return Intl.getCanonicalLocales(trimmed)[0] ?? trimmed;
|
||||
} catch {
|
||||
return DEFAULT_SPEECH_RECOGNITION_LANGUAGE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, it } from "@rstest/core";
|
||||
|
||||
import {
|
||||
appendSpeechTranscript,
|
||||
getSpeechRecognitionConstructor,
|
||||
getSpeechRecognitionLanguage,
|
||||
mapSpeechRecognitionError,
|
||||
readSpeechRecognitionTranscript,
|
||||
shouldRestartSpeechRecognition,
|
||||
type SpeechRecognitionConstructor,
|
||||
} from "@/core/voice-input/speech-recognition";
|
||||
|
||||
describe("speech recognition helpers", () => {
|
||||
it("prefers the standard constructor and falls back to webkit", () => {
|
||||
const standard = makeSpeechRecognitionConstructor();
|
||||
const webkit = makeSpeechRecognitionConstructor();
|
||||
|
||||
expect(
|
||||
getSpeechRecognitionConstructor({
|
||||
SpeechRecognition: standard as unknown as SpeechRecognitionConstructor,
|
||||
webkitSpeechRecognition:
|
||||
webkit as unknown as SpeechRecognitionConstructor,
|
||||
}),
|
||||
).toBe(standard);
|
||||
expect(
|
||||
getSpeechRecognitionConstructor({
|
||||
webkitSpeechRecognition:
|
||||
webkit as unknown as SpeechRecognitionConstructor,
|
||||
}),
|
||||
).toBe(webkit);
|
||||
expect(getSpeechRecognitionConstructor({})).toBeNull();
|
||||
});
|
||||
|
||||
it("maps DeerFlow locales to browser recognition locales", () => {
|
||||
expect(getSpeechRecognitionLanguage("zh-CN")).toBe("zh-CN");
|
||||
expect(getSpeechRecognitionLanguage("zh-Hans")).toBe("zh-CN");
|
||||
expect(getSpeechRecognitionLanguage("en-US")).toBe("en-US");
|
||||
expect(getSpeechRecognitionLanguage("en-GB")).toBe("en-GB");
|
||||
expect(getSpeechRecognitionLanguage("fr-FR")).toBe("fr-FR");
|
||||
expect(getSpeechRecognitionLanguage("ja-JP")).toBe("ja-JP");
|
||||
expect(getSpeechRecognitionLanguage("xx-YY")).toBe("en-US");
|
||||
expect(getSpeechRecognitionLanguage("not a locale")).toBe("en-US");
|
||||
});
|
||||
|
||||
it("combines final and interim transcripts with whitespace cleanup", () => {
|
||||
expect(
|
||||
readSpeechRecognitionTranscript({
|
||||
length: 3,
|
||||
0: { isFinal: true, length: 1, 0: { transcript: " hello " } },
|
||||
1: { isFinal: true, length: 1, 0: { transcript: " world" } },
|
||||
2: { isFinal: false, length: 1, 0: { transcript: " again " } },
|
||||
}),
|
||||
).toEqual({
|
||||
finalText: "hello world",
|
||||
interimText: "again",
|
||||
text: "hello world again",
|
||||
});
|
||||
});
|
||||
|
||||
it("appends transcript to an existing draft without duplicating whitespace", () => {
|
||||
expect(appendSpeechTranscript("", " hello world ")).toBe("hello world");
|
||||
expect(appendSpeechTranscript("Draft", "voice text")).toBe(
|
||||
"Draft voice text",
|
||||
);
|
||||
expect(appendSpeechTranscript("Draft\n", "voice text")).toBe(
|
||||
"Draft voice text",
|
||||
);
|
||||
expect(appendSpeechTranscript("Draft", " ")).toBe("Draft");
|
||||
});
|
||||
|
||||
it("normalizes browser speech recognition errors", () => {
|
||||
expect(mapSpeechRecognitionError("not-allowed")).toBe("permission_denied");
|
||||
expect(mapSpeechRecognitionError("service-not-allowed")).toBe(
|
||||
"permission_denied",
|
||||
);
|
||||
expect(mapSpeechRecognitionError("audio-capture")).toBe(
|
||||
"microphone_unavailable",
|
||||
);
|
||||
expect(mapSpeechRecognitionError("language-not-supported")).toBe(
|
||||
"unsupported_language",
|
||||
);
|
||||
expect(mapSpeechRecognitionError("network")).toBe("network");
|
||||
expect(mapSpeechRecognitionError("no-speech")).toBe("no_speech");
|
||||
expect(mapSpeechRecognitionError("aborted")).toBe("cancelled");
|
||||
expect(mapSpeechRecognitionError("bad-grammar")).toBe("unknown");
|
||||
});
|
||||
|
||||
it("restarts only after browser auto-end conditions", () => {
|
||||
expect(shouldRestartSpeechRecognition(null)).toBe(true);
|
||||
expect(shouldRestartSpeechRecognition("no_speech")).toBe(true);
|
||||
expect(shouldRestartSpeechRecognition("cancelled")).toBe(false);
|
||||
expect(shouldRestartSpeechRecognition("permission_denied")).toBe(false);
|
||||
expect(shouldRestartSpeechRecognition("network")).toBe(false);
|
||||
expect(shouldRestartSpeechRecognition("unknown")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
function makeSpeechRecognitionConstructor(): SpeechRecognitionConstructor {
|
||||
return class {
|
||||
continuous = false;
|
||||
interimResults = false;
|
||||
lang = "";
|
||||
maxAlternatives = 1;
|
||||
onend = null;
|
||||
onerror = null;
|
||||
onresult = null;
|
||||
|
||||
start() {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
stop() {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
abort() {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user