Merge pull request #2777 from xinnan-tech/update_blasr_paraformer

Update blasr paraformer

main/manager-api/src/main/resources/db/changelog/202512301430.sql

@@ -0,0 +1,94 @@
+-- 添加阿里百炼Paraformer实时语音识别服务配置
+delete from `ai_model_provider` where id = 'SYSTEM_ASR_AliyunBLStream';
+INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
+('SYSTEM_ASR_AliyunBLStream', 'ASR', 'aliyunbl_stream', '阿里百炼Paraformer实时语音识别', '[{"key":"api_key","label":"API密钥","type":"password"},{"key":"model","label":"模型名称","type":"string"},{"key":"format","label":"音频格式","type":"string"},{"key":"sample_rate","label":"采样率","type":"number"},{"key":"output_dir","label":"输出目录","type":"string"}]', 18, 1, NOW(), 1, NOW());
+
+delete from `ai_model_config` where id = 'ASR_AliyunBLStream';
+INSERT INTO `ai_model_config` VALUES ('ASR_AliyunBLStream', 'ASR', 'AliyunBLStream', '阿里百炼Paraformer实时语音识别', 0, 1, '{"type": "aliyunbl_stream", "api_key": "", "model": "paraformer-realtime-v2", "format": "pcm", "sample_rate": 16000, "disfluency_removal_enabled": false, "semantic_punctuation_enabled": false, "max_sentence_silence": 200, "multi_threshold_mode_enabled": false, "punctuation_prediction_enabled": true, "inverse_text_normalization_enabled": true, "output_dir": "tmp/"}', 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service', '支持多语言、热词定制、语义断句等高级功能', 21, NULL, NULL, NULL, NULL);
+
+-- 更新阿里百炼Paraformer模型配置的说明文档
+UPDATE `ai_model_config` SET
+`doc_link` = 'https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service',
+`remark` = '阿里百炼Paraformer实时语音识别配置说明：
+1. 登录阿里云百炼平台 https://bailian.console.aliyun.com/
+2. 创建API-KEY https://bailian.console.aliyun.com/#/api-key
+3. 支持模型：paraformer-realtime-v2(推荐)、paraformer-realtime-8k-v2、paraformer-realtime-v1、paraformer-realtime-8k-v1
+4. 功能特性：
+   - 多语言支持(中文含方言、英文、日语、韩语、德语、法语、俄语)
+   - 热词定制(vocabulary_id参数)，详细说明请参考：https://help.aliyun.com/zh/model-studio/custom-hot-words?
+   - 语义断句/VAD断句(semantic_punctuation_enabled参数)
+   - 自动标点符号、ITN、过滤语气词等
+5. 参数说明：
+   - model: 模型名称，推荐paraformer-realtime-v2
+   - sample_rate: 采样率(Hz)，v2支持任意采样率，v1仅支持16000，8k版本仅支持8000
+   - semantic_punctuation_enabled: false为VAD断句(低延迟)，true为语义断句(高准确)
+   - max_sentence_silence: VAD断句静音时长阈值(200-6000ms)
+' WHERE `id` = 'ASR_AliyunBLStream';
+
+
+-- 更新豆包流式ASR供应器，增加配置
+delete from `ai_model_provider` where id = 'SYSTEM_ASR_DoubaoStreamASR';
+INSERT INTO `ai_model_provider` (`id`, `model_type`, `provider_code`, `name`, `fields`, `sort`, `creator`, `create_date`, `updater`, `update_date`) VALUES
+('SYSTEM_ASR_DoubaoStreamASR', 'ASR', 'doubao_stream', '火山引擎语音识别(流式)', '[{"key":"appid","label":"应用ID","type":"string"},{"key":"access_token","label":"访问令牌","type":"string"},{"key":"cluster","label":"集群","type":"string"},{"key":"boosting_table_name","label":"热词文件名称","type":"string"},{"key":"correct_table_name","label":"替换词文件名称","type":"string"},{"key":"output_dir","label":"输出目录","type":"string"},{"key":"end_window_size","label":"静音判定时长(ms)","type":"number"},{"key":"enable_multilingual","label":"是否开启多语种识别模式","type":"boolean"},{"key":"language","label":"指定语言编码","type":"string"}]', 3, 1, NOW(), 1, NOW());
+UPDATE `ai_model_config` SET 
+`remark` = '豆包ASR配置说明：
+1. 豆包ASR和豆包(流式)ASR的区别是：豆包ASR是按次收费，豆包(流式)ASR是按时收费
+2. 一般来说按次收费的更便宜，但是豆包(流式)ASR使用了大模型技术，效果更好
+3. 需要在火山引擎控制台创建应用并获取appid和access_token
+4. 支持中文语音识别
+5. 需要网络连接
+6. 输出文件保存在tmp/目录
+申请步骤：
+1. 访问 https://console.volcengine.com/speech/app
+2. 创建新应用
+3. 获取appid和access_token
+4. 填入配置文件中
+如需设置热词，请参考：https://www.volcengine.com/docs/6561/155738
+如开启多语种识别模式，请设置language当该键为空时，该模型支持中英文、上海话、闽南语，四川、陕西、粤语识别。其他语种请参考：https://www.volcengine.com/docs/6561/1354869
+' WHERE `id` = 'ASR_DoubaoStreamASR';
+
+-- 更新豆包流式ASR模型配置，增加enable_multilingual默认值
+UPDATE `ai_model_config` SET
+`config_json` = JSON_SET(
+    `config_json`, 
+    '$.enable_multilingual', false,
+    '$.language', 'zh-CN'
+)
+WHERE `id` = 'ASR_DoubaoStreamASR' 
+AND JSON_EXTRACT(`config_json`, '$.enable_multilingual') IS NULL 
+AND JSON_EXTRACT(`config_json`, '$.language') IS NULL;
+
+
+-- 更新HuoshanDoubleStreamTTS供应器配置，增加多情感音色参数
+UPDATE `ai_model_provider`
+SET `fields` = '[{"key": "ws_url", "type": "string", "label": "WebSocket地址"}, {"key": "appid", "type": "string", "label": "应用ID"}, {"key": "access_token", "type": "string", "label": "访问令牌"}, {"key": "resource_id", "type": "string", "label": "资源ID"}, {"key": "speaker", "type": "string", "label": "默认音色"}, {"key": "enable_ws_reuse", "type": "boolean", "label": "是否开启链接复用", "default": true}, {"key": "speech_rate", "type": "number", "label": "语速(-50~100)"}, {"key": "loudness_rate", "type": "number", "label": "音量(-50~100)"}, {"key": "pitch", "type": "number", "label": "音高(-12~12)"}, {"key": "emotion_scale", "type": "number", "label": "情感强度(1-5)"}, {"key": "emotion", "type": "string", "label": "情感类型"}]'
+WHERE `id` = 'SYSTEM_TTS_HSDSTTS';
+
+-- 更新默认值
+UPDATE `ai_model_config` SET
+`config_json` = JSON_SET(
+    `config_json`,
+    '$.emotion', 'neutral',
+    '$.emotion_scale', 4
+)
+WHERE `id` = 'TTS_HuoshanDoubleStreamTTS'
+AND JSON_EXTRACT(`config_json`, '$.emotion') IS NULL 
+AND JSON_EXTRACT(`config_json`, '$.emotion_scale') IS NULL;
+
+-- 增加文档链接和备注
+UPDATE `ai_model_config` SET 
+`doc_link` = 'https://console.volcengine.com/speech/service/10007',
+`remark` = '火山引擎语音合成服务配置说明：
+1. 访问 https://www.volcengine.com/ 注册并开通火山引擎账号
+2. 访问 https://console.volcengine.com/speech/service/10007 开通语音合成大模型，购买音色
+3. 在页面底部获取appid和access_token
+5. 资源ID固定为：volc.service_type.10029（大模型语音合成及混音）
+6. 链接复用：开启WebSocket连接复用，默认true减少链接损耗（注意：复用后设备处于聆听状态时空闲链接会占并发数）
+7. 语速：-50~100，可不填，正常默认值0，可填-50~100
+8. 音量：-50~100，可不填，正常默认值0，可填-50~100
+9. 音高：-12~12，可不填，正常默认值0，可填-12~12
+10. 多情感参数（当前仅部分音色支持设置情感）：
+   相关音色列表：https://www.volcengine.com/docs/6561/1257544
+    - emotion_scale：情感强度，可选值为：1~5，默认值为4
+    - emotion：情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised
+' WHERE `id` = 'TTS_HuoshanDoubleStreamTTS';

main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -472,4 +472,11 @@ databaseChangeLog:
       changes:
         - sqlFile:
             encoding: utf8
-            path: classpath:db/changelog/202512221117.sql
+            path: classpath:db/changelog/202512221117.sql
+  - changeSet:
+      id: 202512301430
+      author: RanChen
+      changes:
+        - sqlFile:
+            encoding: utf8
+            path: classpath:db/changelog/202512301430.sql

main/xiaozhi-server/config.yaml

@@ -346,6 +346,11 @@ ASR:
     # 热词、替换词使用流程：https://www.volcengine.com/docs/6561/155738
     boosting_table_name: （选填）你的热词文件名称
     correct_table_name: （选填）你的替换词文件名称
+    # 是否开启多语种识别模式
+    enable_multilingual: False
+    # 多语种识别当该键为空时，该模型支持中英文、上海话、闽南语，四川、陕西、粤语识别。当将其设置为特定键时，它可以识别指定语言。
+    # 详细语言列表参考 https://www.volcengine.com/docs/6561/1354869
+    # language: zh-cn
     # 静音判定时长(ms)，默认200ms
     end_window_size: 200
     output_dir: tmp/
@@ -478,7 +483,32 @@ ASR:
     accent: mandarin # 方言，mandarin:普通话
     # 调整音频处理参数以提高长语音识别质量
     output_dir: tmp/
-  
+  AliyunBLStreamASR:
+    # 阿里百炼Paraformer实时语音识别服务
+    # WebSocket实时流式语音识别，支持多语言、热词定制、语义断句等高级功能
+    # 平台地址：https://bailian.console.aliyun.com/
+    # API Key地址：https://bailian.console.aliyun.com/#/api-key
+    # 文档地址：https://help.aliyun.com/zh/model-studio/websocket-for-paraformer-real-time-service
+    # 支持模型：paraformer-realtime-v2(推荐), paraformer-realtime-8k-v2, paraformer-realtime-v1, paraformer-realtime-8k-v1
+    type: aliyunbl_stream
+    # 必填参数
+    api_key: 你的阿里云百炼API密钥
+    # 模型选择，推荐使用v2版本
+    model: paraformer-realtime-v2
+    # 音频格式和采样率
+    format: pcm
+    sample_rate: 16000  # v2支持任意采样率，v1仅支持16000，8k版本仅支持8000
+    # 可选参数
+    disfluency_removal_enabled: false  # 是否过滤语气词(如"嗯"、"啊"等)
+    semantic_punctuation_enabled: false  # 语义断句(true:会议场景,准确;false:VAD断句,交互场景,低延迟)
+    max_sentence_silence: 200  # VAD断句静音时长阈值(ms)，范围200-6000，仅VAD断句时生效
+    multi_threshold_mode_enabled: false  # 防止VAD断句切割过长，仅VAD断句时生效
+    punctuation_prediction_enabled: true  # 是否自动添加标点符号
+    inverse_text_normalization_enabled: true  # 是否开启ITN(中文数字转阿拉伯数字)
+    # 热词定制文档地址：https://help.aliyun.com/zh/model-studio/custom-hot-words?
+    # vocabulary_id: vocab-xxx-24ee19fa8cfb4d52902170a0xxxxxxxx  # 热词ID(可选)
+    # language_hints: ["zh", "en"]  # 指定语言(可选)，支持zh、en、ja、yue、ko、de、fr、ru
+    output_dir: tmp/  
 VAD:
   SileroVAD:
     type: silero
@@ -687,6 +717,10 @@ TTS:
     speech_rate: 0
     loudness_rate: 0
     pitch: 0
+    # 多情感音色参数，注意：当前仅部分音色支持设置情感。
+    # 相关音色列表：https://www.volcengine.com/docs/6561/1257544
+    emotion: "neutral"  # 情感类型，可选值为：neutral、happy、sad、angry、fearful、disgusted、surprised
+    emotion_scale: 4  # 情感强度，可选值为：1~5，默认值为4
   CosyVoiceSiliconflow:
     type: siliconflow
     # 硅基流动TTS

main/xiaozhi-server/core/providers/asr/aliyunbl_stream.py

@@ -0,0 +1,343 @@
+import json
+import uuid
+import asyncio
+import websockets
+import opuslib_next
+from typing import List
+from config.logger import setup_logging
+from core.providers.asr.base import ASRProviderBase
+from core.providers.asr.dto.dto import InterfaceType
+
+TAG = __name__
+logger = setup_logging()
+
+
+class ASRProvider(ASRProviderBase):
+    def __init__(self, config, delete_audio_file):
+        super().__init__()
+        self.interface_type = InterfaceType.STREAM
+        self.config = config
+        self.text = ""
+        self.decoder = opuslib_next.Decoder(16000, 1)
+        self.asr_ws = None
+        self.forward_task = None
+        self.is_processing = False
+        self.server_ready = False  # 服务器准备状态
+        self.task_id = None  # 当前任务ID
+
+        # 阿里百炼配置
+        self.api_key = config.get("api_key")
+        self.model = config.get("model", "paraformer-realtime-v2")
+        self.sample_rate = config.get("sample_rate", 16000)
+        self.format = config.get("format", "pcm")
+
+        # 可选参数
+        self.vocabulary_id = config.get("vocabulary_id")
+        self.disfluency_removal_enabled = config.get("disfluency_removal_enabled", False)
+        self.language_hints = config.get("language_hints")
+        self.semantic_punctuation_enabled = config.get("semantic_punctuation_enabled", False)
+        max_sentence_silence = config.get("max_sentence_silence")
+        self.max_sentence_silence = int(max_sentence_silence) if max_sentence_silence else 200
+        self.multi_threshold_mode_enabled = config.get("multi_threshold_mode_enabled", False)
+        self.punctuation_prediction_enabled = config.get("punctuation_prediction_enabled", True)
+        self.inverse_text_normalization_enabled = config.get("inverse_text_normalization_enabled", True)
+
+        # WebSocket URL
+        self.ws_url = "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
+
+        self.output_dir = config.get("output_dir", "./audio_output")
+        self.delete_audio_file = delete_audio_file
+
+    async def open_audio_channels(self, conn):
+        await super().open_audio_channels(conn)
+
+    async def receive_audio(self, conn, audio, audio_have_voice):
+        # 初始化音频缓存
+        if not hasattr(conn, 'asr_audio_for_voiceprint'):
+            conn.asr_audio_for_voiceprint = []
+
+        # 存储音频数据
+        if audio:
+            conn.asr_audio_for_voiceprint.append(audio)
+
+        conn.asr_audio.append(audio)
+        conn.asr_audio = conn.asr_audio[-10:]
+
+        # 只在有声音且没有连接时建立连接
+        if audio_have_voice and not self.is_processing and not self.asr_ws:
+            try:
+                await self._start_recognition(conn)
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"开始识别失败: {str(e)}")
+                await self._cleanup()
+                return
+
+        # 发送音频数据
+        if self.asr_ws and self.is_processing and self.server_ready:
+            try:
+                pcm_frame = self.decoder.decode(audio, 960)
+                # 直接发送PCM音频数据(二进制)
+                await self.asr_ws.send(pcm_frame)
+            except Exception as e:
+                logger.bind(tag=TAG).warning(f"发送音频失败: {str(e)}")
+                await self._cleanup()
+
+    async def _start_recognition(self, conn):
+        """开始识别会话"""
+        try:
+            # 如果为手动模式,设置超时时长为最大值
+            if conn.client_listen_mode == "manual":
+                self.max_sentence_silence = 6000
+
+            self.is_processing = True
+            self.task_id = uuid.uuid4().hex
+
+            # 建立WebSocket连接
+            headers = {
+                "Authorization": f"Bearer {self.api_key}"
+            }
+
+            logger.bind(tag=TAG).debug(f"正在连接阿里百炼ASR服务, task_id: {self.task_id}")
+
+            self.asr_ws = await websockets.connect(
+                self.ws_url,
+                additional_headers=headers,
+                max_size=1000000000,
+                ping_interval=None,
+                ping_timeout=None,
+                close_timeout=5,
+            )
+
+            logger.bind(tag=TAG).debug("WebSocket连接建立成功")
+
+            self.server_ready = False
+            self.forward_task = asyncio.create_task(self._forward_results(conn))
+
+            # 发送run-task指令
+            run_task_msg = self._build_run_task_message()
+            await self.asr_ws.send(json.dumps(run_task_msg, ensure_ascii=False))
+            logger.bind(tag=TAG).debug("已发送run-task指令，等待服务器准备...")
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"建立ASR连接失败: {str(e)}")
+            if self.asr_ws:
+                await self.asr_ws.close()
+                self.asr_ws = None
+            self.is_processing = False
+            raise
+
+    def _build_run_task_message(self) -> dict:
+        """构建run-task指令"""
+        message = {
+            "header": {
+                "action": "run-task",
+                "task_id": self.task_id,
+                "streaming": "duplex"
+            },
+            "payload": {
+                "task_group": "audio",
+                "task": "asr",
+                "function": "recognition",
+                "model": self.model,
+                "parameters": {
+                    "format": self.format,
+                    "sample_rate": self.sample_rate,
+                    "disfluency_removal_enabled": self.disfluency_removal_enabled,
+                    "semantic_punctuation_enabled": self.semantic_punctuation_enabled,
+                    "max_sentence_silence": self.max_sentence_silence,
+                    "multi_threshold_mode_enabled": self.multi_threshold_mode_enabled,
+                    "punctuation_prediction_enabled": self.punctuation_prediction_enabled,
+                    "inverse_text_normalization_enabled": self.inverse_text_normalization_enabled,
+                },
+                "input": {}
+            }
+        }
+
+        # 只有当模型名称以v2结尾时才添加vocabulary_id参数
+        if self.model.lower().endswith("v2"):
+            message["payload"]["parameters"]["vocabulary_id"] = self.vocabulary_id
+
+        if self.language_hints:
+            message["payload"]["parameters"]["language_hints"] = self.language_hints
+
+        return message
+
+    async def _forward_results(self, conn):
+        """转发识别结果"""
+        try:
+            while not conn.stop_event.is_set():
+                try:
+                    response = await asyncio.wait_for(self.asr_ws.recv(), timeout=1.0)
+                    result = json.loads(response)
+
+                    header = result.get("header", {})
+                    payload = result.get("payload", {})
+                    event = header.get("event", "")
+
+                    # 处理task-started事件
+                    if event == "task-started":
+                        self.server_ready = True
+                        logger.bind(tag=TAG).debug("服务器已准备，开始发送缓存音频...")
+
+                        # 发送缓存音频
+                        if conn.asr_audio:
+                            for cached_audio in conn.asr_audio[-10:]:
+                                try:
+                                    pcm_frame = self.decoder.decode(cached_audio, 960)
+                                    await self.asr_ws.send(pcm_frame)
+                                except Exception as e:
+                                    logger.bind(tag=TAG).warning(f"发送缓存音频失败: {e}")
+                                    break
+                        continue
+
+                    # 处理result-generated事件
+                    elif event == "result-generated":
+                        output = payload.get("output", {})
+                        sentence = output.get("sentence", {})
+
+                        text = sentence.get("text", "")
+                        sentence_end = sentence.get("sentence_end", False)
+                        end_time = sentence.get("end_time")
+
+                        # 判断是否为最终结果(sentence_end为True且end_time不为null)
+                        is_final = sentence_end and end_time is not None
+
+                        if is_final:
+                            logger.bind(tag=TAG).info(f"识别到文本: {text}")
+
+                            # 手动模式下累积识别结果
+                            if conn.client_listen_mode == "manual":
+                                if self.text:
+                                    self.text += text
+                                else:
+                                    self.text = text
+
+                                # 手动模式下,只有在收到stop信号后才触发处理
+                                if conn.client_voice_stop:
+                                    audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                    if len(audio_data) > 0:
+                                        logger.bind(tag=TAG).debug("收到最终识别结果，触发处理")
+                                        await self.handle_voice_stop(conn, audio_data)
+                                        # 清理音频缓存
+                                        conn.asr_audio.clear()
+                                        conn.reset_vad_states()
+                                    break
+                            else:
+                                # 自动模式下直接覆盖
+                                self.text = text
+                                conn.reset_vad_states()
+                                audio_data = getattr(conn, 'asr_audio_for_voiceprint', [])
+                                await self.handle_voice_stop(conn, audio_data)
+                                break
+
+                    # 处理task-finished事件
+                    elif event == "task-finished":
+                        logger.bind(tag=TAG).debug("任务已完成")
+                        break
+
+                    # 处理task-failed事件
+                    elif event == "task-failed":
+                        error_code = header.get("error_code", "UNKNOWN")
+                        error_message = header.get("error_message", "未知错误")
+                        logger.bind(tag=TAG).error(f"任务失败: {error_code} - {error_message}")
+                        break
+
+                except asyncio.TimeoutError:
+                    continue
+                except websockets.ConnectionClosed:
+                    logger.bind(tag=TAG).info("ASR服务连接已关闭")
+                    self.is_processing = False
+                    break
+                except Exception as e:
+                    logger.bind(tag=TAG).error(f"处理结果失败: {str(e)}")
+                    break
+
+        except Exception as e:
+            logger.bind(tag=TAG).error(f"结果转发失败: {str(e)}")
+        finally:
+            # 清理连接的音频缓存
+            await self._cleanup()
+            if conn:
+                if hasattr(conn, 'asr_audio_for_voiceprint'):
+                    conn.asr_audio_for_voiceprint = []
+                if hasattr(conn, 'asr_audio'):
+                    conn.asr_audio = []
+
+    async def _send_stop_request(self):
+        """发送停止请求(用于手动模式停止录音)"""
+        if self.asr_ws:
+            try:
+                # 先停止音频发送
+                self.is_processing = False
+
+                logger.bind(tag=TAG).debug("收到停止请求，发送finish-task指令")
+                await self._send_finish_task()
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送停止请求失败: {e}")
+
+    async def _send_finish_task(self):
+        """发送finish-task指令"""
+        if self.asr_ws and self.task_id:
+            try:
+                finish_msg = {
+                    "header": {
+                        "action": "finish-task",
+                        "task_id": self.task_id,
+                        "streaming": "duplex"
+                    },
+                    "payload": {
+                        "input": {}
+                    }
+                }
+                await self.asr_ws.send(json.dumps(finish_msg, ensure_ascii=False))
+                logger.bind(tag=TAG).debug("已发送finish-task指令")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"发送finish-task指令失败: {e}")
+
+    async def _cleanup(self):
+        """清理资源"""
+        logger.bind(tag=TAG).debug(f"开始ASR会话清理 | 当前状态: processing={self.is_processing}, server_ready={self.server_ready}")
+
+        # 状态重置
+        self.is_processing = False
+        self.server_ready = False
+        logger.bind(tag=TAG).debug("ASR状态已重置")
+
+        # 关闭连接
+        if self.asr_ws:
+            try:
+                # 先发送finish-task指令
+                await self._send_finish_task()
+                # 等待一小段时间让服务器处理
+                await asyncio.sleep(0.1)
+
+                logger.bind(tag=TAG).debug("正在关闭WebSocket连接")
+                await asyncio.wait_for(self.asr_ws.close(), timeout=2.0)
+                logger.bind(tag=TAG).debug("WebSocket连接已关闭")
+            except Exception as e:
+                logger.bind(tag=TAG).error(f"关闭WebSocket连接失败: {e}")
+            finally:
+                self.asr_ws = None
+
+        # 清理任务引用
+        self.forward_task = None
+        self.task_id = None
+
+        logger.bind(tag=TAG).debug("ASR会话清理完成")
+
+    async def speech_to_text(self, opus_data, session_id, audio_format):
+        """获取识别结果"""
+        result = self.text
+        self.text = ""
+        return result, None
+
+    async def close(self):
+        """关闭资源"""
+        await self._cleanup()
+        if hasattr(self, 'decoder') and self.decoder is not None:
+            try:
+                del self.decoder
+                self.decoder = None
+                logger.bind(tag=TAG).debug("Aliyun BL decoder resources released")
+            except Exception as e:
+                logger.bind(tag=TAG).debug(f"释放Aliyun BL decoder资源时出错: {e}")

main/xiaozhi-server/core/providers/asr/doubao_stream.py

@@ -33,7 +33,12 @@ class ASRProvider(ASRProviderBase):
         self.delete_audio_file = delete_audio_file
 
         # 火山引擎ASR配置
-        self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
+        enable_multilingual = config.get("enable_multilingual", False)
+        self.enable_multilingual = False if str(enable_multilingual).lower() == 'false' else True
+        if self.enable_multilingual:
+            self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel_nostream"
+        else:
+            self.ws_url = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
         self.uid = config.get("uid", "streaming_asr_service")
         self.workflow = config.get(
             "workflow", "audio_in,resample,partition,vad,fe,decode,itn,nlu_punctuate"
@@ -42,7 +47,8 @@ class ASRProvider(ASRProviderBase):
         self.format = config.get("format", "pcm")
         self.codec = config.get("codec", "pcm")
         self.rate = config.get("sample_rate", 16000)
-        self.language = config.get("language", "zh-CN")
+        # language参数仅在多语种模式(bigmodel_nostream)下有效
+        self.language = config.get("language") if self.enable_multilingual else None
         self.bits = config.get("bits", 16)
         self.channel = config.get("channel", 1)
         self.auth_method = config.get("auth_method", "token")
@@ -175,7 +181,8 @@ class ASRProvider(ASRProviderBase):
                             utterances = payload["result"].get("utterances", [])
                             # 检查duration和空文本的情况
                             if (
-                                payload.get("audio_info", {}).get("duration", 0) > 2000
+                                not self.enable_multilingual # 注意：多语种模式不返回中间结果，需要等待最终结果
+                                and payload.get("audio_info", {}).get("duration", 0) > 2000
                                 and not utterances
                                 and not payload["result"].get("text")
                                 and conn.client_listen_mode != "manual"
@@ -189,6 +196,10 @@ class ASRProvider(ASRProviderBase):
 
                             # 专门处理没有文本的识别结果（手动模式下可能已经识别完成但是没松按键）
                             elif not payload["result"].get("text") and not utterances:
+                                # 多语种模式会持续返回空文本，直到最后返回完整结果，所以需要排除
+                                if self.enable_multilingual:
+                                    continue
+
                                 if conn.client_listen_mode == "manual" and conn.client_voice_stop and len(audio_data) > 0:
                                     logger.bind(tag=TAG).debug("消息结束收到停止信号，触发处理")
                                     await self.handle_voice_stop(conn, audio_data)
@@ -299,12 +310,16 @@ class ASRProvider(ASRProviderBase):
                 "format": self.format,
                 "codec": self.codec,
                 "rate": self.rate,
-                "language": self.language,
                 "bits": self.bits,
                 "channel": self.channel,
                 "sample_rate": self.rate,
             },
         }
+
+        # language参数仅在多语种模式下添加
+        if self.enable_multilingual and self.language:
+            req["audio"]["language"] = self.language
+
         logger.bind(tag=TAG).debug(
             f"构造请求参数: {json.dumps(req, ensure_ascii=False)}"
         )

main/xiaozhi-server/core/providers/tts/huoshan_double_stream.py

@@ -160,11 +160,16 @@ class TTSProvider(TTSProviderBase):
         self.speech_rate = int(speech_rate) if speech_rate else 0
         self.loudness_rate = int(loudness_rate) if loudness_rate else 0
         self.pitch = int(pitch) if pitch else 0
+        # 多情感音色参数
+        self.emotion = config.get("emotion", "neutral")  
+        emotion_scale = config.get("emotion_scale", "4")
+        self.emotion_scale = int(emotion_scale) if emotion_scale else 4
+
         self.ws_url = config.get("ws_url")
         self.authorization = config.get("authorization")
         self.header = {"Authorization": f"{self.authorization}{self.access_token}"}
         enable_ws_reuse_value = config.get("enable_ws_reuse", True)
-        self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() in ('false', 'False') else True
+        self.enable_ws_reuse = False if str(enable_ws_reuse_value).lower() == 'false' else True
         self.tts_text = ""
         self.opus_encoder = opus_encoder_utils.OpusEncoderUtils(
             sample_rate=16000, channels=1, frame_size_ms=60
@@ -642,6 +647,19 @@ class TTSProvider(TTSProviderBase):
         audio_format="pcm",
         audio_sample_rate=16000,
     ):
+        audio_params = {
+            "format": audio_format,
+            "sample_rate": audio_sample_rate,
+            "speech_rate": self.speech_rate,
+            "loudness_rate": self.loudness_rate
+        }
+
+        # 如果是多情感音色,添加情感参数
+        if '_emo_' in self.voice:
+            if self.emotion:
+                audio_params["emotion"] = self.emotion
+                audio_params["emotion_scale"] = self.emotion_scale
+
         return str.encode(
             json.dumps(
                 {
@@ -651,12 +669,7 @@ class TTSProvider(TTSProviderBase):
                     "req_params": {
                         "text": text,
                         "speaker": speaker,
-                        "audio_params": {
-                            "format": audio_format,
-                            "sample_rate": audio_sample_rate,
-                            "speech_rate": self.speech_rate,
-                            "loudness_rate": self.loudness_rate
-                        },
+                        "audio_params": audio_params,
                         "additions": json.dumps({
                             "post_process": {
                                 "pitch": self.pitch