furyhawk/xiaozhi-esp32-server
1
0
Fork 0
mirror of https://github.com/xinnan-tech/xiaozhi-esp32-server.git synced 2026-07-21 10:55:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2798 from xinnan-tech/py_asr_emotion_language

Browse Source 
Py asr emotion language
This commit is contained in:
hrz
2026-01-05 15:03:50 +08:00
committed by GitHub
parent a2023a65c0 3f3f3fcd57
commit 815af687ea
11 changed files with 157 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/manager-api/src/main/resources/db/changelog/202601051433.sql
+11
View File
@@ -0,0 +1,11 @@
-- 更改FunASRServer说明文档模型为SenseVoiceSmall
UPDATE `ai_model_config` SET
`doc_link` = 'https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md',
`remark` = '独立部署FunASR，使用FunASR的API服务，只需要五句话
第一句：mkdir -p ./funasr-runtime-resources/models
第二句：sudo docker run -d -p 10096:10095 --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir iic/SenseVoiceSmall-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
上一句话执行后会进入到容器，继续第五句：tail -f log.txt
第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer';
main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml
+7
View File
@@ -480,3 +480,10 @@ databaseChangeLog:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202512301430.sql
- changeSet:
id: 202601051433
author: RanChen
changes:
- sqlFile:
encoding: utf8
path: classpath:db/changelog/202601051433.sql
main/xiaozhi-server/config.yaml
+1 -1
View File
@@ -296,7 +296,7 @@ ASR:
# 第一句：mkdir -p ./funasr-runtime-resources/models
# 第二句：sudo docker run -p 10096:10095 -it --privileged=true -v $PWD/funasr-runtime-resources/models:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-online-cpu-0.1.12
# 上一句话执行后会进入到容器，继续第三句：cd FunASR/runtime
# 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
# 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir iic/SenseVoiceSmall-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 &
# 上一句话执行后会进入到容器，继续第五句：tail -f log.txt
# 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了
# 以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md
main/xiaozhi-server/core/connection.py
+2
View File
@@ -130,6 +130,8 @@ class ConnectionHandler:
# 所以涉及到ASR的变量，需要在这里定义，属于connection的私有变量
self.asr_audio = []
self.asr_audio_queue = queue.Queue()
self.current_speaker = None # 存储当前说话人
self.current_language_tag = None # 存储当前ASR识别的语言标签
# llm相关变量
self.llm_finish_task = True
main/xiaozhi-server/core/handle/receiveAudioHandle.py
+7
View File
@@ -40,6 +40,7 @@ async def resume_vad_detection(conn):
async def startToChat(conn, text):
# 检查输入是否是JSON格式（包含说话人信息）
speaker_name = None
language_tag = None
actual_text = text
try:
@@ -48,6 +49,7 @@ async def startToChat(conn, text):
data = json.loads(text)
if "speaker" in data and "content" in data:
speaker_name = data["speaker"]
language_tag = data["language"]
actual_text = data["content"]
conn.logger.bind(tag=TAG).info(f"解析到说话人信息: {speaker_name}")
@@ -62,6 +64,11 @@ async def startToChat(conn, text):
conn.current_speaker = speaker_name
else:
conn.current_speaker = None
# 保存语种信息到连接对象
if language_tag:
conn.current_language_tag = language_tag
else:
conn.current_language_tag = "zh"
if conn.need_bind:
await check_bind_device(conn)
main/xiaozhi-server/core/providers/asr/base.py
+31 -9
View File
@@ -118,23 +118,45 @@ class ASRProviderBase(ABC):
else:
speaker_name = voiceprint_result
if raw_text:
logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
if speaker_name:
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
# 判断 ASR 结果类型
if isinstance(raw_text, dict):
# FunASR 返回的 dict 格式
if speaker_name:
raw_text["speaker"] = speaker_name
# 记录识别结果
if raw_text.get("language"):
logger.bind(tag=TAG).info(f"识别语言: {raw_text['language']}")
if raw_text.get("emotion"):
logger.bind(tag=TAG).info(f"识别情绪: {raw_text['emotion']}")
if raw_text.get("content"):
logger.bind(tag=TAG).info(f"识别文本: {raw_text['content']}")
if speaker_name:
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
# 转换为 JSON 字符串用于下游
enhanced_text = json.dumps(raw_text, ensure_ascii=False)
content_for_length_check = raw_text.get("content", "")
else:
# 其他 ASR 返回的纯文本
if raw_text:
logger.bind(tag=TAG).info(f"识别文本: {raw_text}")
if speaker_name:
logger.bind(tag=TAG).info(f"识别说话人: {speaker_name}")
# 构建包含说话人信息的JSON字符串
enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
content_for_length_check = raw_text
# 性能监控
total_time = time.monotonic() - total_start_time
logger.bind(tag=TAG).debug(f"总处理耗时: {total_time:.3f}s")
# 检查文本长度
text_len, _ = remove_punctuation_and_length(raw_text)
text_len, _ = remove_punctuation_and_length(content_for_length_check)
self.stop_ws_connection()
if text_len > 0:
# 构建包含说话人信息的JSON字符串
enhanced_text = self._build_enhanced_text(raw_text, speaker_name)
# 使用自定义模块进行上报
await startToChat(conn, enhanced_text)
enqueue_asr_report(conn, enhanced_text, asr_audio_task)
@@ -145,7 +167,7 @@ class ASRProviderBase(ABC):
logger.bind(tag=TAG).debug(f"异常详情: {traceback.format_exc()}")
def _build_enhanced_text(self, text: str, speaker_name: Optional[str]) -> str:
"""构建包含说话人信息的文本"""
"""构建包含说话人信息的文本（仅用于纯文本ASR）"""
if speaker_name and speaker_name.strip():
return json.dumps({
"speaker": speaker_name,
main/xiaozhi-server/core/providers/asr/fun_local.py
+4 -4
View File
@@ -6,10 +6,10 @@ import shutil
import psutil
import asyncio
from funasr import AutoModel
from config.logger import setup_logging
from typing import Optional, Tuple, List
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
from core.providers.asr.utils import lang_tag_filter
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.dto.dto import InterfaceType
@@ -102,9 +102,9 @@ class ASRProvider(ASRProviderBase):
use_itn=True,
batch_size_s=60,
)
text = await asyncio.to_thread(rich_transcription_postprocess, result[0]["text"])
text = lang_tag_filter(result[0]["text"])
logger.bind(tag=TAG).debug(
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text}"
f"语音识别耗时: {time.time() - start_time:.3f}s | 结果: {text['content']}"
)
return text, file_path
main/xiaozhi-server/core/providers/asr/fun_server.py
+14 -9
View File
@@ -1,12 +1,13 @@
from typing import Optional, Tuple, List
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.dto.dto import InterfaceType
import ssl
import json
import websockets
from config.logger import setup_logging
import asyncio
import re
import websockets
from config.logger import setup_logging
from typing import Optional, Tuple, List
from core.providers.asr.base import ASRProviderBase
from core.providers.asr.utils import lang_tag_filter
from core.providers.asr.dto.dto import InterfaceType
TAG = __name__
logger = setup_logging()
@@ -151,9 +152,13 @@ class ASRProvider(ASRProviderBase):
# Get the result from the receive task
result = receive_task.result()
match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
if match:
result = match.group(4).strip()
# match = re.match(r"<\|(.*?)\|><\|(.*?)\|><\|(.*?)\|>(.*)", result)
# if match:
# result = match.group(4).strip()
# Handle tags
result = lang_tag_filter(result)
return (
result,
file_path,
main/xiaozhi-server/core/providers/asr/utils.py
+79
View File
@@ -0,0 +1,79 @@
import re
from config.logger import setup_logging
TAG = __name__
logger = setup_logging()
EMOTION_EMOJI_MAP = {
"HAPPY": "🙂",
"SAD": "😔",
"ANGRY": "😡",
"NEUTRAL": "😶",
"FEARFUL": "😰",
"DISGUSTED": "🤢",
"SURPRISED": "😲",
"EMO_UNKNOWN": "😶", # 未知情绪默认用中性表情
}
# EVENT_EMOJI_MAP = {
# "<|BGM|>": "🎼",
# "<|Speech|>": "",
# "<|Applause|>": "👏",
# "<|Laughter|>": "😀",
# "<|Cry|>": "😭",
# "<|Sneeze|>": "🤧",
# "<|Breath|>": "",
# "<|Cough|>": "🤧",
# }
def lang_tag_filter(text: str) -> dict | str:
"""
解析 FunASR 识别结果，按顺序提取标签和纯文本内容
Args:
text: ASR 识别的原始文本，可能包含多种标签
Returns:
dict: {"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好"} 如果有标签
str: 纯文本，如果没有标签
Examples:
FunASR 输出格式：<|语种|><|情绪|><|事件|><|其他选项|>原文
>>> lang_tag_filter("<|zh|><|SAD|><|Speech|><|withitn|>你好啊，测试测试。")
{"language": "zh", "emotion": "SAD", "emoji": "😔", "content": "你好啊，测试测试。"}
>>> lang_tag_filter("<|en|><|HAPPY|><|Speech|><|withitn|>Hello hello.")
{"language": "en", "emotion": "HAPPY", "emoji": "🙂", "content": "Hello hello."}
>>> lang_tag_filter("plain text")
"plain text"
"""
# 提取所有标签（按顺序）
tag_pattern = r"<\|([^|]+)\|>"
all_tags = re.findall(tag_pattern, text)
# 移除所有 <|...|> 格式的标签，获取纯文本
clean_text = re.sub(tag_pattern, "", text).strip()
# 如果没有标签，直接返回纯文本
if not all_tags:
return clean_text
# 按照 FunASR 的固定顺序提取标签，返回 dict
language = all_tags[0] if len(all_tags) > 0 else "zh"
emotion = all_tags[1] if len(all_tags) > 1 else "NEUTRAL"
# event = all_tags[2] if len(all_tags) > 2 else "Speech" # 事件标签暂不使用
result = {
"content": clean_text,
"language": language,
"emotion": emotion,
# "event": event,
}
# 添加 emoji 映射
if emotion in EMOTION_EMOJI_MAP:
result["emotion"] = EMOTION_EMOJI_MAP[emotion]
# 事件标签暂不使用
# if event in EVENT_EMOJI_MAP:
# result["event"] = EVENT_EMOJI_MAP[event]
return result
main/xiaozhi-server/core/providers/tts/alibl_stream.py
+1 -1
View File
@@ -519,4 +519,4 @@ class TTSProvider(TTSProviderBase):
except Exception as e:
logger.bind(tag=TAG).error(f"生成音频数据失败: {str(e)}")
return []
return []
main/xiaozhi-server/core/utils/util.py
-23
View File
@@ -15,29 +15,6 @@ from pydub import AudioSegment
from typing import Callable, Any
TAG = __name__
emoji_map = {
"neutral": "😶",
"happy": "🙂",
"laughing": "😆",
"funny": "😂",
"sad": "😔",
"angry": "😠",
"crying": "😭",
"loving": "😍",
"embarrassed": "😳",
"surprised": "😲",
"shocked": "😱",
"thinking": "🤔",
"winking": "😉",
"cool": "😎",
"relaxed": "😌",
"delicious": "🤤",
"kissy": "😘",
"confident": "😏",
"sleepy": "😴",
"silly": "😜",
"confused": "🙄",
}
def get_local_ip():