update:新增SLM唤醒词回复

main/xiaozhi-server/core/connection.py

@@ -134,6 +134,7 @@ class ConnectionHandler:
         self._asr = _asr
         self._vad = _vad
         self.llm = _llm
+        self.slm = None  # 小参数模型，用于唤醒词等轻量级任务
         self.memory = _memory
         self.intent = _intent
 
@@ -480,10 +481,12 @@ class ConnectionHandler:
         try:
             if self.tts is None:
                 self.tts = self._initialize_tts()
-            # 打开语音合成通道
-            asyncio.run_coroutine_threadsafe(
-                self.tts.open_audio_channels(self), self.loop
-            )
+            # 打开语音合成通道（无论 TTS 是谁创建的都需要打开）
+            tts_thread = getattr(self.tts, 'tts_priority_thread', None)
+            if self.tts is not None and (tts_thread is None or not tts_thread.is_alive()):
+                asyncio.run_coroutine_threadsafe(
+                    self.tts.open_audio_channels(self), self.loop
+                )
             if self.need_bind:
                 self.bind_completed_event.set()
                 return
@@ -493,7 +496,7 @@ class ConnectionHandler:
             self.logger = create_connection_logger(self.selected_module_str)
 
             """初始化组件"""
-            if self.config.get("prompt") is not None:
+            if self.config.get("prompt") is not None and self.prompt is None:
                 user_prompt = self.config["prompt"]
                 # 使用快速提示词进行初始化
                 prompt = self.prompt_manager.get_quick_prompt(user_prompt)
@@ -735,6 +738,10 @@ class ConnectionHandler:
             self.config["selected_module"]["VLLM"] = private_config["selected_module"][
                 "VLLM"
             ]
+        # SLM 配置：selected_module.SLM 指向 LLM 节点下的某个模型 ID
+        slm_model_id = private_config.get("selected_module", {}).get("SLM")
+        if slm_model_id:
+            self.config["selected_module"]["SLM"] = slm_model_id
         if private_config.get("Memory", None) is not None:
             init_memory = True
             self.config["Memory"] = private_config["Memory"]
@@ -899,6 +906,31 @@ class ConnectionHandler:
         if hasattr(self, "loop") and self.loop:
             asyncio.run_coroutine_threadsafe(self.func_handler._initialize(), self.loop)
 
+    def _initialize_slm(self):
+        """初始化小参数模型(SLM)，用于唤醒词等轻量级任务"""
+        slm_model_id = self.config.get("selected_module", {}).get("SLM")
+        if not slm_model_id:
+            self.logger.bind(tag=TAG).info("未配置SLM模型，唤醒词回复将使用固定短语")
+            return
+
+        llm_configs = self.config.get("LLM", {})
+        slm_config = llm_configs.get(slm_model_id)
+        if not slm_config:
+            self.logger.bind(tag=TAG).warning(f"SLM模型配置未找到: {slm_model_id}")
+            return
+
+        try:
+            from core.utils import llm as llm_utils
+
+            slm_type = slm_config.get("type", slm_model_id)
+            self.slm = llm_utils.create_instance(slm_type, slm_config)
+            self.logger.bind(tag=TAG).info(
+                f"初始化SLM模型成功: {slm_model_id}, 类型: {slm_type}"
+            )
+        except Exception as e:
+            self.logger.bind(tag=TAG).error(f"初始化SLM模型失败: {e}")
+            self.slm = None
+
     def change_system_prompt(self, prompt):
         self.prompt = prompt
         # 更新系统prompt至上下文

main/xiaozhi-server/core/handle/helloHandle.py

@@ -3,18 +3,38 @@ import json
 import uuid
 import random
 import asyncio
+
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from core.connection import ConnectionHandler
 from core.utils.dialogue import Message
 from core.utils.util import audio_to_data
-from core.providers.tts.dto.dto import SentenceType
+from core.providers.tts.dto.dto import SentenceType, ContentType, TTSMessageDTO
 from core.utils.wakeup_word import WakeupWordsConfig
 from core.handle.sendAudioHandle import sendAudioMessage, send_tts_message
 from core.utils.util import remove_punctuation_and_length, opus_datas_to_wav_bytes
 from core.providers.tools.device_mcp import MCPClient, send_mcp_initialize_message
 
+# 预加载 LLM 工具模块和提供者模块
+# 避免唤醒词快速路径初始化 SLM 时，与后台 initialize_modules 产生循环导入冲突
+import importlib as _importlib
+import sys as _sys
+import core.utils.llm  # noqa: F401 — 触发模块注册到 sys.modules
+# 遍历所有 LLM 提供者目录，预加载到 sys.modules
+import os as _os
+_llm_dir = _os.path.join(_os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))), "providers", "llm")
+if _os.path.isdir(_llm_dir):
+    for _name in _os.listdir(_llm_dir):
+        _mod_path = _os.path.join(_llm_dir, _name, f"{_name}.py")
+        if _os.path.isfile(_mod_path):
+            _fq = f"core.providers.llm.{_name}.{_name}"
+            if _fq not in _sys.modules:
+                try:
+                    _sys.modules[_fq] = _importlib.import_module(_fq)
+                except Exception:
+                    pass
+
 TAG = __name__
 
 WAKEUP_CONFIG = {
@@ -32,6 +52,17 @@ WAKEUP_CONFIG = {
     ],
 }
 
+# SLM 唤醒词回复的系统提示词模板
+# 从 agent-base-prompt.txt 精简而来，保留角色风格和语言规则，剔除工具/TTS格式等无关内容
+SLM_WAKEUP_SYSTEM_PROMPT = """用户跟你打招呼了，请用{language}回应，简短自然。
+{identity}
+规则：
+- 说一两句话表示你在，风格随意，像朋友之间，不要太长，20字以内
+- 不要反问、不要追问、不要展开话题
+- 不要用"烦心事"、"有趣的事"、"好玩的事"、"综上所述"
+- 不要念出给你的编号
+回复："""
+
 # 创建全局的唤醒词配置管理器
 wakeup_words_config = WakeupWordsConfig()
 
@@ -60,31 +91,184 @@ async def handleHelloMessage(conn: "ConnectionHandler", msg_json):
     await conn.websocket.send(json.dumps(conn.welcome_msg))
 
 
+def _build_slm_prompt(conn: "ConnectionHandler") -> str:
+    """构建 SLM 唤醒词回复的系统提示词"""
+    # 提取用户智能体的人设
+    identity = ""
+    user_prompt = conn.config.get("prompt", "")
+    if user_prompt:
+        identity = f"<identity>\n{user_prompt}\n</identity>"
+
+    # 获取语言配置，与大模型保持一致
+    language = (
+        conn.config.get("TTS", {})
+        .get(conn.config.get("selected_module", {}).get("TTS", ""), {})
+        .get("language")
+        or "中文"
+    )
+
+    return SLM_WAKEUP_SYSTEM_PROMPT.format(
+        language=language,
+        identity=identity,
+    )
+
+
+async def _generate_slm_response(conn: "ConnectionHandler") -> str | None:
+    """调用 SLM 生成唤醒词回复，失败返回 None"""
+    if not conn.slm:
+        return None
+
+    try:
+        system_prompt = _build_slm_prompt(conn)
+        user_msg = random.choice([
+            "用户来了，打个招呼吧，自然点。",
+            "嘿，用户在呢，随意应一下就行。",
+            "用户喊你啦，用你自己的方式回应。",
+            "叮咚，用户找你，简单回应一下。",
+            "用户跟你打招呼了，随意回一句。",
+            "有人叫你名字，应一声吧。",
+            "用户跟你说话了，简单回一下。",
+            "打个招呼吧，用你的风格。",
+            "用户在你旁边，说句话吧。",
+            "随便打个招呼，别太正式。",
+        ])
+        result = await asyncio.wait_for(
+            asyncio.to_thread(
+                conn.slm.response_no_stream,
+                system_prompt,
+                user_msg,
+                max_tokens=25,
+                temperature=0.9,
+            ),
+            timeout=5.0,
+        )
+        if result and len(result.strip()) > 0:
+            conn.logger.bind(tag=TAG).info(f"SLM生成唤醒词回复: {result.strip()}")
+            return result.strip()
+        return None
+    except asyncio.TimeoutError:
+        conn.logger.bind(tag=TAG).warning("SLM生成唤醒词回复超时，回退到固定短语")
+        return None
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"SLM生成唤醒词回复失败: {e}")
+        return None
+
+
+async def _send_wakeup_response(conn: "ConnectionHandler", response_text: str):
+    """将回复文本通过流式 TTS 队列发送给设备"""
+    conn.client_abort = False
+    sentence_id = str(uuid.uuid4().hex)
+    conn.sentence_id = sentence_id
+
+    conn.logger.bind(tag=TAG).info(f"播放唤醒词回复: {response_text}")
+
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=sentence_id,
+            sentence_type=SentenceType.FIRST,
+            content_type=ContentType.ACTION,
+        )
+    )
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=sentence_id,
+            sentence_type=SentenceType.MIDDLE,
+            content_type=ContentType.TEXT,
+            content_detail=response_text,
+        )
+    )
+    conn.tts.tts_text_queue.put(
+        TTSMessageDTO(
+            sentence_id=sentence_id,
+            sentence_type=SentenceType.LAST,
+            content_type=ContentType.ACTION,
+        )
+    )
+
+    conn.dialogue.put(Message(role="assistant", content=response_text))
+
+
+async def _wakeup_ensure_prompt(conn: "ConnectionHandler"):
+    """唤醒词快速路径：初始化提示词（SLM 生成回复依赖 prompt）"""
+    if conn.prompt is not None:
+        return
+    user_prompt = conn.config.get("prompt")
+    if user_prompt:
+        quick_prompt = conn.prompt_manager.get_quick_prompt(user_prompt)
+        conn.change_system_prompt(quick_prompt)
+
+
+async def _wakeup_ensure_slm(conn: "ConnectionHandler"):
+    """唤醒词快速路径：按需初始化 SLM"""
+    if conn.slm is not None:
+        return
+    try:
+        await asyncio.get_running_loop().run_in_executor(None, conn._initialize_slm)
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"唤醒词快速初始化SLM失败: {e}")
+
+
+async def _wakeup_ensure_tts(conn: "ConnectionHandler"):
+    """唤醒词快速路径：按需初始化 TTS + 音频通道"""
+    # 检查 TTS 实例是否已创建且音频通道已打开
+    if conn.tts is not None:
+        tts_thread = getattr(conn.tts, 'tts_priority_thread', None)
+        if tts_thread is not None and tts_thread.is_alive():
+            return  # TTS 已就绪，无需重复初始化
+
+    def _do_init():
+        if conn.tts is None:
+            tts_instance = conn._initialize_tts()
+            conn.tts = tts_instance
+        # 打开音频通道（即使 TTS 已存在但通道未打开）
+        if conn.tts is not None:
+            tts_thread = getattr(conn.tts, 'tts_priority_thread', None)
+            if tts_thread is None or not tts_thread.is_alive():
+                future = asyncio.run_coroutine_threadsafe(
+                    conn.tts.open_audio_channels(conn), conn.loop
+                )
+                future.result(timeout=5)  # 等待音频通道打开完成
+
+    try:
+        await asyncio.get_running_loop().run_in_executor(None, _do_init)
+    except Exception as e:
+        conn.logger.bind(tag=TAG).error(f"唤醒词快速初始化TTS失败: {e}")
+
+
 async def checkWakeupWords(conn: "ConnectionHandler", text):
     enable_wakeup_words_response_cache = conn.config[
         "enable_wakeup_words_response_cache"
     ]
 
-    # 等待tts初始化，最多等待3秒
-    start_time = time.time()
-    while time.time() - start_time < 3:
-        if conn.tts:
-            break
-        await asyncio.sleep(0.1)
-    else:
-        return False
-
-    if not enable_wakeup_words_response_cache:
-        return False
-
     _, filtered_text = remove_punctuation_and_length(text)
     if filtered_text not in conn.config.get("wakeup_words"):
         return False
 
     conn.just_woken_up = True
-    await send_tts_message(conn, "start")
 
-    # 获取当前音色
+    # 唤醒词快速路径：按需初始化 提示词 → SLM → TTS
+    # 后台 _initialize_components 会跳过已初始化的组件，不会冲突
+    await _wakeup_ensure_prompt(conn)
+    await _wakeup_ensure_slm(conn)
+    await _wakeup_ensure_tts(conn)
+
+    if conn.tts is None:
+        return False
+
+    # SLM 路径：生成个性化回复
+    if conn.slm:
+        slm_response = await _generate_slm_response(conn)
+        if slm_response:
+            await send_tts_message(conn, "start")
+            await _send_wakeup_response(conn, slm_response)
+            return True
+        # SLM 生成失败，回退到固定短语缓存逻辑
+        conn.logger.bind(tag=TAG).info("SLM不可用，回退到固定短语唤醒词回复")
+
+    if not enable_wakeup_words_response_cache:
+        return False
+
+    # 固定短语缓存路径（无 SLM 或 SLM 失败时的降级逻辑）
     voice = getattr(conn.tts, "voice", "default")
     if not voice:
         voice = "default"