diff --git a/docs/fish-speech-integration.md b/docs/fish-speech-integration.md new file mode 100644 index 00000000..b916e619 --- /dev/null +++ b/docs/fish-speech-integration.md @@ -0,0 +1,72 @@ +登录AutoDL，租赁镜像 +选择镜像: +``` +PyTorch / 2.1.0 / 3.10(ubuntu22.04) / cuda 12.1 +``` + +机器开机后，设置学术加速 +``` +source /etc/network_turbo +``` + +进入工作目录 +``` +cd autodl-tmp/ +``` + +拉取项目 +``` +git clone https://gitclone.com/github.com/fishaudio/fish-speech.git ; cd fish-speech +``` + +安装依赖 +``` +pip install -e. +``` + +如果报错，安装portaudio +``` +apt-get install portaudio19-dev -y +``` + +安装后执行 +``` +pip install torch==2.3.1 torchvision==0.18.1 torchaudio==2.3.1 --index-url https://download.pytorch.org/whl/cu121 +``` + +下载模型 +``` +cd tools +python download_models.py +``` + +下载完模型后运行接口 +``` +python -m tools.api_server --listen 0.0.0.0:6006 +``` + +然后用浏览器去到aotodl实例页面 +``` +https://autodl.com/console/instance/list +``` + +如下图点击你刚才机器的`自定义服务`按钮，开启端口转发服务 +![自定义服务](images/fishspeech/autodl-01.png) + +端口转发服务设置完成后，你本地电脑打开网址`http://localhost:6006/`，就可以访问fish-speech的接口了 +![服务预览](images/fishspeech/autodl-02.png) + + +如果你是单模块部署，核心配置如下 +``` +selected_module: + TTS: FishSpeech +TTS: + FishSpeech: + reference_audio: ["config/assets/wakeup_words.wav",] + reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",] + api_key: "123" + api_url: "http://127.0.0.1:6006/v1/tts" +``` + +然后重启服务 \ No newline at end of file diff --git a/docs/images/fishspeech/autodl-01.png b/docs/images/fishspeech/autodl-01.png new file mode 100644 index 00000000..695b94b8 Binary files /dev/null and b/docs/images/fishspeech/autodl-01.png differ diff --git a/docs/images/fishspeech/autodl-02.png b/docs/images/fishspeech/autodl-02.png new file mode 100644 index 00000000..e3f72afe Binary files /dev/null and b/docs/images/fishspeech/autodl-02.png differ diff --git a/main/manager-api/src/main/resources/db/changelog/202505091554.sql b/main/manager-api/src/main/resources/db/changelog/202505091555.sql similarity index 79% rename from main/manager-api/src/main/resources/db/changelog/202505091554.sql rename to main/manager-api/src/main/resources/db/changelog/202505091555.sql index 87d879dd..b7ad91d4 100644 --- a/main/manager-api/src/main/resources/db/changelog/202505091554.sql +++ b/main/manager-api/src/main/resources/db/changelog/202505091555.sql @@ -12,4 +12,14 @@ config_json = '{"host": "127.0.0.1", "port": 10096, "type": "fun_server", "is_ss 不要退出容器，继续在容器中执行第四句：nohup bash run_server_2pass.sh --download-model-dir /workspace/models --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx --model-dir damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx --punc-dir damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst --itn-dir thuduj12/fst_itn_zh --hotword /workspace/models/hotwords.txt > log.txt 2>&1 & 上一句话执行后会进入到容器，继续第五句：tail -f log.txt 第五句话执行完后，会看到模型下载日志，下载完后就可以连接使用了 -以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer'; \ No newline at end of file +以上是使用CPU推理，如果有GPU，详细参考：https://github.com/modelscope/FunASR/blob/main/runtime/docs/SDK_advanced_guide_online_zh.md' WHERE `id` = 'ASR_FunASRServer'; + +-- FishSpeech配置说明 +UPDATE `ai_model_config` SET +`doc_link` = 'https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md', +`remark` = 'FishSpeech配置说明： +1. 需要本地部署FishSpeech服务 +2. 支持自定义音色 +3. 本地推理，无需网络连接 +4. 输出文件保存在tmp/目录 +5. 可参照教程https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md' WHERE `id` = 'TTS_FishSpeech'; diff --git a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml index 2d5147c6..2dfd9831 100755 --- a/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/main/manager-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -115,12 +115,12 @@ databaseChangeLog: encoding: utf8 path: classpath:db/changelog/202505091409.sql - changeSet: - id: 202505091554 + id: 202505091555 author: whosmyqueen changes: - sqlFile: encoding: utf8 - path: classpath:db/changelog/202505091554.sql + path: classpath:db/changelog/202505091555.sql - changeSet: id: 202505111914 author: hrz diff --git a/main/xiaozhi-server/config.yaml b/main/xiaozhi-server/config.yaml index 7a5918cf..bc06f983 100644 --- a/main/xiaozhi-server/config.yaml +++ b/main/xiaozhi-server/config.yaml @@ -476,20 +476,13 @@ TTS: speed: 1 output_dir: tmp/ FishSpeech: - # 定义TTS API类型 - #启动tts方法： - #python -m tools.api_server - #--listen 0.0.0.0:8080 - #--llama-checkpoint-path "checkpoints/fish-speech-1.5" - #--decoder-checkpoint-path "checkpoints/fish-speech-1.5/firefly-gan-vq-fsq-8x1024-21hz-generator.pth" - #--decoder-config-name firefly_gan_vq - #--compile + # 参照教程：https://github.com/xinnan-tech/xiaozhi-esp32-server/blob/main/docs/fish-speech-integration.md type: fishspeech output_dir: tmp/ response_format: wav reference_id: null - reference_audio: ["/tmp/test.wav",] - reference_text: ["你弄来这些吟词宴曲来看，还是这些混话来欺负我。",] + reference_audio: ["config/assets/wakeup_words.wav",] + reference_text: ["哈啰啊，我是小智啦，声音好听的台湾女孩一枚，超开心认识你耶，最近在忙啥，别忘了给我来点有趣的料哦，我超爱听八卦的啦",] normalize: true max_new_tokens: 1024 chunk_length: 200 diff --git a/main/xiaozhi-server/core/providers/tts/fishspeech.py b/main/xiaozhi-server/core/providers/tts/fishspeech.py index 687e27ef..bf195101 100644 --- a/main/xiaozhi-server/core/providers/tts/fishspeech.py +++ b/main/xiaozhi-server/core/providers/tts/fishspeech.py @@ -85,7 +85,9 @@ class TTSProvider(TTSProviderBase): def __init__(self, config, delete_audio_file): super().__init__(config, delete_audio_file) - self.reference_id = config.get("reference_id") + self.reference_id = ( + None if not config.get("reference_id") else config.get("reference_id") + ) self.reference_audio = parse_string_to_list(config.get("reference_audio")) self.reference_text = parse_string_to_list(config.get("reference_text")) self.format = config.get("response_format", "wav") diff --git a/main/xiaozhi-server/core/utils/util.py b/main/xiaozhi-server/core/utils/util.py index effbf38e..3965a24f 100644 --- a/main/xiaozhi-server/core/utils/util.py +++ b/main/xiaozhi-server/core/utils/util.py @@ -930,7 +930,6 @@ def check_vad_update(before_config, new_config): if "type" not in new_config["VAD"][new_vad_module] else new_config["VAD"][new_vad_module]["type"] ) - print(f"前vad:{current_vad_type}，后vad:{new_vad_type}") update_vad = current_vad_type != new_vad_type return update_vad @@ -954,6 +953,5 @@ def check_asr_update(before_config, new_config): if "type" not in new_config["ASR"][new_asr_module] else new_config["ASR"][new_asr_module]["type"] ) - print(f"前asr:{current_asr_type}，后asr:{new_asr_type}") update_asr = current_asr_type != new_asr_type return update_asr