From e2935a59795365b606c2c914ae92d10698a1681e Mon Sep 17 00:00:00 2001 From: Xiaoxia Date: Wed, 15 Jul 2026 04:39:56 +0800 Subject: [PATCH] refactor(audio): unify speech processing engines --- main/CMakeLists.txt | 13 +- main/Kconfig.projbuild | 6 +- main/audio/README.md | 120 ++-- main/audio/{processors => }/audio_debugger.cc | 19 +- main/audio/{processors => }/audio_debugger.h | 6 +- main/audio/audio_engine.h | 40 ++ main/audio/audio_processor.h | 26 - main/audio/audio_service.cc | 147 +++-- main/audio/audio_service.h | 23 +- main/audio/engines/afe_audio_engine.cc | 521 ++++++++++++++++++ main/audio/engines/afe_audio_engine.h | 98 ++++ main/audio/engines/lite_audio_engine.cc | 150 +++++ main/audio/engines/lite_audio_engine.h | 58 ++ main/audio/processors/afe_audio_processor.cc | 202 ------- main/audio/processors/afe_audio_processor.h | 48 -- main/audio/processors/no_audio_processor.cc | 71 --- main/audio/processors/no_audio_processor.h | 35 -- main/audio/wake_words/afe_wake_word.cc | 263 --------- main/audio/wake_words/afe_wake_word.h | 62 --- main/audio/wake_words/custom_wake_word.cc | 95 ++-- main/audio/wake_words/custom_wake_word.h | 7 +- main/audio/wake_words/esp_wake_word.cc | 25 +- main/audio/wake_words/esp_wake_word.h | 1 + .../audio/wake_words/wake_word_audio_cache.cc | 98 ++++ main/audio/wake_words/wake_word_audio_cache.h | 30 + 25 files changed, 1218 insertions(+), 946 deletions(-) rename main/audio/{processors => }/audio_debugger.cc (91%) rename main/audio/{processors => }/audio_debugger.h (97%) create mode 100644 main/audio/audio_engine.h delete mode 100644 main/audio/audio_processor.h create mode 100644 main/audio/engines/afe_audio_engine.cc create mode 100644 main/audio/engines/afe_audio_engine.h create mode 100644 main/audio/engines/lite_audio_engine.cc create mode 100644 main/audio/engines/lite_audio_engine.h delete mode 100644 main/audio/processors/afe_audio_processor.cc delete mode 100644 main/audio/processors/afe_audio_processor.h delete mode 100644 main/audio/processors/no_audio_processor.cc delete mode 100644 main/audio/processors/no_audio_processor.h delete mode 100644 main/audio/wake_words/afe_wake_word.cc delete mode 100644 main/audio/wake_words/afe_wake_word.h create mode 100644 main/audio/wake_words/wake_word_audio_cache.cc create mode 100644 main/audio/wake_words/wake_word_audio_cache.h diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index ca597713..d5c90782 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,5 +1,6 @@ # Define source files set(SOURCES "audio/audio_codec.cc" + "audio/audio_debugger.cc" "audio/audio_service.cc" "audio/demuxer/ogg_demuxer.cc" "audio/codecs/no_audio_codec.cc" @@ -9,7 +10,6 @@ set(SOURCES "audio/audio_codec.cc" "audio/codecs/es8388_audio_codec.cc" "audio/codecs/es8389_audio_codec.cc" "audio/codecs/dummy_audio_codec.cc" - "audio/processors/audio_debugger.cc" "led/single_led.cc" "led/circular_strip.cc" "led/gpio_led.cc" @@ -885,16 +885,13 @@ else() endif() list(APPEND SOURCES ${BOARD_SOURCES}) -# Select audio processor according to Kconfig -if(CONFIG_USE_AUDIO_PROCESSOR) - list(APPEND SOURCES "audio/processors/afe_audio_processor.cc") -else() - list(APPEND SOURCES "audio/processors/no_audio_processor.cc") -endif() +# Select one audio engine for each chip family if(CONFIG_IDF_TARGET_ESP32S3 OR CONFIG_IDF_TARGET_ESP32P4) - list(APPEND SOURCES "audio/wake_words/afe_wake_word.cc") + list(APPEND SOURCES "audio/engines/afe_audio_engine.cc") list(APPEND SOURCES "audio/wake_words/custom_wake_word.cc") + list(APPEND SOURCES "audio/wake_words/wake_word_audio_cache.cc") else() + list(APPEND SOURCES "audio/engines/lite_audio_engine.cc") list(APPEND SOURCES "audio/wake_words/esp_wake_word.cc") endif() diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index f9cfa02b..fff7b9b7 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -897,11 +897,13 @@ config WAKE_WORD_DETECTION_IN_LISTENING When disabled (default), wake word detection is turned off during listening. config USE_AUDIO_PROCESSOR - bool "Enable Audio Noise Reduction" + bool "Enable AFE Audio Processing" default y depends on (IDF_TARGET_ESP32S3 || IDF_TARGET_ESP32P4) && SPIRAM help - Requires ESP32 S3 and PSRAM + Enable the shared AFE uplink path with AEC and VAD. Noise suppression + is not enabled because this project does not provide an NSNet model. + Requires ESP32-S3 or ESP32-P4 with PSRAM. config USE_DEVICE_AEC bool "Enable Device-Side AEC" diff --git a/main/audio/README.md b/main/audio/README.md index 6159f9ad..cca987f8 100644 --- a/main/audio/README.md +++ b/main/audio/README.md @@ -1,88 +1,72 @@ # Audio Service Architecture -The audio service is a core component responsible for managing all audio-related functionalities, including capturing audio from the microphone, processing it, encoding/decoding, and playing back audio through the speaker. It is designed to be modular and efficient, running its main operations in dedicated FreeRTOS tasks to ensure real-time performance. +`AudioService` owns one input engine and keeps codec I/O, Opus encoding/decoding, +and network queues independent from the chip-specific speech pipeline. -## Key Components +## Input engines -- **`AudioService`**: The central orchestrator. It initializes and manages all other audio components, tasks, and data queues. -- **`AudioCodec`**: A hardware abstraction layer (HAL) for the physical audio codec chip. It handles the raw I2S communication for audio input and output. -- **`AudioProcessor`**: Performs real-time audio processing on the microphone input stream. This typically includes Acoustic Echo Cancellation (AEC), noise suppression, and Voice Activity Detection (VAD). `AfeAudioProcessor` is the default implementation, utilizing the ESP-ADF Audio Front-End. -- **`WakeWord`**: Detects keywords (e.g., "你好，小智", "Hi, ESP") from the audio stream. It runs independently from the main audio processor until a wake word is detected. -- **`OpusEncoderWrapper` / `OpusDecoderWrapper`**: Manages the encoding of PCM audio to the Opus format and decoding Opus packets back to PCM. Opus is used for its high compression and low latency, making it ideal for voice streaming. -- **`OpusResampler`**: A utility to convert audio streams between different sample rates (e.g., resampling from the codec's native sample rate to the required 16kHz for processing). +The old `AudioProcessor + WakeWord` combinations have been replaced by a single +`AudioEngine` interface. `AudioInputTask` reads PCM once and feeds exactly one +engine. -## Threading Model +| Target | Engine | Wake word | Uplink processing | +| --- | --- | --- | --- | +| ESP32-S3 / ESP32-P4 | `AfeAudioEngine` | WakeNet inside AFE, or MultiNet fed from AFE output | FD AEC + VAD when audio processing is enabled | +| ESP32 / ESP32-C3 / ESP32-C5 / ESP32-C6 | `LiteAudioEngine` | Standalone WakeNet when configured | Raw mono PCM | -The service operates on three primary tasks to handle the different stages of the audio pipeline concurrently: +`AfeAudioEngine` owns a single FD AFE instance. WakeNet and voice uplink share +that instance, so enabling both no longer creates two AFE pipelines. For custom +MultiNet wake words, AFE fetch output is passed to `CustomWakeWord`; MultiNet is +not created on the smaller targets. -1. **`AudioInputTask`**: Solely responsible for reading raw PCM data from the `AudioCodec`. It then feeds this data to either the `WakeWord` engine or the `AudioProcessor` based on the current state. -2. **`AudioOutputTask`**: Responsible for playing audio. It retrieves decoded PCM data from the `audio_playback_queue_` and sends it to the `AudioCodec` to be played on the speaker. -3. **`OpusCodecTask`**: A worker task that handles both encoding and decoding. It fetches raw audio from `audio_encode_queue_`, encodes it into Opus packets, and places them in the `audio_send_queue_`. Concurrently, it fetches Opus packets from `audio_decode_queue_`, decodes them into PCM, and places the result in the `audio_playback_queue_`. +The AFE configuration currently uses `FD_LOW_COST` AEC with +`AEC_NLP_LEVEL_VERYAGGR`. WebRTC/NSNet noise suppression is intentionally +disabled because the project does not ship an NSNet model. -## Data Flow +When wake-word audio upload is enabled, the most recent two seconds of PCM are +stored in a single 64 KB PSRAM ring buffer. WakeNet and MultiNet share the same +cache implementation, and the encoder reads it one Opus frame at a time. This +avoids the previous per-chunk internal-SRAM allocations and temporary PCM +concatenation buffer. -There are two primary data flows: audio input (uplink) and audio output (downlink). - -### 1. Audio Input (Uplink) Flow - -This flow captures audio from the microphone, processes it, encodes it, and prepares it for sending to a server. +## Input data flow ```mermaid -graph TD - subgraph Device - Mic[("Microphone")] -->|I2S| Codec(AudioCodec) - - subgraph AudioInputTask - Codec -->|Raw PCM| Read(ReadAudioData) - Read -->|16kHz PCM| Processor(AudioProcessor) - end - - subgraph OpusCodecTask - Processor -->|Clean PCM| EncodeQueue(audio_encode_queue_) - EncodeQueue --> Encoder(OpusEncoder) - Encoder -->|Opus Packet| SendQueue(audio_send_queue_) - end - - SendQueue --> |"PopPacketFromSendQueue()"| App(Application Layer) - end - - App -->|Network| Server((Cloud Server)) +flowchart LR + Mic[Microphone] --> Codec[AudioCodec] + Codec --> Input[AudioInputTask] + Input --> Engine[One AudioEngine] + Engine --> Wake[Wake-word event] + Engine --> PCM[16 kHz mono PCM] + PCM --> EncodeQueue[audio_encode_queue_] + EncodeQueue --> Opus[OpusCodecTask] + Opus --> SendQueue[audio_send_queue_] + SendQueue --> App[Application / network] ``` -- The `AudioInputTask` continuously reads raw PCM data from the `AudioCodec`. -- This data is fed into an `AudioProcessor` for cleaning (AEC, VAD). -- The processed PCM data is pushed into the `audio_encode_queue_`. -- The `OpusCodecTask` picks up the PCM data, encodes it into Opus format, and pushes the resulting packet to the `audio_send_queue_`. -- The application can then retrieve these Opus packets and send them over the network. +Wake-word detection and voice processing are independent runtime states on the +same engine. On AFE targets, AEC stays active while wake-word detection is active +so playback reference remains available for wake-up during device playback. It +also stays active during voice processing when device AEC is requested. -### 2. Audio Output (Downlink) Flow - -This flow receives encoded audio data, decodes it, and plays it on the speaker. +## Output data flow ```mermaid -graph TD - Server((Cloud Server)) -->|Network| App(Application Layer) - - subgraph Device - App -->|"PushPacketToDecodeQueue()"| DecodeQueue(audio_decode_queue_) - - subgraph OpusCodecTask - DecodeQueue -->|Opus Packet| Decoder(OpusDecoder) - Decoder -->|PCM| PlaybackQueue(audio_playback_queue_) - end - - subgraph AudioOutputTask - PlaybackQueue -->|PCM| Codec(AudioCodec) - end - - Codec -->|I2S| Speaker[("Speaker")] - end +flowchart LR + App[Application / network] --> DecodeQueue[audio_decode_queue_] + DecodeQueue --> Opus[OpusCodecTask] + Opus --> PlaybackQueue[audio_playback_queue_] + PlaybackQueue --> Output[AudioOutputTask] + Output --> Codec[AudioCodec] + Codec --> Speaker[Speaker] ``` -- The application receives Opus packets from the network and pushes them into the `audio_decode_queue_`. -- The `OpusCodecTask` retrieves these packets, decodes them back into PCM data, and pushes the data to the `audio_playback_queue_`. -- The `AudioOutputTask` takes the PCM data from the queue and sends it to the `AudioCodec` for playback. +## Tasks and power management -## Power Management +- `AudioInputTask` reads codec input and feeds the selected engine. +- `AudioOutputTask` drains decoded PCM to the codec output. +- `OpusCodecTask` encodes uplink PCM and decodes downlink packets. +- `AfeAudioEngine` has its own AFE fetch task on S3/P4. -To conserve energy, the audio codec's input (ADC) and output (DAC) channels are automatically disabled after a period of inactivity (`AUDIO_POWER_TIMEOUT_MS`). A timer (`audio_power_timer_`) periodically checks for activity and manages the power state. The channels are automatically re-enabled when new audio needs to be captured or played. \ No newline at end of file +The audio power timer still enables and disables codec ADC/DAC channels based on +activity; the engine refactor does not change that policy. diff --git a/main/audio/processors/audio_debugger.cc b/main/audio/audio_debugger.cc similarity index 91% rename from main/audio/processors/audio_debugger.cc rename to main/audio/audio_debugger.cc index 630057cf..c38f88b5 100644 --- a/main/audio/processors/audio_debugger.cc +++ b/main/audio/audio_debugger.cc @@ -1,35 +1,34 @@ #include "audio_debugger.h" + #include "sdkconfig.h" #if CONFIG_USE_AUDIO_DEBUGGER -#include #include -#include -#include #include +#include +#include #include +#include #endif #define TAG "AudioDebugger" - AudioDebugger::AudioDebugger() { #if CONFIG_USE_AUDIO_DEBUGGER udp_sockfd_ = socket(AF_INET, SOCK_DGRAM, 0); if (udp_sockfd_ >= 0) { - // 解析配置的服务器地址 "IP:PORT" std::string server_addr = CONFIG_AUDIO_DEBUG_UDP_SERVER; size_t colon_pos = server_addr.find(':'); - + if (colon_pos != std::string::npos) { std::string ip = server_addr.substr(0, colon_pos); int port = std::stoi(server_addr.substr(colon_pos + 1)); - + memset(&udp_server_addr_, 0, sizeof(udp_server_addr_)); udp_server_addr_.sin_family = AF_INET; udp_server_addr_.sin_port = htons(port); inet_pton(AF_INET, ip.c_str(), &udp_server_addr_.sin_addr); - + ESP_LOGI(TAG, "Initialized server address: %s", CONFIG_AUDIO_DEBUG_UDP_SERVER); } else { ESP_LOGW(TAG, "Invalid server address: %s, should be IP:PORT", CONFIG_AUDIO_DEBUG_UDP_SERVER); @@ -55,7 +54,7 @@ void AudioDebugger::Feed(const std::vector& data) { #if CONFIG_USE_AUDIO_DEBUGGER if (udp_sockfd_ >= 0) { ssize_t sent = sendto(udp_sockfd_, data.data(), data.size() * sizeof(int16_t), 0, - (struct sockaddr*)&udp_server_addr_, sizeof(udp_server_addr_)); + reinterpret_cast(&udp_server_addr_), sizeof(udp_server_addr_)); if (sent < 0) { ESP_LOGW(TAG, "Failed to send audio data to %s: %d", CONFIG_AUDIO_DEBUG_UDP_SERVER, errno); } else { @@ -64,5 +63,3 @@ void AudioDebugger::Feed(const std::vector& data) { } #endif } - - \ No newline at end of file diff --git a/main/audio/processors/audio_debugger.h b/main/audio/audio_debugger.h similarity index 97% rename from main/audio/processors/audio_debugger.h rename to main/audio/audio_debugger.h index a81336cb..a45364e9 100644 --- a/main/audio/processors/audio_debugger.h +++ b/main/audio/audio_debugger.h @@ -1,11 +1,11 @@ #ifndef AUDIO_DEBUGGER_H #define AUDIO_DEBUGGER_H -#include #include +#include -#include #include +#include class AudioDebugger { public: @@ -19,4 +19,4 @@ private: struct sockaddr_in udp_server_addr_; }; -#endif \ No newline at end of file +#endif diff --git a/main/audio/audio_engine.h b/main/audio/audio_engine.h new file mode 100644 index 00000000..7c2e191a --- /dev/null +++ b/main/audio/audio_engine.h @@ -0,0 +1,40 @@ +#ifndef AUDIO_ENGINE_H +#define AUDIO_ENGINE_H + +#include +#include +#include +#include +#include + +#include + +#include "audio_codec.h" + +class AudioEngine { +public: + virtual ~AudioEngine() = default; + + virtual bool Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) = 0; + virtual void Feed(std::vector&& data) = 0; + + virtual void EnableWakeWordDetection(bool enable) = 0; + virtual void EnableVoiceProcessing(bool enable) = 0; + virtual void EnableDeviceAec(bool enable) = 0; + + virtual bool HasWakeWord() const = 0; + virtual bool IsWakeWordDetectionEnabled() const = 0; + virtual bool IsVoiceProcessingEnabled() const = 0; + virtual bool IsAfeWakeWord() const = 0; + virtual size_t GetFeedSize() const = 0; + + virtual void OnWakeWordDetected(std::function callback) = 0; + virtual void OnOutput(std::function&& data)> callback) = 0; + virtual void OnVadStateChange(std::function callback) = 0; + + virtual void EncodeWakeWordData() = 0; + virtual bool GetWakeWordOpus(std::vector& opus) = 0; + virtual const std::string& GetLastDetectedWakeWord() const = 0; +}; + +#endif diff --git a/main/audio/audio_processor.h b/main/audio/audio_processor.h deleted file mode 100644 index 543c7ae0..00000000 --- a/main/audio/audio_processor.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef AUDIO_PROCESSOR_H -#define AUDIO_PROCESSOR_H - -#include -#include -#include - -#include -#include "audio_codec.h" - -class AudioProcessor { -public: - virtual ~AudioProcessor() = default; - - virtual void Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) = 0; - virtual void Feed(std::vector&& data) = 0; - virtual void Start() = 0; - virtual void Stop() = 0; - virtual bool IsRunning() = 0; - virtual void OnOutput(std::function&& data)> callback) = 0; - virtual void OnVadStateChange(std::function callback) = 0; - virtual size_t GetFeedSize() = 0; - virtual void EnableDeviceAec(bool enable) = 0; -}; - -#endif diff --git a/main/audio/audio_service.cc b/main/audio/audio_service.cc index 350ac586..24cc2727 100644 --- a/main/audio/audio_service.cc +++ b/main/audio/audio_service.cc @@ -22,17 +22,10 @@ .self_delimited = false, \ } -#if CONFIG_USE_AUDIO_PROCESSOR -#include "processors/afe_audio_processor.h" -#else -#include "processors/no_audio_processor.h" -#endif - #if CONFIG_IDF_TARGET_ESP32S3 || CONFIG_IDF_TARGET_ESP32P4 -#include "wake_words/afe_wake_word.h" -#include "wake_words/custom_wake_word.h" +#include "engines/afe_audio_engine.h" #else -#include "wake_words/esp_wake_word.h" +#include "engines/lite_audio_engine.h" #endif #define TAG "AudioService" @@ -92,22 +85,26 @@ void AudioService::Initialize(AudioCodec* codec) { } } -#if CONFIG_USE_AUDIO_PROCESSOR - audio_processor_ = std::make_unique(); +#if CONFIG_IDF_TARGET_ESP32S3 || CONFIG_IDF_TARGET_ESP32P4 + audio_engine_ = std::make_unique(); #else - audio_processor_ = std::make_unique(); + audio_engine_ = std::make_unique(); #endif - - audio_processor_->OnOutput([this](std::vector&& data) { + audio_engine_->OnOutput([this](std::vector&& data) { PushTaskToEncodeQueue(kAudioTaskTypeEncodeToSendQueue, std::move(data)); }); - - audio_processor_->OnVadStateChange([this](bool speaking) { + audio_engine_->OnVadStateChange([this](bool speaking) { voice_detected_ = speaking; if (callbacks_.on_vad_change) { callbacks_.on_vad_change(speaking); } }); + audio_engine_->OnWakeWordDetected([this](const std::string& wake_word) { + xEventGroupClearBits(event_group_, AS_EVENT_WAKE_WORD_RUNNING); + if (callbacks_.on_wake_word_detected) { + callbacks_.on_wake_word_detected(wake_word); + } + }); esp_timer_create_args_t audio_power_timer_args = { .callback = [](void* arg) { @@ -265,17 +262,12 @@ void AudioService::AudioInputTask() { } } - /* Feed the wake word and/or audio processor */ + /* Feed the selected audio engine */ if (bits & (AS_EVENT_WAKE_WORD_RUNNING | AS_EVENT_AUDIO_PROCESSOR_RUNNING)) { int samples = 160; // 10ms std::vector data; if (ReadAudioData(data, 16000, samples)) { - if (bits & AS_EVENT_WAKE_WORD_RUNNING) { - wake_word_->Feed(data); - } - if (bits & AS_EVENT_AUDIO_PROCESSOR_RUNNING) { - audio_processor_->Feed(std::move(data)); - } + audio_engine_->Feed(std::move(data)); continue; } } @@ -529,76 +521,68 @@ std::unique_ptr AudioService::PopPacketFromSendQueue() { } void AudioService::EncodeWakeWord() { - if (wake_word_) { - wake_word_->EncodeWakeWordData(); + if (audio_engine_) { + audio_engine_->EncodeWakeWordData(); } } const std::string& AudioService::GetLastWakeWord() const { - return wake_word_->GetLastDetectedWakeWord(); + static const std::string empty; + return audio_engine_ ? audio_engine_->GetLastDetectedWakeWord() : empty; } std::unique_ptr AudioService::PopWakeWordPacket() { auto packet = std::make_unique(); - if (wake_word_->GetWakeWordOpus(packet->payload)) { + if (audio_engine_ && audio_engine_->GetWakeWordOpus(packet->payload)) { return packet; } return nullptr; } void AudioService::EnableWakeWordDetection(bool enable) { - if (!wake_word_) { - return; - } - ESP_LOGD(TAG, "%s wake word detection", enable ? "Enabling" : "Disabling"); if (enable) { - if (!wake_word_initialized_) { - if (!wake_word_->Initialize(codec_, models_list_)) { - ESP_LOGE(TAG, "Failed to initialize wake word"); - return; - } - wake_word_initialized_ = true; + if (!InitializeAudioEngine() || !audio_engine_->HasWakeWord()) { + xEventGroupClearBits(event_group_, AS_EVENT_WAKE_WORD_RUNNING); + return; } - // Reset input resampler to clear cached data from previous mode (e.g. AudioProcessor) - // This prevents buffer overflow when switching between different feed sizes { std::lock_guard lock(input_resampler_mutex_); if (input_resampler_ != nullptr) { esp_ae_rate_cvt_reset(input_resampler_); } } - wake_word_->Start(); + audio_engine_->EnableWakeWordDetection(true); xEventGroupSetBits(event_group_, AS_EVENT_WAKE_WORD_RUNNING); } else { - wake_word_->Stop(); + if (audio_engine_initialized_) { + audio_engine_->EnableWakeWordDetection(false); + } xEventGroupClearBits(event_group_, AS_EVENT_WAKE_WORD_RUNNING); } } void AudioService::EnableVoiceProcessing(bool enable) { ESP_LOGD(TAG, "%s voice processing", enable ? "Enabling" : "Disabling"); - if (enable) { - if (!audio_processor_initialized_) { - audio_processor_->Initialize(codec_, OPUS_FRAME_DURATION_MS, models_list_); - audio_processor_initialized_ = true; - } - /* We should make sure no audio is playing */ + if (enable) { + if (!InitializeAudioEngine()) { + return; + } ResetDecoder(); audio_input_need_warmup_ = true; - // Reset input resampler to clear cached data from previous mode (e.g. WakeWord) - // This prevents buffer overflow when switching between different feed sizes { std::lock_guard lock(input_resampler_mutex_); if (input_resampler_ != nullptr) { esp_ae_rate_cvt_reset(input_resampler_); } } - audio_processor_->Start(); + audio_engine_->EnableVoiceProcessing(true); xEventGroupSetBits(event_group_, AS_EVENT_AUDIO_PROCESSOR_RUNNING); } else { - audio_processor_->Stop(); + if (audio_engine_initialized_) { + audio_engine_->EnableVoiceProcessing(false); + } xEventGroupClearBits(event_group_, AS_EVENT_AUDIO_PROCESSOR_RUNNING); } } @@ -618,12 +602,13 @@ void AudioService::EnableAudioTesting(bool enable) { void AudioService::EnableDeviceAec(bool enable) { ESP_LOGI(TAG, "%s device AEC", enable ? "Enabling" : "Disabling"); - if (!audio_processor_initialized_) { - audio_processor_->Initialize(codec_, OPUS_FRAME_DURATION_MS, models_list_); - audio_processor_initialized_ = true; - } + device_aec_enabled_ = enable; - audio_processor_->EnableDeviceAec(enable); + if (audio_engine_initialized_) { + audio_engine_->EnableDeviceAec(enable); + } else { + ESP_LOGI(TAG, "Deferring AEC change until the audio engine is initialized"); + } } void AudioService::SetCallbacks(AudioServiceCallbacks& callbacks) { @@ -698,37 +683,29 @@ void AudioService::CheckAndUpdateAudioPowerState() { } void AudioService::SetModelsList(srmodel_list_t* models_list) { + if (audio_engine_initialized_ && models_list_ != models_list) { + ESP_LOGW(TAG, "Ignoring speech model replacement after audio engine initialization"); + return; + } models_list_ = models_list; - -#if CONFIG_IDF_TARGET_ESP32S3 || CONFIG_IDF_TARGET_ESP32P4 - if (esp_srmodel_filter(models_list_, ESP_MN_PREFIX, NULL) != nullptr) { - wake_word_ = std::make_unique(); - } else if (esp_srmodel_filter(models_list_, ESP_WN_PREFIX, NULL) != nullptr) { - wake_word_ = std::make_unique(); - } else { - wake_word_ = nullptr; - } -#else - if (esp_srmodel_filter(models_list_, ESP_WN_PREFIX, NULL) != nullptr) { - wake_word_ = std::make_unique(); - } else { - wake_word_ = nullptr; - } -#endif - - if (wake_word_) { - wake_word_->OnWakeWordDetected([this](const std::string& wake_word) { - if (callbacks_.on_wake_word_detected) { - callbacks_.on_wake_word_detected(wake_word); - } - }); - } } bool AudioService::IsAfeWakeWord() { -#if CONFIG_IDF_TARGET_ESP32S3 || CONFIG_IDF_TARGET_ESP32P4 - return wake_word_ != nullptr && dynamic_cast(wake_word_.get()) != nullptr; -#else - return false; -#endif + return audio_engine_initialized_ && audio_engine_->IsAfeWakeWord(); +} + +bool AudioService::InitializeAudioEngine() { + if (!audio_engine_) { + return false; + } + if (audio_engine_initialized_) { + return true; + } + if (!audio_engine_->Initialize(codec_, OPUS_FRAME_DURATION_MS, models_list_)) { + ESP_LOGE(TAG, "Failed to initialize audio engine"); + return false; + } + audio_engine_initialized_ = true; + audio_engine_->EnableDeviceAec(device_aec_enabled_); + return true; } diff --git a/main/audio/audio_service.h b/main/audio/audio_service.h index 99db5461..c3afa134 100644 --- a/main/audio/audio_service.h +++ b/main/audio/audio_service.h @@ -19,18 +19,17 @@ #include "esp_audio_types.h" #include "audio_codec.h" -#include "audio_processor.h" -#include "processors/audio_debugger.h" -#include "wake_word.h" +#include "audio_debugger.h" +#include "audio_engine.h" #include "protocol.h" #include "ogg_demuxer.h" /* * There are two types of audio data flow: - * 1. (MIC) -> [Processors] -> {Encode Queue} -> [Opus Encoder] -> {Send Queue} -> (Server) + * 1. (MIC) -> [Audio Engine] -> {Encode Queue} -> [Opus Encoder] -> {Send Queue} -> (Server) * 2. (Server) -> {Decode Queue} -> [Opus Decoder] -> {Playback Queue} -> (Speaker) * - * We use one task for MIC / Speaker / Processors, and one task for Opus Encoder / Opus Decoder. + * We use dedicated tasks for input, output, and Opus encoding/decoding. * * Decode Queue and Send Queue are the main queues, because Opus packets are quite smaller than PCM packets. * @@ -137,8 +136,7 @@ public: private: AudioCodec* codec_ = nullptr; AudioServiceCallbacks callbacks_; - std::unique_ptr audio_processor_; - std::unique_ptr wake_word_; + std::unique_ptr audio_engine_; std::unique_ptr audio_debugger_; void* opus_encoder_ = nullptr; void* opus_decoder_ = nullptr; @@ -174,9 +172,13 @@ private: // For server AEC std::deque timestamp_queue_; - bool wake_word_initialized_ = false; - bool audio_processor_initialized_ = false; + bool audio_engine_initialized_ = false; bool voice_detected_ = false; +#if CONFIG_USE_DEVICE_AEC + bool device_aec_enabled_ = true; +#else + bool device_aec_enabled_ = false; +#endif bool service_stopped_ = true; bool audio_input_need_warmup_ = false; @@ -188,8 +190,9 @@ private: void AudioOutputTask(); void OpusCodecTask(); void PushTaskToEncodeQueue(AudioTaskType type, std::vector&& pcm); + bool InitializeAudioEngine(); void SetDecodeSampleRate(int sample_rate, int frame_duration); void CheckAndUpdateAudioPowerState(); }; -#endif \ No newline at end of file +#endif diff --git a/main/audio/engines/afe_audio_engine.cc b/main/audio/engines/afe_audio_engine.cc new file mode 100644 index 00000000..d32b6579 --- /dev/null +++ b/main/audio/engines/afe_audio_engine.cc @@ -0,0 +1,521 @@ +#include "afe_audio_engine.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "audio_service.h" +#include "wake_words/custom_wake_word.h" + +#define TAG "AfeAudioEngine" + +#if CONFIG_USE_AUDIO_PROCESSOR +static constexpr bool kUseAfeForVoiceProcessing = true; +#else +static constexpr bool kUseAfeForVoiceProcessing = false; +#endif + +AfeAudioEngine::AfeAudioEngine() { + event_group_ = xEventGroupCreate(); +} + +AfeAudioEngine::~AfeAudioEngine() { + custom_wake_word_.reset(); + if (afe_data_ != nullptr) { + afe_iface_->destroy(afe_data_); + } + if (wake_word_encode_task_stack_ != nullptr) { + heap_caps_free(wake_word_encode_task_stack_); + } + if (wake_word_encode_task_buffer_ != nullptr) { + heap_caps_free(wake_word_encode_task_buffer_); + } + if (owns_models_ && models_ != nullptr) { + esp_srmodel_deinit(models_); + } + if (event_group_ != nullptr) { + vEventGroupDelete(event_group_); + } +} + +bool AfeAudioEngine::Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) { + if (afe_data_ != nullptr || (codec_ != nullptr && !kUseAfeForVoiceProcessing && wake_detector_ == WakeDetector::kNone)) { + return true; + } + if (codec == nullptr) { + ESP_LOGE(TAG, "Codec is null"); + return false; + } + + codec_ = codec; + frame_samples_ = frame_duration_ms * 16000 / 1000; + output_buffer_.reserve(frame_samples_); + + if (models_list == nullptr) { + models_ = esp_srmodel_init("model"); + owns_models_ = models_ != nullptr; + } else { + models_ = models_list; + } + + char* wakenet_model_name = nullptr; + char* multinet_model_name = nullptr; + if (models_ != nullptr && models_->num > 0) { + wakenet_model_name = esp_srmodel_filter(models_, ESP_WN_PREFIX, nullptr); + multinet_model_name = esp_srmodel_filter(models_, ESP_MN_PREFIX, nullptr); + for (int i = 0; i < models_->num; ++i) { + ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]); + } + } + + if (multinet_model_name != nullptr) { + wake_detector_ = WakeDetector::kMultiNet; + custom_wake_word_ = std::make_unique(); + custom_wake_word_->OnWakeWordDetected([this](const std::string& wake_word) { + last_detected_wake_word_ = wake_word; + xEventGroupClearBits(event_group_, kWakeWordEnabled); + UpdateActiveState(); + if (wake_word_detected_callback_) { + wake_word_detected_callback_(wake_word); + } + }); + if (!custom_wake_word_->Initialize(codec_, models_)) { + ESP_LOGE(TAG, "Failed to initialize MultiNet wake-word detector"); + custom_wake_word_.reset(); + wake_detector_ = WakeDetector::kNone; + return false; + } + } else if (wakenet_model_name != nullptr) { + wake_detector_ = WakeDetector::kWakeNet; + auto words = esp_srmodel_get_wake_words(models_, wakenet_model_name); + if (words != nullptr) { + std::stringstream stream(words); + std::string word; + while (std::getline(stream, word, ';')) { + wake_words_.push_back(word); + } + } +#if CONFIG_SEND_WAKE_WORD_DATA + if (!wake_word_audio_cache_.Initialize(16000 * 2)) { + ESP_LOGW(TAG, "Wake-word audio upload disabled: PSRAM cache allocation failed"); + } +#endif + } + + const bool needs_afe = kUseAfeForVoiceProcessing || wake_detector_ != WakeDetector::kNone; + if (!needs_afe) { + ESP_LOGI(TAG, "Initialized as raw engine because AFE features are disabled"); + return true; + } + + int ref_num = codec_->input_reference() ? 1 : 0; + std::string input_format; + for (int i = 0; i < codec_->input_channels() - ref_num; ++i) { + input_format.push_back('M'); + } + for (int i = 0; i < ref_num; ++i) { + input_format.push_back('R'); + } + + char* vad_model_name = models_ == nullptr + ? nullptr + : esp_srmodel_filter(models_, ESP_VADN_PREFIX, nullptr); + afe_config_t* afe_config = afe_config_init( + input_format.c_str(), models_, AFE_TYPE_FD, AFE_MODE_LOW_COST); + if (afe_config == nullptr) { + ESP_LOGE(TAG, "Failed to create AFE configuration"); + return false; + } + + afe_config->aec_init = codec_->input_reference(); + afe_config->aec_mode = AEC_MODE_FD_LOW_COST; + afe_config->aec_nlp_level = AEC_NLP_LEVEL_VERYAGGR; + afe_config->ns_init = false; + afe_config->vad_init = kUseAfeForVoiceProcessing; + afe_config->vad_mode = VAD_MODE_0; + afe_config->vad_min_noise_ms = 100; + if (vad_model_name != nullptr) { + afe_config->vad_model_name = vad_model_name; + } + afe_config->wakenet_init = wake_detector_ == WakeDetector::kWakeNet; + afe_config->wakenet_model_name = wake_detector_ == WakeDetector::kWakeNet + ? wakenet_model_name + : nullptr; + afe_config->agc_init = false; + afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; + + afe_iface_ = esp_afe_handle_from_config(afe_config); + if (afe_iface_ != nullptr) { + afe_data_ = afe_iface_->create_from_config(afe_config); + } + afe_config_free(afe_config); + + if (afe_iface_ == nullptr || afe_data_ == nullptr) { + ESP_LOGE(TAG, "Failed to create FD AFE instance"); + afe_iface_ = nullptr; + afe_data_ = nullptr; + return false; + } + + if (wake_detector_ == WakeDetector::kWakeNet) { + afe_iface_->disable_wakenet(afe_data_); + } + if (codec_->input_reference()) { + afe_iface_->disable_aec(afe_data_); + } + afe_iface_->print_pipeline(afe_data_); + + BaseType_t task_created = xTaskCreate([](void* arg) { + auto* engine = static_cast(arg); + engine->ProcessingTask(); + vTaskDelete(nullptr); + }, "audio_afe", 4096, this, 3, &processing_task_); + if (task_created != pdPASS) { + ESP_LOGE(TAG, "Failed to create AFE processing task"); + afe_iface_->destroy(afe_data_); + afe_data_ = nullptr; + afe_iface_ = nullptr; + return false; + } + + const char* detector = wake_detector_ == WakeDetector::kWakeNet + ? "WakeNet" + : (wake_detector_ == WakeDetector::kMultiNet ? "MultiNet" : "none"); + ESP_LOGI(TAG, "Initialized FD AFE, detector: %s, NS: off, feed: %d, fetch: %d", + detector, afe_iface_->get_feed_chunksize(afe_data_), afe_iface_->get_fetch_chunksize(afe_data_)); + return true; +} + +void AfeAudioEngine::Feed(std::vector&& data) { + EventBits_t bits = xEventGroupGetBits(event_group_); + if ((bits & kVoiceProcessingEnabled) && !kUseAfeForVoiceProcessing) { + OutputRawAudio(data); + } + if (afe_data_ == nullptr || (bits & kAfeActive) == 0) { + return; + } + + std::lock_guard lock(input_buffer_mutex_); + if ((xEventGroupGetBits(event_group_) & kAfeActive) == 0) { + return; + } + input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); + size_t chunk_size = afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); + while (input_buffer_.size() >= chunk_size) { + afe_iface_->feed(afe_data_, input_buffer_.data()); + input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + chunk_size); + } +} + +void AfeAudioEngine::EnableWakeWordDetection(bool enable) { + if (!HasWakeWord()) { + return; + } + + if (enable) { + if (wake_detector_ == WakeDetector::kWakeNet) { + afe_iface_->enable_wakenet(afe_data_); + } else { + custom_wake_word_->Start(); + } + xEventGroupSetBits(event_group_, kWakeWordEnabled); + } else { + xEventGroupClearBits(event_group_, kWakeWordEnabled); + if (wake_detector_ == WakeDetector::kWakeNet) { + afe_iface_->disable_wakenet(afe_data_); + } else { + custom_wake_word_->Stop(); + } + } + UpdateActiveState(); +} + +void AfeAudioEngine::EnableVoiceProcessing(bool enable) { + if (enable) { + xEventGroupSetBits(event_group_, kVoiceProcessingEnabled); + } else { + xEventGroupClearBits(event_group_, kVoiceProcessingEnabled); + is_speaking_ = false; + } + UpdateActiveState(); +} + +void AfeAudioEngine::EnableDeviceAec(bool enable) { + device_aec_enabled_ = enable; + if (enable && (codec_ == nullptr || !codec_->input_reference())) { + ESP_LOGW(TAG, "Device AEC requires a playback reference channel"); + } + UpdateAecState(); +} + +bool AfeAudioEngine::HasWakeWord() const { + return wake_detector_ != WakeDetector::kNone; +} + +bool AfeAudioEngine::IsWakeWordDetectionEnabled() const { + return event_group_ != nullptr && (xEventGroupGetBits(event_group_) & kWakeWordEnabled) != 0; +} + +bool AfeAudioEngine::IsVoiceProcessingEnabled() const { + return event_group_ != nullptr && (xEventGroupGetBits(event_group_) & kVoiceProcessingEnabled) != 0; +} + +size_t AfeAudioEngine::GetFeedSize() const { + return afe_data_ == nullptr ? frame_samples_ : afe_iface_->get_feed_chunksize(afe_data_); +} + +void AfeAudioEngine::OnWakeWordDetected(std::function callback) { + wake_word_detected_callback_ = std::move(callback); +} + +void AfeAudioEngine::OnOutput(std::function&& data)> callback) { + output_callback_ = std::move(callback); +} + +void AfeAudioEngine::OnVadStateChange(std::function callback) { + vad_state_change_callback_ = std::move(callback); +} + +void AfeAudioEngine::UpdateActiveState() { + EventBits_t bits = xEventGroupGetBits(event_group_); + const bool afe_active = afe_data_ != nullptr && + ((bits & kWakeWordEnabled) || + (kUseAfeForVoiceProcessing && (bits & kVoiceProcessingEnabled))); + if (afe_active) { + xEventGroupSetBits(event_group_, kAfeActive); + } else { + xEventGroupClearBits(event_group_, kAfeActive); + std::lock_guard lock(input_buffer_mutex_); + input_buffer_.clear(); + if (afe_data_ != nullptr) { + afe_iface_->reset_buffer(afe_data_); + } + } + if ((bits & kVoiceProcessingEnabled) == 0) { + output_buffer_.clear(); + } + UpdateAecState(); +} + +void AfeAudioEngine::UpdateAecState() { + if (afe_data_ == nullptr || codec_ == nullptr || !codec_->input_reference()) { + return; + } + EventBits_t bits = xEventGroupGetBits(event_group_); + const bool enable = (bits & kWakeWordEnabled) || + (device_aec_enabled_ && (bits & kVoiceProcessingEnabled)); + if (enable) { + afe_iface_->enable_aec(afe_data_); + } else { + afe_iface_->disable_aec(afe_data_); + } +} + +void AfeAudioEngine::ProcessingTask() { + while (true) { + xEventGroupWaitBits(event_group_, kAfeActive, pdFALSE, pdTRUE, portMAX_DELAY); + auto* result = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); + if ((xEventGroupGetBits(event_group_) & kAfeActive) == 0) { + continue; + } + if (result == nullptr || result->ret_value == ESP_FAIL) { + if (result != nullptr) { + ESP_LOGW(TAG, "AFE fetch failed: %d", result->ret_value); + } + continue; + } + + EventBits_t bits = xEventGroupGetBits(event_group_); + if (bits & kWakeWordEnabled) { + HandleWakeWordResult(result); + } + if (kUseAfeForVoiceProcessing && (bits & kVoiceProcessingEnabled)) { + HandleVoiceResult(result); + } + } +} + +void AfeAudioEngine::HandleWakeWordResult(const afe_fetch_result_t* result) { + if (wake_detector_ == WakeDetector::kMultiNet) { + custom_wake_word_->FeedMono( + result->data, result->data_size / sizeof(int16_t)); + return; + } + +#if CONFIG_SEND_WAKE_WORD_DATA + wake_word_audio_cache_.Store(result->data, result->data_size / sizeof(int16_t)); +#endif + if (result->wakeup_state != WAKENET_DETECTED) { + return; + } + + int model_index = result->wakenet_model_index - 1; + if (model_index < 0 || model_index >= static_cast(wake_words_.size())) { + ESP_LOGE(TAG, "Invalid WakeNet model index: %d", result->wakenet_model_index); + return; + } + + last_detected_wake_word_ = wake_words_[model_index]; + xEventGroupClearBits(event_group_, kWakeWordEnabled); + afe_iface_->disable_wakenet(afe_data_); + UpdateActiveState(); + if (wake_word_detected_callback_) { + wake_word_detected_callback_(last_detected_wake_word_); + } +} + +void AfeAudioEngine::HandleVoiceResult(const afe_fetch_result_t* result) { + if (vad_state_change_callback_) { + if (result->vad_state == VAD_SPEECH && !is_speaking_) { + is_speaking_ = true; + vad_state_change_callback_(true); + } else if (result->vad_state == VAD_SILENCE && is_speaking_) { + is_speaking_ = false; + vad_state_change_callback_(false); + } + } + if (!output_callback_) { + return; + } + + size_t samples = result->data_size / sizeof(int16_t); + output_buffer_.insert(output_buffer_.end(), result->data, result->data + samples); + while (output_buffer_.size() >= static_cast(frame_samples_)) { + if (output_buffer_.size() == static_cast(frame_samples_)) { + output_callback_(std::move(output_buffer_)); + output_buffer_.clear(); + output_buffer_.reserve(frame_samples_); + } else { + output_callback_(std::vector( + output_buffer_.begin(), output_buffer_.begin() + frame_samples_)); + output_buffer_.erase(output_buffer_.begin(), output_buffer_.begin() + frame_samples_); + } + } +} + +void AfeAudioEngine::OutputRawAudio(const std::vector& data) { + if (!output_callback_ || codec_ == nullptr) { + return; + } + const size_t channels = codec_->input_channels(); + if (channels <= 1) { + output_buffer_.insert(output_buffer_.end(), data.begin(), data.end()); + } else { + for (size_t i = 0; i < data.size(); i += channels) { + output_buffer_.push_back(data[i]); + } + } + while (output_buffer_.size() >= static_cast(frame_samples_)) { + output_callback_(std::vector( + output_buffer_.begin(), output_buffer_.begin() + frame_samples_)); + output_buffer_.erase(output_buffer_.begin(), output_buffer_.begin() + frame_samples_); + } +} + +void AfeAudioEngine::EncodeWakeWordData() { + if (wake_detector_ == WakeDetector::kMultiNet) { + custom_wake_word_->EncodeWakeWordData(); + return; + } + if (wake_detector_ != WakeDetector::kWakeNet) { + return; + } + + const size_t stack_size = 4096 * 6; + wake_word_opus_.clear(); + if (wake_word_encode_task_stack_ == nullptr) { + wake_word_encode_task_stack_ = static_cast( + heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM)); + assert(wake_word_encode_task_stack_ != nullptr); + } + if (wake_word_encode_task_buffer_ == nullptr) { + wake_word_encode_task_buffer_ = static_cast( + heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL)); + assert(wake_word_encode_task_buffer_ != nullptr); + } + + wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { + auto* engine = static_cast(arg); + auto start_time = esp_timer_get_time(); + esp_opus_enc_config_t opus_enc_cfg = AS_OPUS_ENC_CONFIG(); + void* encoder_handle = nullptr; + auto ret = esp_opus_enc_open(&opus_enc_cfg, sizeof(esp_opus_enc_config_t), &encoder_handle); + if (encoder_handle == nullptr) { + ESP_LOGE(TAG, "Failed to create wake-word encoder: %d", ret); + engine->wake_word_audio_cache_.Clear(); + { + std::lock_guard lock(engine->wake_word_mutex_); + engine->wake_word_opus_.emplace_back(); + engine->wake_word_cv_.notify_all(); + } + vTaskDelete(nullptr); + return; + } + + int frame_size = 0; + int outbuf_size = 0; + esp_opus_enc_get_frame_size(encoder_handle, &frame_size, &outbuf_size); + frame_size /= sizeof(int16_t); + int packets = 0; + std::vector input(frame_size); + esp_audio_enc_in_frame_t in = {}; + esp_audio_enc_out_frame_t out = {}; + + const size_t cached_samples = engine->wake_word_audio_cache_.Size(); + for (size_t offset = 0; + offset + static_cast(frame_size) <= cached_samples; + offset += frame_size) { + if (engine->wake_word_audio_cache_.Read( + offset, input.data(), frame_size) != static_cast(frame_size)) { + break; + } + std::vector opus_buf(outbuf_size); + in.buffer = reinterpret_cast(input.data()); + in.len = frame_size * sizeof(int16_t); + out.buffer = opus_buf.data(); + out.len = outbuf_size; + out.encoded_bytes = 0; + ret = esp_opus_enc_process(encoder_handle, &in, &out); + if (ret == ESP_AUDIO_ERR_OK) { + std::lock_guard lock(engine->wake_word_mutex_); + engine->wake_word_opus_.emplace_back( + opus_buf.data(), opus_buf.data() + out.encoded_bytes); + engine->wake_word_cv_.notify_all(); + ++packets; + } else { + ESP_LOGE(TAG, "Failed to encode wake-word audio: %d", ret); + } + } + + engine->wake_word_audio_cache_.Clear(); + esp_opus_enc_close(encoder_handle); + ESP_LOGI(TAG, "Encoded wake word into %d packets in %ld ms", packets, + static_cast((esp_timer_get_time() - start_time) / 1000)); + { + std::lock_guard lock(engine->wake_word_mutex_); + engine->wake_word_opus_.emplace_back(); + engine->wake_word_cv_.notify_all(); + } + vTaskDelete(nullptr); + }, "encode_wake_word", stack_size, this, 2, + wake_word_encode_task_stack_, wake_word_encode_task_buffer_); +} + +bool AfeAudioEngine::GetWakeWordOpus(std::vector& opus) { + if (wake_detector_ == WakeDetector::kMultiNet) { + return custom_wake_word_->GetWakeWordOpus(opus); + } + if (wake_detector_ != WakeDetector::kWakeNet) { + return false; + } + std::unique_lock lock(wake_word_mutex_); + wake_word_cv_.wait(lock, [this]() { return !wake_word_opus_.empty(); }); + opus.swap(wake_word_opus_.front()); + wake_word_opus_.pop_front(); + return !opus.empty(); +} diff --git a/main/audio/engines/afe_audio_engine.h b/main/audio/engines/afe_audio_engine.h new file mode 100644 index 00000000..65385e44 --- /dev/null +++ b/main/audio/engines/afe_audio_engine.h @@ -0,0 +1,98 @@ +#ifndef AFE_AUDIO_ENGINE_H +#define AFE_AUDIO_ENGINE_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "audio_engine.h" +#include "wake_words/wake_word_audio_cache.h" + +class CustomWakeWord; + +class AfeAudioEngine : public AudioEngine { +public: + AfeAudioEngine(); + ~AfeAudioEngine() override; + + bool Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) override; + void Feed(std::vector&& data) override; + + void EnableWakeWordDetection(bool enable) override; + void EnableVoiceProcessing(bool enable) override; + void EnableDeviceAec(bool enable) override; + + bool HasWakeWord() const override; + bool IsWakeWordDetectionEnabled() const override; + bool IsVoiceProcessingEnabled() const override; + bool IsAfeWakeWord() const override { return HasWakeWord(); } + size_t GetFeedSize() const override; + + void OnWakeWordDetected(std::function callback) override; + void OnOutput(std::function&& data)> callback) override; + void OnVadStateChange(std::function callback) override; + + void EncodeWakeWordData() override; + bool GetWakeWordOpus(std::vector& opus) override; + const std::string& GetLastDetectedWakeWord() const override { return last_detected_wake_word_; } + +private: + enum class WakeDetector { + kNone, + kWakeNet, + kMultiNet, + }; + + static constexpr EventBits_t kWakeWordEnabled = 1 << 0; + static constexpr EventBits_t kVoiceProcessingEnabled = 1 << 1; + static constexpr EventBits_t kAfeActive = 1 << 2; + + AudioCodec* codec_ = nullptr; + srmodel_list_t* models_ = nullptr; + bool owns_models_ = false; + const esp_afe_sr_iface_t* afe_iface_ = nullptr; + esp_afe_sr_data_t* afe_data_ = nullptr; + EventGroupHandle_t event_group_ = nullptr; + TaskHandle_t processing_task_ = nullptr; + int frame_samples_ = 0; + bool is_speaking_ = false; + bool device_aec_enabled_ = false; + WakeDetector wake_detector_ = WakeDetector::kNone; + + std::unique_ptr custom_wake_word_; + std::vector wake_words_; + std::string last_detected_wake_word_; + std::vector input_buffer_; + std::vector output_buffer_; + std::mutex input_buffer_mutex_; + + std::function wake_word_detected_callback_; + std::function&&)> output_callback_; + std::function vad_state_change_callback_; + + TaskHandle_t wake_word_encode_task_ = nullptr; + StaticTask_t* wake_word_encode_task_buffer_ = nullptr; + StackType_t* wake_word_encode_task_stack_ = nullptr; + WakeWordAudioCache wake_word_audio_cache_; + std::deque> wake_word_opus_; + std::mutex wake_word_mutex_; + std::condition_variable wake_word_cv_; + + void ProcessingTask(); + void UpdateActiveState(); + void UpdateAecState(); + void OutputRawAudio(const std::vector& data); + void HandleWakeWordResult(const afe_fetch_result_t* result); + void HandleVoiceResult(const afe_fetch_result_t* result); +}; + +#endif diff --git a/main/audio/engines/lite_audio_engine.cc b/main/audio/engines/lite_audio_engine.cc new file mode 100644 index 00000000..59c64664 --- /dev/null +++ b/main/audio/engines/lite_audio_engine.cc @@ -0,0 +1,150 @@ +#include "lite_audio_engine.h" + +#include +#include + +#include "wake_words/esp_wake_word.h" + +#define TAG "LiteAudioEngine" + +LiteAudioEngine::LiteAudioEngine() = default; +LiteAudioEngine::~LiteAudioEngine() = default; + +bool LiteAudioEngine::Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) { + codec_ = codec; + frame_samples_ = frame_duration_ms * 16000 / 1000; + output_buffer_.reserve(frame_samples_); + + bool has_wakenet = models_list != nullptr && + esp_srmodel_filter(models_list, ESP_WN_PREFIX, nullptr) != nullptr; +#if CONFIG_USE_ESP_WAKE_WORD + has_wakenet = has_wakenet || models_list == nullptr; +#endif + if (has_wakenet) { + wake_word_ = std::make_unique(); + wake_word_->OnWakeWordDetected([this](const std::string& wake_word) { + wake_word_enabled_ = false; + if (wake_word_detected_callback_) { + wake_word_detected_callback_(wake_word); + } + }); + if (!wake_word_->Initialize(codec_, models_list)) { + ESP_LOGE(TAG, "Failed to initialize standalone WakeNet"); + wake_word_.reset(); + return false; + } + } + + ESP_LOGI(TAG, "Initialized, WakeNet: %s", wake_word_ ? "yes" : "no"); + return true; +} + +void LiteAudioEngine::Feed(std::vector&& data) { + if (wake_word_enabled_ && wake_word_) { + wake_word_->Feed(data); + } + if (voice_processing_enabled_) { + OutputRawAudio(data); + } +} + +void LiteAudioEngine::EnableWakeWordDetection(bool enable) { + if (!wake_word_) { + wake_word_enabled_ = false; + return; + } + + wake_word_enabled_ = enable; + if (enable) { + wake_word_->Start(); + } else { + wake_word_->Stop(); + } +} + +void LiteAudioEngine::EnableVoiceProcessing(bool enable) { + voice_processing_enabled_ = enable; + if (!enable) { + std::lock_guard lock(output_mutex_); + output_buffer_.clear(); + } +} + +void LiteAudioEngine::EnableDeviceAec(bool enable) { + if (enable) { + ESP_LOGW(TAG, "Device AEC is not supported by the lite engine"); + } +} + +bool LiteAudioEngine::HasWakeWord() const { + return wake_word_ != nullptr; +} + +bool LiteAudioEngine::IsWakeWordDetectionEnabled() const { + return wake_word_enabled_; +} + +bool LiteAudioEngine::IsVoiceProcessingEnabled() const { + return voice_processing_enabled_; +} + +size_t LiteAudioEngine::GetFeedSize() const { + if (wake_word_) { + return wake_word_->GetFeedSize(); + } + return frame_samples_; +} + +void LiteAudioEngine::OnWakeWordDetected(std::function callback) { + wake_word_detected_callback_ = std::move(callback); +} + +void LiteAudioEngine::OnOutput(std::function&& data)> callback) { + output_callback_ = std::move(callback); +} + +void LiteAudioEngine::OnVadStateChange(std::function callback) { + vad_state_change_callback_ = std::move(callback); +} + +void LiteAudioEngine::EncodeWakeWordData() { + if (wake_word_) { + wake_word_->EncodeWakeWordData(); + } +} + +bool LiteAudioEngine::GetWakeWordOpus(std::vector& opus) { + return wake_word_ && wake_word_->GetWakeWordOpus(opus); +} + +const std::string& LiteAudioEngine::GetLastDetectedWakeWord() const { + return wake_word_ ? wake_word_->GetLastDetectedWakeWord() : empty_wake_word_; +} + +void LiteAudioEngine::OutputRawAudio(const std::vector& data) { + if (!output_callback_ || codec_ == nullptr) { + return; + } + + std::lock_guard lock(output_mutex_); + const size_t channels = codec_->input_channels(); + if (channels <= 1) { + output_buffer_.insert(output_buffer_.end(), data.begin(), data.end()); + } else { + for (size_t i = 0; i < data.size(); i += channels) { + output_buffer_.push_back(data[i]); + } + } + + while (output_buffer_.size() >= static_cast(frame_samples_)) { + if (output_buffer_.size() == static_cast(frame_samples_)) { + output_callback_(std::move(output_buffer_)); + output_buffer_.clear(); + output_buffer_.reserve(frame_samples_); + } else { + output_callback_(std::vector( + output_buffer_.begin(), output_buffer_.begin() + frame_samples_)); + output_buffer_.erase(output_buffer_.begin(), output_buffer_.begin() + frame_samples_); + } + } +} diff --git a/main/audio/engines/lite_audio_engine.h b/main/audio/engines/lite_audio_engine.h new file mode 100644 index 00000000..7e56498d --- /dev/null +++ b/main/audio/engines/lite_audio_engine.h @@ -0,0 +1,58 @@ +#ifndef LITE_AUDIO_ENGINE_H +#define LITE_AUDIO_ENGINE_H + +#include +#include +#include +#include +#include +#include + +#include "audio_engine.h" + +class EspWakeWord; + +class LiteAudioEngine : public AudioEngine { +public: + LiteAudioEngine(); + ~LiteAudioEngine() override; + + bool Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) override; + void Feed(std::vector&& data) override; + + void EnableWakeWordDetection(bool enable) override; + void EnableVoiceProcessing(bool enable) override; + void EnableDeviceAec(bool enable) override; + + bool HasWakeWord() const override; + bool IsWakeWordDetectionEnabled() const override; + bool IsVoiceProcessingEnabled() const override; + bool IsAfeWakeWord() const override { return false; } + size_t GetFeedSize() const override; + + void OnWakeWordDetected(std::function callback) override; + void OnOutput(std::function&& data)> callback) override; + void OnVadStateChange(std::function callback) override; + + void EncodeWakeWordData() override; + bool GetWakeWordOpus(std::vector& opus) override; + const std::string& GetLastDetectedWakeWord() const override; + +private: + AudioCodec* codec_ = nullptr; + std::unique_ptr wake_word_; + std::atomic wake_word_enabled_ = false; + std::atomic voice_processing_enabled_ = false; + int frame_samples_ = 0; + std::vector output_buffer_; + std::mutex output_mutex_; + + std::function wake_word_detected_callback_; + std::function&&)> output_callback_; + std::function vad_state_change_callback_; + std::string empty_wake_word_; + + void OutputRawAudio(const std::vector& data); +}; + +#endif diff --git a/main/audio/processors/afe_audio_processor.cc b/main/audio/processors/afe_audio_processor.cc deleted file mode 100644 index 617bb2be..00000000 --- a/main/audio/processors/afe_audio_processor.cc +++ /dev/null @@ -1,202 +0,0 @@ -#include "afe_audio_processor.h" -#include - -#define PROCESSOR_RUNNING 0x01 - -#define TAG "AfeAudioProcessor" - -AfeAudioProcessor::AfeAudioProcessor() - : afe_data_(nullptr) { - event_group_ = xEventGroupCreate(); -} - -void AfeAudioProcessor::Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) { - codec_ = codec; - frame_samples_ = frame_duration_ms * 16000 / 1000; - - // Pre-allocate output buffer capacity - output_buffer_.reserve(frame_samples_); - - int ref_num = codec_->input_reference() ? 1 : 0; - - std::string input_format; - for (int i = 0; i < codec_->input_channels() - ref_num; i++) { - input_format.push_back('M'); - } - for (int i = 0; i < ref_num; i++) { - input_format.push_back('R'); - } - - srmodel_list_t *models; - if (models_list == nullptr) { - models = esp_srmodel_init("model"); - } else { - models = models_list; - } - - char* ns_model_name = esp_srmodel_filter(models, ESP_NSNET_PREFIX, NULL); - char* vad_model_name = esp_srmodel_filter(models, ESP_VADN_PREFIX, NULL); - - afe_config_t* afe_config = afe_config_init(input_format.c_str(), NULL, AFE_TYPE_FD, AFE_MODE_LOW_COST); - afe_config->aec_mode = AEC_MODE_FD_LOW_COST; - afe_config->aec_nlp_level = AEC_NLP_LEVEL_VERYAGGR; - afe_config->vad_mode = VAD_MODE_0; - afe_config->vad_min_noise_ms = 100; - if (vad_model_name != nullptr) { - afe_config->vad_model_name = vad_model_name; - } - - if (ns_model_name != nullptr) { - afe_config->ns_init = true; - afe_config->ns_model_name = ns_model_name; - afe_config->afe_ns_mode = AFE_NS_MODE_NET; - } else { - afe_config->ns_init = false; - } - - afe_config->agc_init = false; - afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; - -#ifdef CONFIG_USE_DEVICE_AEC - afe_config->aec_init = true; - afe_config->vad_init = false; -#else - afe_config->aec_init = false; - afe_config->vad_init = true; -#endif - - afe_iface_ = esp_afe_handle_from_config(afe_config); - afe_data_ = afe_iface_->create_from_config(afe_config); - - xTaskCreate([](void* arg) { - auto this_ = (AfeAudioProcessor*)arg; - this_->AudioProcessorTask(); - vTaskDelete(NULL); - }, "audio_communication", 4096, this, 3, NULL); -} - -AfeAudioProcessor::~AfeAudioProcessor() { - if (afe_data_ != nullptr) { - afe_iface_->destroy(afe_data_); - } - vEventGroupDelete(event_group_); -} - -size_t AfeAudioProcessor::GetFeedSize() { - if (afe_data_ == nullptr) { - return 0; - } - return afe_iface_->get_feed_chunksize(afe_data_); -} - -void AfeAudioProcessor::Feed(std::vector&& data) { - if (afe_data_ == nullptr) { - return; - } - - std::lock_guard lock(input_buffer_mutex_); - // Check running state inside lock to avoid TOCTOU race with Stop() - if (!IsRunning()) { - return; - } - input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); - size_t chunk_size = afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); - while (input_buffer_.size() >= chunk_size) { - afe_iface_->feed(afe_data_, input_buffer_.data()); - input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + chunk_size); - } -} - -void AfeAudioProcessor::Start() { - xEventGroupSetBits(event_group_, PROCESSOR_RUNNING); -} - -void AfeAudioProcessor::Stop() { - xEventGroupClearBits(event_group_, PROCESSOR_RUNNING); - - std::lock_guard lock(input_buffer_mutex_); - if (afe_data_ != nullptr) { - afe_iface_->reset_buffer(afe_data_); - } - input_buffer_.clear(); -} - -bool AfeAudioProcessor::IsRunning() { - return xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING; -} - -void AfeAudioProcessor::OnOutput(std::function&& data)> callback) { - output_callback_ = callback; -} - -void AfeAudioProcessor::OnVadStateChange(std::function callback) { - vad_state_change_callback_ = callback; -} - -void AfeAudioProcessor::AudioProcessorTask() { - auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); - auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); - ESP_LOGI(TAG, "Audio communication task started, feed size: %d fetch size: %d", - feed_size, fetch_size); - - while (true) { - xEventGroupWaitBits(event_group_, PROCESSOR_RUNNING, pdFALSE, pdTRUE, portMAX_DELAY); - - auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); - if ((xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING) == 0) { - continue; - } - if (res == nullptr || res->ret_value == ESP_FAIL) { - if (res != nullptr) { - ESP_LOGI(TAG, "Error code: %d", res->ret_value); - } - continue; - } - - // VAD state change - if (vad_state_change_callback_) { - if (res->vad_state == VAD_SPEECH && !is_speaking_) { - is_speaking_ = true; - vad_state_change_callback_(true); - } else if (res->vad_state == VAD_SILENCE && is_speaking_) { - is_speaking_ = false; - vad_state_change_callback_(false); - } - } - - if (output_callback_) { - size_t samples = res->data_size / sizeof(int16_t); - - // Add data to buffer - output_buffer_.insert(output_buffer_.end(), res->data, res->data + samples); - - // Output complete frames when buffer has enough data - while (output_buffer_.size() >= frame_samples_) { - if (output_buffer_.size() == frame_samples_) { - // If buffer size equals frame size, move the entire buffer - output_callback_(std::move(output_buffer_)); - output_buffer_.clear(); - output_buffer_.reserve(frame_samples_); - } else { - // If buffer size exceeds frame size, copy one frame and remove it - output_callback_(std::vector(output_buffer_.begin(), output_buffer_.begin() + frame_samples_)); - output_buffer_.erase(output_buffer_.begin(), output_buffer_.begin() + frame_samples_); - } - } - } - } -} - -void AfeAudioProcessor::EnableDeviceAec(bool enable) { - if (enable) { -#if CONFIG_USE_DEVICE_AEC - afe_iface_->disable_vad(afe_data_); - afe_iface_->enable_aec(afe_data_); -#else - ESP_LOGE(TAG, "Device AEC is not supported"); -#endif - } else { - afe_iface_->disable_aec(afe_data_); - afe_iface_->enable_vad(afe_data_); - } -} diff --git a/main/audio/processors/afe_audio_processor.h b/main/audio/processors/afe_audio_processor.h deleted file mode 100644 index fe2f0250..00000000 --- a/main/audio/processors/afe_audio_processor.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef AFE_AUDIO_PROCESSOR_H -#define AFE_AUDIO_PROCESSOR_H - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "audio_processor.h" -#include "audio_codec.h" - -class AfeAudioProcessor : public AudioProcessor { -public: - AfeAudioProcessor(); - ~AfeAudioProcessor(); - - void Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) override; - void Feed(std::vector&& data) override; - void Start() override; - void Stop() override; - bool IsRunning() override; - void OnOutput(std::function&& data)> callback) override; - void OnVadStateChange(std::function callback) override; - size_t GetFeedSize() override; - void EnableDeviceAec(bool enable) override; - -private: - EventGroupHandle_t event_group_ = nullptr; - const esp_afe_sr_iface_t* afe_iface_ = nullptr; - esp_afe_sr_data_t* afe_data_ = nullptr; - std::function&& data)> output_callback_; - std::function vad_state_change_callback_; - AudioCodec* codec_ = nullptr; - int frame_samples_ = 0; - bool is_speaking_ = false; - std::vector input_buffer_; - std::mutex input_buffer_mutex_; - std::vector output_buffer_; - - void AudioProcessorTask(); -}; - -#endif \ No newline at end of file diff --git a/main/audio/processors/no_audio_processor.cc b/main/audio/processors/no_audio_processor.cc deleted file mode 100644 index 90f997a9..00000000 --- a/main/audio/processors/no_audio_processor.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "no_audio_processor.h" -#include - -#define TAG "NoAudioProcessor" - -void NoAudioProcessor::Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) { - codec_ = codec; - frame_samples_ = frame_duration_ms * 16000 / 1000; - output_buffer_.reserve(frame_samples_); -} - -void NoAudioProcessor::Feed(std::vector&& data) { - if (!is_running_ || !output_callback_) { - return; - } - - // Convert stereo to mono if needed - if (codec_->input_channels() == 2) { - for (size_t i = 0, j = 0; i < data.size() / 2; ++i, j += 2) { - output_buffer_.push_back(data[j]); - } - } else { - output_buffer_.insert(output_buffer_.end(), data.begin(), data.end()); - } - - // Output complete frames when buffer has enough data - while (output_buffer_.size() >= (size_t)frame_samples_) { - if (output_buffer_.size() == (size_t)frame_samples_) { - output_callback_(std::move(output_buffer_)); - output_buffer_.clear(); - output_buffer_.reserve(frame_samples_); - } else { - output_callback_(std::vector(output_buffer_.begin(), output_buffer_.begin() + frame_samples_)); - output_buffer_.erase(output_buffer_.begin(), output_buffer_.begin() + frame_samples_); - } - } -} - -void NoAudioProcessor::Start() { - is_running_ = true; -} - -void NoAudioProcessor::Stop() { - is_running_ = false; - output_buffer_.clear(); -} - -bool NoAudioProcessor::IsRunning() { - return is_running_; -} - -void NoAudioProcessor::OnOutput(std::function&& data)> callback) { - output_callback_ = callback; -} - -void NoAudioProcessor::OnVadStateChange(std::function callback) { - vad_state_change_callback_ = callback; -} - -size_t NoAudioProcessor::GetFeedSize() { - if (!codec_) { - return 0; - } - return frame_samples_; -} - -void NoAudioProcessor::EnableDeviceAec(bool enable) { - if (enable) { - ESP_LOGE(TAG, "Device AEC is not supported"); - } -} diff --git a/main/audio/processors/no_audio_processor.h b/main/audio/processors/no_audio_processor.h deleted file mode 100644 index 70ef1d16..00000000 --- a/main/audio/processors/no_audio_processor.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef DUMMY_AUDIO_PROCESSOR_H -#define DUMMY_AUDIO_PROCESSOR_H - -#include -#include -#include - -#include "audio_processor.h" -#include "audio_codec.h" - -class NoAudioProcessor : public AudioProcessor { -public: - NoAudioProcessor() = default; - ~NoAudioProcessor() = default; - - void Initialize(AudioCodec* codec, int frame_duration_ms, srmodel_list_t* models_list) override; - void Feed(std::vector&& data) override; - void Start() override; - void Stop() override; - bool IsRunning() override; - void OnOutput(std::function&& data)> callback) override; - void OnVadStateChange(std::function callback) override; - size_t GetFeedSize() override; - void EnableDeviceAec(bool enable) override; - -private: - AudioCodec* codec_ = nullptr; - int frame_samples_ = 0; - std::vector output_buffer_; - std::function&& data)> output_callback_; - std::function vad_state_change_callback_; - std::atomic is_running_ = false; -}; - -#endif \ No newline at end of file diff --git a/main/audio/wake_words/afe_wake_word.cc b/main/audio/wake_words/afe_wake_word.cc deleted file mode 100644 index 8fc5fefb..00000000 --- a/main/audio/wake_words/afe_wake_word.cc +++ /dev/null @@ -1,263 +0,0 @@ -#include "afe_wake_word.h" -#include "audio_service.h" -#include -#include - -#define DETECTION_RUNNING_EVENT 1 - -#define TAG "AfeWakeWord" - -AfeWakeWord::AfeWakeWord() - : afe_data_(nullptr), - wake_word_pcm_(), - wake_word_opus_() { - - event_group_ = xEventGroupCreate(); -} - -AfeWakeWord::~AfeWakeWord() { - if (afe_data_ != nullptr) { - afe_iface_->destroy(afe_data_); - } - - if (wake_word_encode_task_stack_ != nullptr) { - heap_caps_free(wake_word_encode_task_stack_); - } - - if (wake_word_encode_task_buffer_ != nullptr) { - heap_caps_free(wake_word_encode_task_buffer_); - } - - if (models_ != nullptr) { - esp_srmodel_deinit(models_); - } - - vEventGroupDelete(event_group_); -} - -bool AfeWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) { - codec_ = codec; - int ref_num = codec_->input_reference() ? 1 : 0; - - if (models_list == nullptr) { - models_ = esp_srmodel_init("model"); - } else { - models_ = models_list; - } - - if (models_ == nullptr || models_->num == -1) { - ESP_LOGE(TAG, "Failed to initialize wakenet model"); - return false; - } - for (int i = 0; i < models_->num; i++) { - ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]); - if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) { - wakenet_model_ = models_->model_name[i]; - auto words = esp_srmodel_get_wake_words(models_, wakenet_model_); - // split by ";" to get all wake words - std::stringstream ss(words); - std::string word; - while (std::getline(ss, word, ';')) { - wake_words_.push_back(word); - } - } - } - - std::string input_format; - for (int i = 0; i < codec_->input_channels() - ref_num; i++) { - input_format.push_back('M'); - } - for (int i = 0; i < ref_num; i++) { - input_format.push_back('R'); - } - afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF); - afe_config->aec_init = codec_->input_reference(); - afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF; - afe_config->afe_perferred_core = 1; - afe_config->afe_perferred_priority = 1; - afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM; - - afe_iface_ = esp_afe_handle_from_config(afe_config); - afe_data_ = afe_iface_->create_from_config(afe_config); - - xTaskCreate([](void* arg) { - auto this_ = (AfeWakeWord*)arg; - this_->AudioDetectionTask(); - vTaskDelete(NULL); - }, "audio_detection", 4096, this, 3, nullptr); - - return true; -} - -void AfeWakeWord::OnWakeWordDetected(std::function callback) { - wake_word_detected_callback_ = callback; -} - -void AfeWakeWord::Start() { - xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT); -} - -void AfeWakeWord::Stop() { - xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT); - - std::lock_guard lock(input_buffer_mutex_); - if (afe_data_ != nullptr) { - afe_iface_->reset_buffer(afe_data_); - } - input_buffer_.clear(); -} - -void AfeWakeWord::Feed(const std::vector& data) { - if (afe_data_ == nullptr) { - return; - } - - std::lock_guard lock(input_buffer_mutex_); - // Check running state inside lock to avoid TOCTOU race with Stop() - if (!(xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT)) { - return; - } - input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); - size_t chunk_size = afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels(); - while (input_buffer_.size() >= chunk_size) { - afe_iface_->feed(afe_data_, input_buffer_.data()); - input_buffer_.erase(input_buffer_.begin(), input_buffer_.begin() + chunk_size); - } -} - -size_t AfeWakeWord::GetFeedSize() { - if (afe_data_ == nullptr) { - return 0; - } - return afe_iface_->get_feed_chunksize(afe_data_); -} - -void AfeWakeWord::AudioDetectionTask() { - auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_); - auto feed_size = afe_iface_->get_feed_chunksize(afe_data_); - ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d", - feed_size, fetch_size); - - while (true) { - xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY); - - auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY); - if (res == nullptr || res->ret_value == ESP_FAIL) { - continue;; - } - - // Store the wake word data for voice recognition, like who is speaking - StoreWakeWordData(res->data, res->data_size / sizeof(int16_t)); - - if (res->wakeup_state == WAKENET_DETECTED) { - Stop(); - last_detected_wake_word_ = wake_words_[res->wakenet_model_index - 1]; - - if (wake_word_detected_callback_) { - wake_word_detected_callback_(last_detected_wake_word_); - } - } - } -} - -void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) { - // store audio data to wake_word_pcm_ - wake_word_pcm_.emplace_back(std::vector(data, data + samples)); - // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512) - while (wake_word_pcm_.size() > 2000 / 30) { - wake_word_pcm_.pop_front(); - } -} - -void AfeWakeWord::EncodeWakeWordData() { - const size_t stack_size = 4096 * 6; - wake_word_opus_.clear(); - if (wake_word_encode_task_stack_ == nullptr) { - wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM); - assert(wake_word_encode_task_stack_ != nullptr); - } - if (wake_word_encode_task_buffer_ == nullptr) { - wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL); - assert(wake_word_encode_task_buffer_ != nullptr); - } - - wake_word_encode_task_ = xTaskCreateStatic([](void* arg) { - auto this_ = (AfeWakeWord*)arg; - { - auto start_time = esp_timer_get_time(); - // Create encoder - esp_opus_enc_config_t opus_enc_cfg = AS_OPUS_ENC_CONFIG(); - void* encoder_handle = nullptr; - auto ret = esp_opus_enc_open(&opus_enc_cfg, sizeof(esp_opus_enc_config_t), &encoder_handle); - if (encoder_handle == nullptr) { - ESP_LOGE(TAG, "Failed to create audio encoder, error code: %d", ret); - std::lock_guard lock(this_->wake_word_mutex_); - this_->wake_word_opus_.push_back(std::vector()); - this_->wake_word_cv_.notify_all(); - return; - } - - // Get frame size - int frame_size = 0; - int outbuf_size = 0; - esp_opus_enc_get_frame_size(encoder_handle, &frame_size, &outbuf_size); - frame_size = frame_size / sizeof(int16_t); - - // Encode all PCM data - int packets = 0; - std::vector in_buffer; - esp_audio_enc_in_frame_t in = {}; - esp_audio_enc_out_frame_t out = {}; - - for (auto& pcm: this_->wake_word_pcm_) { - if (in_buffer.empty()) { - in_buffer = std::move(pcm); - } else { - in_buffer.reserve(in_buffer.size() + pcm.size()); - in_buffer.insert(in_buffer.end(), pcm.begin(), pcm.end()); - } - - while (in_buffer.size() >= frame_size) { - std::vector opus_buf(outbuf_size); - in.buffer = (uint8_t *)(in_buffer.data()); - in.len = (uint32_t)(frame_size * sizeof(int16_t)); - out.buffer = opus_buf.data(); - out.len = outbuf_size; - out.encoded_bytes = 0; - - ret = esp_opus_enc_process(encoder_handle, &in, &out); - if (ret == ESP_AUDIO_ERR_OK) { - std::lock_guard lock(this_->wake_word_mutex_); - this_->wake_word_opus_.emplace_back(opus_buf.data(), opus_buf.data() + out.encoded_bytes); - this_->wake_word_cv_.notify_all(); - packets++; - } else { - ESP_LOGE(TAG, "Failed to encode audio, error code: %d", ret); - } - - in_buffer.erase(in_buffer.begin(), in_buffer.begin() + frame_size); - } - } - this_->wake_word_pcm_.clear(); - // Close encoder - esp_opus_enc_close(encoder_handle); - auto end_time = esp_timer_get_time(); - ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000)); - - std::lock_guard lock(this_->wake_word_mutex_); - this_->wake_word_opus_.push_back(std::vector()); - this_->wake_word_cv_.notify_all(); - } - vTaskDelete(NULL); - }, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_); -} - -bool AfeWakeWord::GetWakeWordOpus(std::vector& opus) { - std::unique_lock lock(wake_word_mutex_); - wake_word_cv_.wait(lock, [this]() { - return !wake_word_opus_.empty(); - }); - opus.swap(wake_word_opus_.front()); - wake_word_opus_.pop_front(); - return !opus.empty(); -} diff --git a/main/audio/wake_words/afe_wake_word.h b/main/audio/wake_words/afe_wake_word.h deleted file mode 100644 index 6c2bf72d..00000000 --- a/main/audio/wake_words/afe_wake_word.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef AFE_WAKE_WORD_H -#define AFE_WAKE_WORD_H - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "audio_codec.h" -#include "wake_word.h" - -class AfeWakeWord : public WakeWord { -public: - AfeWakeWord(); - ~AfeWakeWord(); - - bool Initialize(AudioCodec* codec, srmodel_list_t* models_list); - void Feed(const std::vector& data); - void OnWakeWordDetected(std::function callback); - void Start(); - void Stop(); - size_t GetFeedSize(); - void EncodeWakeWordData(); - bool GetWakeWordOpus(std::vector& opus); - const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; } - -private: - srmodel_list_t *models_ = nullptr; - const esp_afe_sr_iface_t* afe_iface_ = nullptr; - esp_afe_sr_data_t* afe_data_ = nullptr; - char* wakenet_model_ = NULL; - std::vector wake_words_; - EventGroupHandle_t event_group_; - std::function wake_word_detected_callback_; - AudioCodec* codec_ = nullptr; - std::string last_detected_wake_word_; - std::vector input_buffer_; - std::mutex input_buffer_mutex_; - - TaskHandle_t wake_word_encode_task_ = nullptr; - StaticTask_t* wake_word_encode_task_buffer_ = nullptr; - StackType_t* wake_word_encode_task_stack_ = nullptr; - std::deque> wake_word_pcm_; - std::deque> wake_word_opus_; - std::mutex wake_word_mutex_; - std::condition_variable wake_word_cv_; - - void StoreWakeWordData(const int16_t* data, size_t size); - void AudioDetectionTask(); -}; - -#endif diff --git a/main/audio/wake_words/custom_wake_word.cc b/main/audio/wake_words/custom_wake_word.cc index 982bbe96..681d02c9 100644 --- a/main/audio/wake_words/custom_wake_word.cc +++ b/main/audio/wake_words/custom_wake_word.cc @@ -12,7 +12,7 @@ #define TAG "CustomWakeWord" CustomWakeWord::CustomWakeWord() - : wake_word_pcm_(), wake_word_opus_() { + : wake_word_opus_() { } CustomWakeWord::~CustomWakeWord() { @@ -29,7 +29,7 @@ CustomWakeWord::~CustomWakeWord() { heap_caps_free(wake_word_encode_task_buffer_); } - if (models_ != nullptr) { + if (owns_models_ && models_ != nullptr) { esp_srmodel_deinit(models_); } } @@ -89,6 +89,7 @@ bool CustomWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) if (models_list == nullptr) { language_ = "cn"; models_ = esp_srmodel_init("model"); + owns_models_ = models_ != nullptr; #ifdef CONFIG_CUSTOM_WAKE_WORD threshold_ = CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f; commands_.push_back({CONFIG_CUSTOM_WAKE_WORD, CONFIG_CUSTOM_WAKE_WORD_DISPLAY, "wake"}); @@ -118,6 +119,7 @@ bool CustomWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) multinet_ = esp_mn_handle_from_name(mn_name_); multinet_model_data_ = multinet_->create(mn_name_, duration_); multinet_->set_det_threshold(multinet_model_data_, threshold_); + input_buffer_.reserve(multinet_->get_samp_chunksize(multinet_model_data_)); esp_mn_commands_clear(); for (int i = 0; i < commands_.size(); i++) { esp_mn_commands_add(i + 1, commands_[i].command.c_str()); @@ -125,6 +127,11 @@ bool CustomWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) esp_mn_commands_update(); multinet_->print_active_speech_commands(multinet_model_data_); +#if CONFIG_SEND_WAKE_WORD_DATA + if (!wake_word_audio_cache_.Initialize(16000 * 2)) { + ESP_LOGW(TAG, "Wake-word audio upload disabled: PSRAM cache allocation failed"); + } +#endif return true; } @@ -144,7 +151,15 @@ void CustomWakeWord::Stop() { } void CustomWakeWord::Feed(const std::vector& data) { - if (multinet_model_data_ == nullptr) { + FeedSamples(data.data(), data.size(), false); +} + +void CustomWakeWord::FeedMono(const int16_t* data, size_t samples) { + FeedSamples(data, samples, true); +} + +void CustomWakeWord::FeedSamples(const int16_t* data, size_t samples, bool mono) { + if (multinet_model_data_ == nullptr || data == nullptr || samples == 0) { return; } @@ -155,20 +170,21 @@ void CustomWakeWord::Feed(const std::vector& data) { } // If input channels is 2, we need to fetch the left channel data - if (codec_->input_channels() == 2) { - for (size_t i = 0; i < data.size(); i += 2) { + if (!mono && codec_->input_channels() > 1) { + for (size_t i = 0; i < samples; i += codec_->input_channels()) { input_buffer_.push_back(data[i]); } } else { - input_buffer_.insert(input_buffer_.end(), data.begin(), data.end()); + input_buffer_.insert(input_buffer_.end(), data, data + samples); } int chunksize = multinet_->get_samp_chunksize(multinet_model_data_); while (input_buffer_.size() >= chunksize) { - std::vector chunk(input_buffer_.begin(), input_buffer_.begin() + chunksize); - StoreWakeWordData(chunk); - - esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, chunk.data()); +#if CONFIG_SEND_WAKE_WORD_DATA + wake_word_audio_cache_.Store(input_buffer_.data(), chunksize); +#endif + + esp_mn_state_t mn_state = multinet_->detect(multinet_model_data_, input_buffer_.data()); if (mn_state == ESP_MN_STATE_DETECTED) { esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_); @@ -206,15 +222,6 @@ size_t CustomWakeWord::GetFeedSize() { return multinet_->get_samp_chunksize(multinet_model_data_); } -void CustomWakeWord::StoreWakeWordData(const std::vector& data) { - // store audio data to wake_word_pcm_ - wake_word_pcm_.push_back(data); - // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512) - while (wake_word_pcm_.size() > 2000 / 30) { - wake_word_pcm_.pop_front(); - } -} - void CustomWakeWord::EncodeWakeWordData() { const size_t stack_size = 4096 * 7; wake_word_opus_.clear(); @@ -237,9 +244,11 @@ void CustomWakeWord::EncodeWakeWordData() { auto ret = esp_opus_enc_open(&opus_enc_cfg, sizeof(esp_opus_enc_config_t), &encoder_handle); if (encoder_handle == nullptr) { ESP_LOGE(TAG, "Failed to create audio encoder, error code: %d", ret); + this_->wake_word_audio_cache_.Clear(); std::lock_guard lock(this_->wake_word_mutex_); this_->wake_word_opus_.push_back(std::vector()); this_->wake_word_cv_.notify_all(); + vTaskDelete(nullptr); return; } // Get frame size @@ -249,36 +258,34 @@ void CustomWakeWord::EncodeWakeWordData() { frame_size = frame_size / sizeof(int16_t); // Encode all PCM data int packets = 0; - std::vector in_buffer; + std::vector in_buffer(frame_size); esp_audio_enc_in_frame_t in = {}; esp_audio_enc_out_frame_t out = {}; - for (auto& pcm: this_->wake_word_pcm_) { - if (in_buffer.empty()) { - in_buffer = std::move(pcm); - } else { - in_buffer.reserve(in_buffer.size() + pcm.size()); - in_buffer.insert(in_buffer.end(), pcm.begin(), pcm.end()); + const size_t cached_samples = this_->wake_word_audio_cache_.Size(); + for (size_t offset = 0; + offset + static_cast(frame_size) <= cached_samples; + offset += frame_size) { + if (this_->wake_word_audio_cache_.Read( + offset, in_buffer.data(), frame_size) != static_cast(frame_size)) { + break; } - while (in_buffer.size() >= frame_size) { - std::vector opus_buf(outbuf_size); - in.buffer = (uint8_t *)(in_buffer.data()); - in.len = (uint32_t)(frame_size * sizeof(int16_t)); - out.buffer = opus_buf.data(); - out.len = outbuf_size; - out.encoded_bytes = 0; - ret = esp_opus_enc_process(encoder_handle, &in, &out); - if (ret == ESP_AUDIO_ERR_OK) { - std::lock_guard lock(this_->wake_word_mutex_); - this_->wake_word_opus_.emplace_back(opus_buf.data(), opus_buf.data() + out.encoded_bytes); - this_->wake_word_cv_.notify_all(); - packets++; - } else { - ESP_LOGE(TAG, "Failed to encode audio, error code: %d", ret); - } - in_buffer.erase(in_buffer.begin(), in_buffer.begin() + frame_size); + std::vector opus_buf(outbuf_size); + in.buffer = reinterpret_cast(in_buffer.data()); + in.len = frame_size * sizeof(int16_t); + out.buffer = opus_buf.data(); + out.len = outbuf_size; + out.encoded_bytes = 0; + ret = esp_opus_enc_process(encoder_handle, &in, &out); + if (ret == ESP_AUDIO_ERR_OK) { + std::lock_guard lock(this_->wake_word_mutex_); + this_->wake_word_opus_.emplace_back(opus_buf.data(), opus_buf.data() + out.encoded_bytes); + this_->wake_word_cv_.notify_all(); + packets++; + } else { + ESP_LOGE(TAG, "Failed to encode audio, error code: %d", ret); } } - this_->wake_word_pcm_.clear(); + this_->wake_word_audio_cache_.Clear(); // Close encoder esp_opus_enc_close(encoder_handle); auto end_time = esp_timer_get_time(); diff --git a/main/audio/wake_words/custom_wake_word.h b/main/audio/wake_words/custom_wake_word.h index 645ad1b3..7267c043 100644 --- a/main/audio/wake_words/custom_wake_word.h +++ b/main/audio/wake_words/custom_wake_word.h @@ -16,6 +16,7 @@ #include "audio_codec.h" #include "wake_word.h" +#include "wake_word_audio_cache.h" class CustomWakeWord : public WakeWord { public: @@ -24,6 +25,7 @@ public: bool Initialize(AudioCodec* codec, srmodel_list_t* models_list); void Feed(const std::vector& data); + void FeedMono(const int16_t* data, size_t samples); void OnWakeWordDetected(std::function callback); void Start(); void Stop(); @@ -43,6 +45,7 @@ private: esp_mn_iface_t* multinet_ = nullptr; model_iface_data_t* multinet_model_data_ = nullptr; srmodel_list_t *models_ = nullptr; + bool owns_models_ = false; char* mn_name_ = nullptr; std::string language_ = "cn"; int duration_ = 3000; @@ -59,12 +62,12 @@ private: TaskHandle_t wake_word_encode_task_ = nullptr; StaticTask_t* wake_word_encode_task_buffer_ = nullptr; StackType_t* wake_word_encode_task_stack_ = nullptr; - std::deque> wake_word_pcm_; + WakeWordAudioCache wake_word_audio_cache_; std::deque> wake_word_opus_; std::mutex wake_word_mutex_; std::condition_variable wake_word_cv_; - void StoreWakeWordData(const std::vector& data); + void FeedSamples(const int16_t* data, size_t samples, bool mono); void ParseWakenetModelConfig(); }; diff --git a/main/audio/wake_words/esp_wake_word.cc b/main/audio/wake_words/esp_wake_word.cc index 930aa2d6..51f3d7d5 100644 --- a/main/audio/wake_words/esp_wake_word.cc +++ b/main/audio/wake_words/esp_wake_word.cc @@ -10,6 +10,8 @@ EspWakeWord::EspWakeWord() { EspWakeWord::~EspWakeWord() { if (wakenet_data_ != nullptr) { wakenet_iface_->destroy(wakenet_data_); + } + if (owns_models_ && wakenet_model_ != nullptr) { esp_srmodel_deinit(wakenet_model_); } } @@ -19,6 +21,7 @@ bool EspWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) { if (models_list == nullptr) { wakenet_model_ = esp_srmodel_init("model"); + owns_models_ = wakenet_model_ != nullptr; } else { wakenet_model_ = models_list; } @@ -27,15 +30,25 @@ bool EspWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) { ESP_LOGE(TAG, "Failed to initialize wakenet model"); return false; } - if(wakenet_model_->num > 1) { - ESP_LOGW(TAG, "More than one model found, using the first one"); - } else if (wakenet_model_->num == 0) { + if (wakenet_model_->num == 0) { ESP_LOGE(TAG, "No model found"); return false; } - char *model_name = wakenet_model_->model_name[0]; + char *model_name = esp_srmodel_filter(wakenet_model_, ESP_WN_PREFIX, nullptr); + if (model_name == nullptr) { + ESP_LOGE(TAG, "No WakeNet model found"); + return false; + } wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name); + if (wakenet_iface_ == nullptr) { + ESP_LOGE(TAG, "No WakeNet interface found for %s", model_name); + return false; + } wakenet_data_ = wakenet_iface_->create(model_name, DET_MODE_95); + if (wakenet_data_ == nullptr) { + ESP_LOGE(TAG, "Failed to create WakeNet model %s", model_name); + return false; + } int frequency = wakenet_iface_->get_samp_rate(wakenet_data_); int audio_chunksize = wakenet_iface_->get_samp_chunksize(wakenet_data_); @@ -70,8 +83,8 @@ void EspWakeWord::Feed(const std::vector& data) { return; } - if (codec_->input_channels() == 2) { - for (size_t i = 0; i < data.size(); i += 2) { + if (codec_->input_channels() > 1) { + for (size_t i = 0; i < data.size(); i += codec_->input_channels()) { input_buffer_.push_back(data[i]); } } else { diff --git a/main/audio/wake_words/esp_wake_word.h b/main/audio/wake_words/esp_wake_word.h index 87e792da..c98c8007 100644 --- a/main/audio/wake_words/esp_wake_word.h +++ b/main/audio/wake_words/esp_wake_word.h @@ -33,6 +33,7 @@ private: esp_wn_iface_t *wakenet_iface_ = nullptr; model_iface_data_t *wakenet_data_ = nullptr; srmodel_list_t *wakenet_model_ = nullptr; + bool owns_models_ = false; AudioCodec* codec_ = nullptr; std::atomic running_ = false; diff --git a/main/audio/wake_words/wake_word_audio_cache.cc b/main/audio/wake_words/wake_word_audio_cache.cc new file mode 100644 index 00000000..fe020784 --- /dev/null +++ b/main/audio/wake_words/wake_word_audio_cache.cc @@ -0,0 +1,98 @@ +#include "wake_word_audio_cache.h" + +#include +#include + +#include +#include + +#define TAG "WakeWordCache" + +WakeWordAudioCache::~WakeWordAudioCache() { + if (buffer_ != nullptr) { + heap_caps_free(buffer_); + } +} + +bool WakeWordAudioCache::Initialize(size_t sample_count) { + std::lock_guard lock(mutex_); + if (buffer_ != nullptr) { + return capacity_ == sample_count; + } + if (sample_count == 0) { + return false; + } + + buffer_ = static_cast(heap_caps_malloc( + sample_count * sizeof(int16_t), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (buffer_ == nullptr) { + ESP_LOGE(TAG, "Failed to allocate %u bytes in PSRAM", + static_cast(sample_count * sizeof(int16_t))); + return false; + } + + capacity_ = sample_count; + ESP_LOGI(TAG, "Allocated %u bytes in PSRAM", + static_cast(capacity_ * sizeof(int16_t))); + return true; +} + +void WakeWordAudioCache::Store(const int16_t* data, size_t samples) { + if (data == nullptr || samples == 0) { + return; + } + + std::lock_guard lock(mutex_); + if (buffer_ == nullptr) { + return; + } + + if (samples >= capacity_) { + data += samples - capacity_; + samples = capacity_; + std::memcpy(buffer_, data, samples * sizeof(int16_t)); + size_ = capacity_; + write_position_ = 0; + return; + } + + const size_t first = std::min(samples, capacity_ - write_position_); + std::memcpy(buffer_ + write_position_, data, first * sizeof(int16_t)); + if (samples > first) { + std::memcpy(buffer_, data + first, (samples - first) * sizeof(int16_t)); + } + write_position_ = (write_position_ + samples) % capacity_; + size_ = std::min(capacity_, size_ + samples); +} + +size_t WakeWordAudioCache::Read(size_t offset, int16_t* output, size_t samples) const { + if (output == nullptr || samples == 0) { + return 0; + } + + std::lock_guard lock(mutex_); + if (buffer_ == nullptr || offset >= size_) { + return 0; + } + + samples = std::min(samples, size_ - offset); + const size_t oldest = (write_position_ + capacity_ - size_) % capacity_; + const size_t read_position = (oldest + offset) % capacity_; + const size_t first = std::min(samples, capacity_ - read_position); + std::memcpy(output, buffer_ + read_position, first * sizeof(int16_t)); + if (samples > first) { + std::memcpy(output + first, buffer_, (samples - first) * sizeof(int16_t)); + } + return samples; +} + +size_t WakeWordAudioCache::Size() const { + std::lock_guard lock(mutex_); + return size_; +} + +void WakeWordAudioCache::Clear() { + std::lock_guard lock(mutex_); + size_ = 0; + write_position_ = 0; +} diff --git a/main/audio/wake_words/wake_word_audio_cache.h b/main/audio/wake_words/wake_word_audio_cache.h new file mode 100644 index 00000000..4346dff8 --- /dev/null +++ b/main/audio/wake_words/wake_word_audio_cache.h @@ -0,0 +1,30 @@ +#ifndef WAKE_WORD_AUDIO_CACHE_H +#define WAKE_WORD_AUDIO_CACHE_H + +#include +#include +#include + +class WakeWordAudioCache { +public: + WakeWordAudioCache() = default; + ~WakeWordAudioCache(); + + WakeWordAudioCache(const WakeWordAudioCache&) = delete; + WakeWordAudioCache& operator=(const WakeWordAudioCache&) = delete; + + bool Initialize(size_t sample_count); + void Store(const int16_t* data, size_t samples); + size_t Read(size_t offset, int16_t* output, size_t samples) const; + size_t Size() const; + void Clear(); + +private: + int16_t* buffer_ = nullptr; + size_t capacity_ = 0; + size_t size_ = 0; + size_t write_position_ = 0; + mutable std::mutex mutex_; +}; + +#endif