4#ifdef USE_VOICE_ASSISTANT
17static constexpr uint32_t RESET_CONVERSATION_TIMEOUT_ID = 0;
18static constexpr uint32_t PLAYBACK_TIMEOUT_ID = 1;
19static constexpr uint32_t SPEAKER_IDLE_TIMEOUT_ID = 2;
20static constexpr uint32_t TIMER_TICK_INTERVAL_ID = 3;
26static const size_t SAMPLE_RATE_HZ = 16000;
28static const size_t RING_BUFFER_SAMPLES = 512 * SAMPLE_RATE_HZ / 1000;
29static const size_t RING_BUFFER_SIZE = RING_BUFFER_SAMPLES *
sizeof(int16_t);
30static const size_t SEND_BUFFER_SAMPLES = 32 * SAMPLE_RATE_HZ / 1000;
31static const size_t SEND_BUFFER_SIZE = SEND_BUFFER_SAMPLES *
sizeof(int16_t);
32static const size_t RECEIVE_SIZE = 1024;
35static constexpr uint32_t PLAYBACK_STALL_TIMEOUT_MS = 2000;
37static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE;
42static const uint32_t AUDIO_CHANNEL_STALL_TIMEOUT_MS = 2000;
48 std::shared_ptr<ring_buffer::RingBuffer> temp_ring_buffer = this->
ring_buffer_.lock();
49 if (temp_ring_buffer !=
nullptr) {
50 temp_ring_buffer->write((
void *) capture.
data.data(), capture.
data.size());
57 std::shared_ptr<ring_buffer::RingBuffer> temp_ring_buffer = this->
ring_buffer2_.lock();
58 if (temp_ring_buffer !=
nullptr) {
59 temp_ring_buffer->write((
void *) capture.
data.data(), capture.
data.size());
64#ifdef USE_MEDIA_PLAYER
95 ESP_LOGE(TAG,
"Could not create socket");
100 int err = this->
socket_->setsockopt(SOL_SOCKET, SO_REUSEADDR, &enable,
sizeof(
int));
102 ESP_LOGW(TAG,
"Socket unable to set reuseaddr: errno %d", err);
105 err = this->
socket_->setblocking(
false);
107 ESP_LOGE(TAG,
"Socket unable to set nonblocking mode: errno %d", err);
118 ESP_LOGE(TAG,
"Socket unable to set sockaddr: errno %d", errno);
123 err = this->
socket_->bind((
struct sockaddr *) &server,
sizeof(server));
125 ESP_LOGE(TAG,
"Socket unable to bind: errno %d", errno);
141 ESP_LOGW(TAG,
"Could not allocate speaker buffer");
149 if (temp_ring_buffer ==
nullptr) {
150 ESP_LOGE(TAG,
"Could not allocate ring buffer");
156 ESP_LOGE(TAG,
"Could not allocate audio source");
165 if (temp_ring_buffer ==
nullptr) {
166 ESP_LOGE(TAG,
"Could not allocate second ring buffer");
171 ESP_LOGE(TAG,
"Could not allocate second audio source");
222 ESP_LOGD(TAG,
"reset conversation ID");
237 size_t available2 = 0;
243 const bool channel_empty = (available == 0);
244 const bool channel2_empty = (this->
audio_source2_ !=
nullptr) && (available2 == 0);
245 if (channel_empty || channel2_empty) {
284 if ((available == 0) && (available2 == 0)) {
294 ESP_LOGW(TAG,
"Mic channel %d stalled, stopping stream", (available == 0) ? 0 : 1);
298 this->
defer([
this]() {
329 ESP_LOGD(TAG,
"Starting Microphone");
353 ESP_LOGD(TAG,
"Requesting start");
372#ifdef USE_MEDIA_PLAYER
379 ESP_LOGW(TAG,
"Could not request start");
402 if (available == 0) {
412 sizeof(this->dest_addr_));
421 bool is_running2 =
false;
425 if (is_running || is_running2) {
441 bool is_stopped2 =
true;
445 if (is_stopped && is_stopped2) {
454 bool playing =
false;
461 if (received_len > 0) {
467 ESP_LOGD(TAG,
"Receive buffer full");
477 ESP_LOGD(TAG,
"End of audio stream received");
486#ifdef USE_MEDIA_PLAYER
493 ESP_LOGD(TAG,
"Announcement finished playing");
499 API_LOG_MSG_DROPPED(TAG,
"Announce-finished");
521 ESP_LOGD(TAG,
"Speaker has finished outputting all audio");
534#ifdef USE_MEDIA_PLAYER
575 ESP_LOGV(TAG,
"Speaker buffer full, trying again next loop");
585 ESP_LOGE(TAG,
"Client attempting to unsubscribe that is not the current API Client");
594 char current_peername[socket::SOCKADDR_STR_LEN];
595 char new_peername[socket::SOCKADDR_STR_LEN];
597 "Multiple API Clients attempting to connect to Voice Assistant\n"
598 " Current client: %s (%s)\n"
599 " New client: %s (%s)",
609static const LogString *voice_assistant_state_to_string(
State state) {
612 return LOG_STR(
"IDLE");
614 return LOG_STR(
"START_MICROPHONE");
616 return LOG_STR(
"STARTING_MICROPHONE");
618 return LOG_STR(
"WAIT_FOR_VAD");
620 return LOG_STR(
"WAITING_FOR_VAD");
622 return LOG_STR(
"START_PIPELINE");
624 return LOG_STR(
"STARTING_PIPELINE");
626 return LOG_STR(
"STREAMING_MICROPHONE");
628 return LOG_STR(
"STOP_MICROPHONE");
630 return LOG_STR(
"STOPPING_MICROPHONE");
632 return LOG_STR(
"AWAITING_RESPONSE");
634 return LOG_STR(
"STREAMING_RESPONSE");
636 return LOG_STR(
"RESPONSE_FINISHED");
638 return LOG_STR(
"UNKNOWN");
645 ESP_LOGD(TAG,
"State changed from %s to %s", LOG_STR_ARG(voice_assistant_state_to_string(old_state)),
646 LOG_STR_ARG(voice_assistant_state_to_string(
state)));
652 ESP_LOGD(TAG,
"Desired state set to %s", LOG_STR_ARG(voice_assistant_state_to_string(desired_state)));
659 ESP_LOGD(TAG,
"Desired state set to %s", LOG_STR_ARG(voice_assistant_state_to_string(
state)));
666 ESP_LOGE(TAG,
"Failed to start server. See Home Assistant logs for more details.");
667 this->
error_trigger_.
trigger(
"failed-to-start",
"Failed to start server. See Home Assistant logs for more details.");
677 ESP_LOGD(TAG,
"Client started, streaming microphone");
694 ESP_LOGD(TAG,
"Client started, streaming microphone");
698 ESP_LOGW(TAG,
"UDP audio mode does not support a second microphone channel; only the primary will be streamed");
728 ESP_LOGE(TAG,
"No API client connected");
778#ifdef USE_MEDIA_PLAYER
791#ifdef USE_MEDIA_PLAYER
811 ESP_LOGD(TAG,
"Signaling stop");
815 API_LOG_MSG_DROPPED(TAG,
"Stop request");
820 this->
set_timeout(PLAYBACK_TIMEOUT_ID, timeout_ms, [
this]() {
829 API_LOG_MSG_DROPPED(TAG,
"Announce-finished");
835 ESP_LOGD(TAG,
"Event Type: %" PRId32, msg.
event_type);
838 ESP_LOGD(TAG,
"Assist Pipeline running");
839#ifdef USE_MEDIA_PLAYER
841 for (
const auto &arg : msg.
data) {
842 if (arg.name ==
"url") {
852 ESP_LOGD(TAG,
"Wake word detected");
857 ESP_LOGD(TAG,
"STT started");
867 for (
const auto &arg : msg.
data) {
868 if (arg.name ==
"text") {
873 ESP_LOGW(TAG,
"No text in STT_END event");
875 }
else if (text.length() > 500) {
879 ESP_LOGD(TAG,
"Speech recognised as: \"%s\"", text.c_str());
884 ESP_LOGD(TAG,
"Intent started");
888 ESP_LOGD(TAG,
"Intent progress");
889 std::string tts_url_for_trigger;
890#ifdef USE_MEDIA_PLAYER
892 for (
const auto &arg : msg.
data) {
893 if ((arg.name ==
"tts_start_streaming") && (arg.value ==
"1") && !this->tts_response_url_.empty()) {
912 for (
const auto &arg : msg.
data) {
913 if (arg.name ==
"conversation_id") {
915 }
else if (arg.name ==
"continue_conversation") {
924 for (
const auto &arg : msg.
data) {
925 if (arg.name ==
"text") {
930 ESP_LOGW(TAG,
"No text in TTS_START event");
933 if (text.length() > 500) {
937 ESP_LOGD(TAG,
"Response: \"%s\"", text.c_str());
938 this->
defer([
this, text]() {
950 for (
const auto &arg : msg.
data) {
951 if (arg.name ==
"url") {
956 ESP_LOGW(TAG,
"No url in TTS_END event");
959 ESP_LOGD(TAG,
"Response URL: \"%s\"", url.c_str());
960 this->
defer([
this, url]() {
961#ifdef USE_MEDIA_PLAYER
978 ESP_LOGD(TAG,
"Assist Pipeline ended");
996 for (
const auto &arg : msg.
data) {
997 if (arg.name ==
"code") {
999 }
else if (arg.name ==
"message") {
1003 if (code ==
"wake-word-timeout" || code ==
"wake_word_detection_aborted" || code ==
"no_wake_word") {
1006 }
else if (code ==
"wake-provider-missing" || code ==
"wake-engine-missing") {
1014 ESP_LOGE(TAG,
"Error: %s - %s", code.c_str(),
message.c_str());
1026 ESP_LOGD(TAG,
"TTS stream start");
1036 ESP_LOGD(TAG,
"TTS stream end");
1042 ESP_LOGD(TAG,
"Starting STT by VAD");
1046 ESP_LOGD(TAG,
"STT by VAD end");
1051 ESP_LOGD(TAG,
"Unhandled event type: %" PRId32, msg.
event_type);
1065 ESP_LOGV(TAG,
"Received audio: %u bytes from API", msg.
data_len);
1067 ESP_LOGE(TAG,
"Cannot receive audio, buffer is full");
1075 auto it = this->
timers_.begin();
1076 for (; it != this->
timers_.end(); ++it) {
1080 if (it == this->
timers_.end()) {
1085 it->name = msg.
name;
1093 " Type: %" PRId32
"\n"
1124 for (
auto &timer : this->
timers_) {
1125 if (timer.is_active && timer.seconds_left > 0) {
1126 timer.seconds_left--;
1133#ifdef USE_MEDIA_PLAYER
1166#ifdef USE_MICRO_WAKE_WORD
1174 for (
const auto &ww_id : active_wake_words) {
1176 if (model->get_id() == ww_id) {
1178 ESP_LOGD(TAG,
"Enabled wake word: %s (id=%s)", model->get_wake_word().c_str(), model->get_id().c_str());
1190#ifdef USE_MICRO_WAKE_WORD
1195 if (model->is_enabled()) {
1200 wake_word.
id = model->get_id();
1201 wake_word.
wake_word = model->get_wake_word();
1202 for (
const auto &lang : model->get_trained_languages()) {
1211#ifdef USE_MICRO_WAKE_WORD
uint32_t IRAM_ATTR HOT get_loop_component_start_time() const
Get the cached time in milliseconds from when the current component started its loop execution.
void mark_failed()
Mark this component as failed.
bool cancel_interval(const char *name)
Cancel an interval function.
void status_clear_error()
bool cancel_timeout(const char *name)
Cancel a timeout function.
void set_timeout(const char *name, uint32_t timeout, std::function< void()> &&f)
Set a timeout function with a const char* name.
void defer(const char *name, std::function< void()> &&f)
Defer a callback to the next loop() call with a const char* name.
bool status_has_error() const
void set_interval(const char *name, uint32_t interval, std::function< void()> &&f)
Set an interval function with a const char* name.
An STL allocator that uses SPI or internal RAM.
void deallocate(T *p, size_t n)
StringRef is a reference to a string owned by something else.
constexpr bool empty() const
void trigger(const Ts &...x) ESPHOME_ALWAYS_INLINE
Inform the parent automation that the event has triggered.
const char * get_peername_to(std::span< char, socket::SOCKADDR_STR_LEN > buf) const
Get peer name (IP address) into caller-provided buffer, returns buf for convenience.
const char * get_name() const
bool send_message(const T &msg)
Returns false as soon as the TCP buffer is full.
StringRef preannounce_media_id
uint32_t noise_suppression_level
enums::VoiceAssistantEvent event_type
std::vector< VoiceAssistantEventData > data
StringRef conversation_id
StringRef wake_word_phrase
VoiceAssistantAudioSettings audio_settings
enums::VoiceAssistantTimerEvent event_type
static std::unique_ptr< RingBufferAudioSource > create(std::shared_ptr< ring_buffer::RingBuffer > ring_buffer, size_t max_fill_bytes, uint8_t alignment_bytes=1)
Creates a new ring-buffer-backed audio source after validating its parameters.
std::vector< WakeWordModel * > get_wake_words()
void add_data_callback(F &&data_callback)
Same callbacks as Microphone::add_data_callback; stream_info is the processed format.
static std::unique_ptr< RingBuffer > create(size_t len, MemoryPreference preference=MemoryPreference::EXTERNAL_FIRST)
virtual size_t play(const uint8_t *data, size_t length)=0
Plays the provided audio data.
virtual bool has_buffered_data() const =0
Trigger intent_end_trigger_
std::unique_ptr< socket::Socket > socket_
const Configuration & get_configuration()
size_t speaker_buffer_size_
bool started_streaming_tts_
std::string tts_response_url_
microphone::MicrophoneSource * mic_source2_
bool wait_for_stream_end_
size_t speaker_buffer_index_
Trigger intent_start_trigger_
void on_timer_event(const api::VoiceAssistantTimerEventResponse &msg)
Trigger tts_stream_start_trigger_
void cancel_response_()
Stop a playing or queued response and tell Home Assistant when the TTS stream is still open.
Trigger client_connected_trigger_
Trigger< Timer > timer_cancelled_trigger_
std::vector< Timer > timers_
void on_audio(const api::VoiceAssistantAudio &msg)
std::unique_ptr< audio::RingBufferAudioSource > audio_source_
std::weak_ptr< ring_buffer::RingBuffer > ring_buffer2_
std::weak_ptr< ring_buffer::RingBuffer > ring_buffer_
media_player::MediaPlayer * media_player_
float get_setup_priority() const override
Trigger wake_word_detected_trigger_
void set_state_(State state)
void start_playback_timeout_(uint32_t timeout_ms)
uint8_t * speaker_buffer_
Trigger stt_vad_end_trigger_
void client_subscription(api::APIConnection *client, bool subscribe)
Trigger< Timer > timer_started_trigger_
MediaPlayerResponseState media_player_response_state_
void deallocate_buffers_()
uint32_t tts_playback_start_timeout_
Trigger listening_trigger_
uint8_t noise_suppression_level_
void on_event(const api::VoiceAssistantEventResponse &msg)
Trigger< std::string, std::string > error_trigger_
void set_state_after_mic_stop_(State state)
Move to state, or only make it the desired state while the microphone is still stopping.
Trigger< std::string > stt_end_trigger_
uint32_t conversation_timeout_
Trigger< const std::vector< Timer > & > timer_tick_trigger_
uint32_t audio_channel_stall_start_
Trigger< std::string > intent_progress_trigger_
void on_announce(const api::VoiceAssistantAnnounceRequest &msg)
void request_start(bool continuous, bool silence_detection)
Trigger< std::string > tts_start_trigger_
api::APIConnection * api_client_
Trigger< std::string > tts_end_trigger_
struct sockaddr_storage dest_addr_
speaker::Speaker * speaker_
Trigger< Timer > timer_updated_trigger_
void handle_channel_stall_(size_t available, size_t available2)
std::unique_ptr< audio::RingBufferAudioSource > audio_source2_
Trigger tts_stream_end_trigger_
Trigger stt_vad_start_trigger_
void reset_conversation_id()
std::string conversation_id_
microphone::MicrophoneSource * mic_source_
micro_wake_word::MicroWakeWord * micro_wake_word_
size_t speaker_bytes_received_
bool continue_conversation_
void on_set_configuration(const std::vector< std::string > &active_wake_words)
Trigger< Timer > timer_finished_trigger_
Trigger client_disconnected_trigger_
const LogString * message
@ VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD
@ VOICE_ASSISTANT_REQUEST_USE_VAD
@ VOICE_ASSISTANT_TIMER_UPDATED
@ VOICE_ASSISTANT_TIMER_STARTED
@ VOICE_ASSISTANT_TIMER_FINISHED
@ VOICE_ASSISTANT_TIMER_CANCELLED
@ VOICE_ASSISTANT_INTENT_END
@ VOICE_ASSISTANT_RUN_START
@ VOICE_ASSISTANT_TTS_END
@ VOICE_ASSISTANT_RUN_END
@ VOICE_ASSISTANT_WAKE_WORD_START
@ VOICE_ASSISTANT_TTS_STREAM_END
@ VOICE_ASSISTANT_STT_END
@ VOICE_ASSISTANT_STT_VAD_START
@ VOICE_ASSISTANT_INTENT_PROGRESS
@ VOICE_ASSISTANT_TTS_START
@ VOICE_ASSISTANT_STT_START
@ VOICE_ASSISTANT_INTENT_START
@ VOICE_ASSISTANT_TTS_STREAM_START
@ VOICE_ASSISTANT_STT_VAD_END
@ VOICE_ASSISTANT_WAKE_WORD_END
constexpr float AFTER_CONNECTION
For components that should be initialized after a data connection (API/MQTT) is connected.
std::unique_ptr< Socket > socket_ip(int type, int protocol)
Create a socket in the newest available IP domain (IPv6 or IPv4) of the given type and protocol.
std::unique_ptr< Socket > socket(int domain, int type, int protocol)
Create a socket of the given domain, type and protocol.
socklen_t set_sockaddr_any(struct sockaddr *addr, socklen_t addrlen, uint16_t port)
Set a sockaddr to the any address and specified port for the IP version used by socket_ip().
ESPHOME_LOG_TAG(TAG, "voice_assistant")
VoiceAssistant * global_voice_assistant
Application App
Global storage of Application pointer - only one Application can exist.
std::span< const uint8_t > data
Only valid during the callback.
std::vector< WakeWord > available_wake_words
std::vector< std::string > active_wake_words
uint32_t max_active_wake_words
static constexpr size_t TO_STR_BUFFER_SIZE
Buffer size for to_str() - sufficient for typical timer names.
std::vector< std::string > trained_languages