ESPHome 2026.11.0b6
Loading...
Searching...
No Matches
voice_assistant.cpp
Go to the documentation of this file.
1#include "voice_assistant.h"
3
4#ifdef USE_VOICE_ASSISTANT
5
8#include "esphome/core/log.h"
9
10#include <cinttypes>
11#include <cstdio>
12
14
15ESPHOME_LOG_TAG(TAG, "voice_assistant");
16
17static constexpr uint32_t RESET_CONVERSATION_TIMEOUT_ID = 0;
18static constexpr uint32_t PLAYBACK_TIMEOUT_ID = 1;
19static constexpr uint32_t SPEAKER_IDLE_TIMEOUT_ID = 2;
20static constexpr uint32_t TIMER_TICK_INTERVAL_ID = 3;
21
22#ifdef SAMPLE_RATE_HZ
23#undef SAMPLE_RATE_HZ
24#endif
25
26static const size_t SAMPLE_RATE_HZ = 16000;
27
28static const size_t RING_BUFFER_SAMPLES = 512 * SAMPLE_RATE_HZ / 1000; // 512 ms * 16 kHz/ 1000 ms
29static const size_t RING_BUFFER_SIZE = RING_BUFFER_SAMPLES * sizeof(int16_t);
30static const size_t SEND_BUFFER_SAMPLES = 32 * SAMPLE_RATE_HZ / 1000; // 32ms * 16kHz / 1000ms
31static const size_t SEND_BUFFER_SIZE = SEND_BUFFER_SAMPLES * sizeof(int16_t);
32static const size_t RECEIVE_SIZE = 1024;
33// Stall watchdog, re-armed from loop() while playback runs. Waiting for playback
34// to start uses tts_playback_start_timeout_ instead (home-assistant-voice-pe#621).
35static constexpr uint32_t PLAYBACK_STALL_TIMEOUT_MS = 2000;
36
37static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE;
38
39// If one microphone channel keeps producing audio while another configured channel produces none for this
40// long, treat the silent channel as failed and stop the stream. A working microphone exposes a chunk every
41// SEND_BUFFER_SAMPLES (32 ms), so this is far longer than any legitimate gap between chunks.
42static const uint32_t AUDIO_CHANNEL_STALL_TIMEOUT_MS = 2000;
43
45
47 this->mic_source_->add_data_callback([this](const microphone::AudioCapture &capture) {
48 std::shared_ptr<ring_buffer::RingBuffer> temp_ring_buffer = this->ring_buffer_.lock();
49 if (temp_ring_buffer != nullptr) {
50 temp_ring_buffer->write((void *) capture.data.data(), capture.data.size());
51 }
52 });
53
54 // Second microphone channel
55 if (this->mic_source2_ != nullptr) {
56 this->mic_source2_->add_data_callback([this](const microphone::AudioCapture &capture) {
57 std::shared_ptr<ring_buffer::RingBuffer> temp_ring_buffer = this->ring_buffer2_.lock();
58 if (temp_ring_buffer != nullptr) {
59 temp_ring_buffer->write((void *) capture.data.data(), capture.data.size());
60 }
61 });
62 }
63
64#ifdef USE_MEDIA_PLAYER
65 if (this->media_player_ != nullptr) {
67 switch (state) {
70 // State changed to announcing after receiving the url
72 }
73 break;
74 default:
76 // No longer announcing the TTS response
78 }
79 break;
80 }
81 });
82 }
83#endif
84}
85
87
89#if USE_NETWORK_IPV4
90 this->socket_ = socket::socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
91#else
92 this->socket_ = socket::socket_ip(SOCK_DGRAM, IPPROTO_IP);
93#endif
94 if (this->socket_ == nullptr) {
95 ESP_LOGE(TAG, "Could not create socket");
96 this->mark_failed();
97 return false;
98 }
99 int enable = 1;
100 int err = this->socket_->setsockopt(SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));
101 if (err != 0) {
102 ESP_LOGW(TAG, "Socket unable to set reuseaddr: errno %d", err);
103 // we can still continue
104 }
105 err = this->socket_->setblocking(false);
106 if (err != 0) {
107 ESP_LOGE(TAG, "Socket unable to set nonblocking mode: errno %d", err);
108 this->mark_failed();
109 return false;
110 }
111
112#ifdef USE_SPEAKER
113 if (this->speaker_ != nullptr) {
114 struct sockaddr_storage server;
115
116 socklen_t sl = socket::set_sockaddr_any((struct sockaddr *) &server, sizeof(server), 6055);
117 if (sl == 0) {
118 ESP_LOGE(TAG, "Socket unable to set sockaddr: errno %d", errno);
119 this->mark_failed();
120 return false;
121 }
122
123 err = this->socket_->bind((struct sockaddr *) &server, sizeof(server));
124 if (err != 0) {
125 ESP_LOGE(TAG, "Socket unable to bind: errno %d", errno);
126 this->mark_failed();
127 return false;
128 }
129 }
130#endif
131 this->udp_socket_running_ = true;
132 return true;
133}
134
136#ifdef USE_SPEAKER
137 if ((this->speaker_ != nullptr) && (this->speaker_buffer_ == nullptr)) {
138 RAMAllocator<uint8_t> speaker_allocator;
139 this->speaker_buffer_ = speaker_allocator.allocate(SPEAKER_BUFFER_SIZE);
140 if (this->speaker_buffer_ == nullptr) {
141 ESP_LOGW(TAG, "Could not allocate speaker buffer");
142 return false;
143 }
144 }
145#endif
146
147 if (this->audio_source_ == nullptr) {
148 std::shared_ptr<ring_buffer::RingBuffer> temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE);
149 if (temp_ring_buffer == nullptr) {
150 ESP_LOGE(TAG, "Could not allocate ring buffer");
151 return false;
152 }
153 // Zero-copy source that reads directly from the ring buffer; frame-aligned to never split an int16 sample.
154 this->audio_source_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t));
155 if (this->audio_source_ == nullptr) {
156 ESP_LOGE(TAG, "Could not allocate audio source");
157 return false;
158 }
159 this->ring_buffer_ = temp_ring_buffer;
160 }
161
162 // Second microphone channel
163 if ((this->mic_source2_ != nullptr) && (this->audio_source2_ == nullptr)) {
164 std::shared_ptr<ring_buffer::RingBuffer> temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE);
165 if (temp_ring_buffer == nullptr) {
166 ESP_LOGE(TAG, "Could not allocate second ring buffer");
167 return false;
168 }
169 this->audio_source2_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t));
170 if (this->audio_source2_ == nullptr) {
171 ESP_LOGE(TAG, "Could not allocate second audio source");
172 return false;
173 }
174 this->ring_buffer2_ = temp_ring_buffer;
175 }
176
177 return true;
178}
179
181 if (this->audio_source_ != nullptr) {
182 this->audio_source_->clear_buffered_data();
183 }
184
185 // Second microphone channel
186 if (this->audio_source2_ != nullptr) {
187 this->audio_source2_->clear_buffered_data();
188 }
189
190 // Reset the multi-channel stall watchdog (see audio_channel_stall_start_).
192
193#ifdef USE_SPEAKER
194 if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) {
195 memset(this->speaker_buffer_, 0, SPEAKER_BUFFER_SIZE);
196
197 this->speaker_buffer_size_ = 0;
198 this->speaker_buffer_index_ = 0;
199 this->speaker_bytes_received_ = 0;
200 }
201#endif
202}
203
205 // Destroying each source releases its ring buffer; the matching weak_ptr then expires automatically.
206 this->audio_source_.reset();
207
208 // Second microphone channel
209 this->audio_source2_.reset();
210
211#ifdef USE_SPEAKER
212 if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) {
213 RAMAllocator<uint8_t> speaker_deallocator;
214 speaker_deallocator.deallocate(this->speaker_buffer_, SPEAKER_BUFFER_SIZE);
215 this->speaker_buffer_ = nullptr;
216 }
217#endif
218}
219
221 this->conversation_id_ = "";
222 ESP_LOGD(TAG, "reset conversation ID");
223}
224
226 // Both microphone channels are sent together, if configured. Home Assistant feeds one of the
227 // channels to its speech-to-text stream and treats an empty payload on that channel as
228 // end-of-stream, and the device cannot know which channel it picked, so only send once every
229 // configured channel has audio exposed, and always send them together. We don't target any
230 // particular message size: Home Assistant re-chunks the audio, and each fill() exposes at most
231 // SEND_BUFFER_SIZE bytes.
232 while (true) {
233 // fill() exposes a new chunk, or returns 0 if a previous chunk is still exposed; available()
234 // reports the currently exposed bytes either way.
235 this->audio_source_->fill(0);
236 size_t available = this->audio_source_->available();
237 size_t available2 = 0;
238 if (this->audio_source2_ != nullptr) {
239 this->audio_source2_->fill(0);
240 available2 = this->audio_source2_->available();
241 }
242
243 const bool channel_empty = (available == 0);
244 const bool channel2_empty = (this->audio_source2_ != nullptr) && (available2 == 0);
245 if (channel_empty || channel2_empty) {
246 // A configured channel has no audio yet, so keep any chunk exposed on the other channel for the
247 // next pass rather than sending an empty payload.
248 this->handle_channel_stall_(available, available2);
249 break;
250 }
251
252 // Both channels have audio exposed; clear any in-progress stall timer.
254
256 // Zero-copy: send_message() copies the data out before we consume it.
257 msg.data = this->audio_source_->data();
258 msg.data_len = available;
259 if (this->audio_source2_ != nullptr) {
260 msg.data2 = this->audio_source2_->data();
261 msg.data2_len = available2;
262 }
263
264 if (!this->api_client_->send_message(msg)) {
265 // Keep the chunk exposed and retry next pass, the same shape as
266 // APIConnection::try_send_camera_image_(): the slice is only lost if
267 // the ring buffer overflows before the TCP buffer clears, instead of
268 // on every refusal. The api layer already reports the refusal at V.
269 return;
270 }
271
272 this->audio_source_->consume(available);
273 if (this->audio_source2_ != nullptr) {
274 this->audio_source2_->consume(available2);
275 }
276 }
277}
278
279void VoiceAssistant::handle_channel_stall_(size_t available, size_t available2) {
280 // Called when at least one configured channel has no audio exposed. When one channel has data and the
281 // other does not, watch how long the empty channel stays starved: Home Assistant has no stream timeout
282 // and would never tell us to stop, so a channel that fails outright would otherwise hang streaming
283 // forever with the live channel's chunk held. Stop the stream with an error after a prolonged imbalance.
284 if ((available == 0) && (available2 == 0)) {
285 // Both channels are idle (no audio buffered yet); normal, not a stalled channel.
287 return;
288 }
289
291 if (this->audio_channel_stall_start_ == 0) {
292 this->audio_channel_stall_start_ = now;
293 } else if ((now - this->audio_channel_stall_start_) >= AUDIO_CHANNEL_STALL_TIMEOUT_MS) {
294 ESP_LOGW(TAG, "Mic channel %d stalled, stopping stream", (available == 0) ? 0 : 1);
296 this->signal_stop_();
298 this->defer([this]() {
299 this->error_trigger_.trigger("mic-channel-stalled", "A microphone channel stopped producing audio");
300 });
301 }
302}
303
305 if (this->api_client_ == nullptr && this->state_ != State::IDLE && this->state_ != State::STOP_MICROPHONE &&
307 if (this->mic_source_->is_running() || (this->mic_source2_ && this->mic_source2_->is_running()) ||
308 this->state_ == State::STARTING_MICROPHONE) {
310 } else {
312 }
313 this->continuous_ = false;
314 this->signal_stop_();
315 this->clear_buffers_();
316 return;
317 }
318 switch (this->state_) {
319 case State::IDLE: {
320 if (this->continuous_ && this->desired_state_ == State::IDLE) {
321 this->idle_trigger_.trigger();
323 } else {
324 this->deallocate_buffers_();
325 }
326 break;
327 }
329 ESP_LOGD(TAG, "Starting Microphone");
330 if (!this->allocate_buffers_()) {
331 this->status_set_error(LOG_STR("Failed to allocate buffers"));
332 return;
333 }
334 if (this->status_has_error()) {
335 this->status_clear_error();
336 }
337 this->clear_buffers_();
338
339 this->mic_source_->start();
340 if (this->mic_source2_) {
341 this->mic_source2_->start();
342 }
344 break;
345 }
347 if (this->mic_source_->is_running() && (!this->mic_source2_ || this->mic_source2_->is_running())) {
348 this->set_state_(this->desired_state_);
349 }
350 break;
351 }
353 ESP_LOGD(TAG, "Requesting start");
354 uint32_t flags = 0;
355 if (!this->continue_conversation_ && this->use_wake_word_)
357 if (this->silence_detection_)
361 audio_settings.auto_gain = this->auto_gain_;
362 audio_settings.volume_multiplier = this->volume_multiplier_;
363
365 msg.start = true;
367 msg.flags = flags;
368 msg.audio_settings = std::move(audio_settings);
370
371 // Reset media player state tracking
372#ifdef USE_MEDIA_PLAYER
373 if (this->media_player_ != nullptr) {
375 }
376#endif
377
378 if (this->api_client_ == nullptr || !this->api_client_->send_message(msg)) {
379 ESP_LOGW(TAG, "Could not request start");
380 this->error_trigger_.trigger("not-connected", "Could not request start");
381 this->continuous_ = false;
383 break;
384 }
386 this->set_timeout(RESET_CONVERSATION_TIMEOUT_ID, this->conversation_timeout_,
387 [this]() { this->reset_conversation_id(); });
388 break;
389 }
391 break; // State changed when udp server port received
392 }
394 if (this->audio_mode_ == AUDIO_MODE_API) {
395 this->stream_api_audio_();
396 } else {
397 // UDP (will eventually be deprecated)
398 // Only the primary microphone channel is used
399 while (true) {
400 this->audio_source_->fill(0);
401 size_t available = this->audio_source_->available();
402 if (available == 0) {
403 break;
404 }
405 if (!this->udp_socket_running_) {
406 if (!this->start_udp_socket_()) {
408 break;
409 }
410 }
411 this->socket_->sendto(this->audio_source_->data(), available, 0, (struct sockaddr *) &this->dest_addr_,
412 sizeof(this->dest_addr_));
413 this->audio_source_->consume(available);
414 }
415 } // audio mode
416 break;
417 }
419 // Check both microphone channels
420 bool is_running = this->mic_source_->is_running();
421 bool is_running2 = false;
422 if (this->mic_source2_) {
423 is_running2 = this->mic_source2_->is_running();
424 }
425 if (is_running || is_running2) {
426 if (is_running) {
427 this->mic_source_->stop();
428 }
429 if (is_running2) {
430 this->mic_source2_->stop();
431 }
433 } else {
434 this->set_state_(this->desired_state_);
435 }
436 break;
437 }
439 // Check both microphone channels
440 bool is_stopped = this->mic_source_->is_stopped();
441 bool is_stopped2 = true;
442 if (this->mic_source2_) {
443 is_stopped2 = this->mic_source2_->is_stopped();
444 }
445 if (is_stopped && is_stopped2) {
446 this->set_state_(this->desired_state_);
447 }
448 break;
449 }
451 break; // State changed by events
452 }
454 bool playing = false;
455#ifdef USE_SPEAKER
456 if (this->speaker_ != nullptr) {
457 ssize_t received_len = 0;
458 if (this->audio_mode_ == AUDIO_MODE_UDP) {
459 if (this->speaker_buffer_index_ + RECEIVE_SIZE < SPEAKER_BUFFER_SIZE) {
460 received_len = this->socket_->read(this->speaker_buffer_ + this->speaker_buffer_index_, RECEIVE_SIZE);
461 if (received_len > 0) {
462 this->speaker_buffer_index_ += received_len;
463 this->speaker_buffer_size_ += received_len;
464 this->speaker_bytes_received_ += received_len;
465 }
466 } else {
467 ESP_LOGD(TAG, "Receive buffer full");
468 }
469 }
470 // Build a small buffer of audio before sending to the speaker
471 bool end_of_stream = this->stream_ended_ && (this->audio_mode_ == AUDIO_MODE_API || received_len < 0);
472 if (this->speaker_bytes_received_ > RECEIVE_SIZE * 4 || end_of_stream)
473 this->write_speaker_();
474 if (this->wait_for_stream_end_) {
475 this->cancel_timeout(PLAYBACK_TIMEOUT_ID);
476 if (end_of_stream) {
477 ESP_LOGD(TAG, "End of audio stream received");
478 this->cancel_timeout(SPEAKER_IDLE_TIMEOUT_ID);
480 }
481 break; // We dont want to timeout here as the STREAM_END event will take care of that.
482 }
483 playing = this->speaker_->is_running();
484 }
485#endif
486#ifdef USE_MEDIA_PLAYER
487 if (this->media_player_ != nullptr) {
489
492 this->cancel_timeout(PLAYBACK_TIMEOUT_ID);
493 ESP_LOGD(TAG, "Announcement finished playing");
495
497 msg.success = true;
498 if (!this->api_client_->send_message(msg)) {
499 API_LOG_MSG_DROPPED(TAG, "Announce-finished");
500 }
501 break;
502 }
503 }
504#endif
505 if (playing) {
506 // Playback is running: short watchdog against a stalled player.
507 this->start_playback_timeout_(PLAYBACK_STALL_TIMEOUT_MS);
508 }
509 break;
510 }
512#ifdef USE_SPEAKER
513 if (this->speaker_ != nullptr) {
514 if (this->speaker_buffer_size_ > 0) {
515 this->write_speaker_();
516 break;
517 }
518 if (this->speaker_->has_buffered_data() || this->speaker_->is_running()) {
519 break;
520 }
521 ESP_LOGD(TAG, "Speaker has finished outputting all audio");
522 this->speaker_->stop();
523 this->cancel_timeout(SPEAKER_IDLE_TIMEOUT_ID);
524 this->cancel_timeout(PLAYBACK_TIMEOUT_ID);
525
526 this->clear_buffers_();
527
528 this->wait_for_stream_end_ = false;
529 this->stream_ended_ = false;
530
532 }
533#endif
534#ifdef USE_MEDIA_PLAYER
535 // Counterpart to the speaker checks above: while the media player is
536 // still announcing our response, the microphone must stay shut, otherwise
537 // the device records its own response and answers it. Only our own
538 // response counts; an unrelated announcement does not hold this state.
539 if ((this->media_player_ != nullptr) &&
541 break;
542 }
543 // The start timeout fired before playback began: skip the follow-up turn, so a late start
544 // cannot play into an open microphone.
545 if ((this->media_player_ != nullptr) &&
548 break;
549 }
550#endif
551 if (this->continue_conversation_) {
553 } else {
555 }
556 break;
557 }
558 default:
559 break;
560 }
561}
562
563#ifdef USE_SPEAKER
565 if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) {
566 if (this->speaker_buffer_size_ > 0) {
567 size_t write_chunk = std::min<size_t>(this->speaker_buffer_size_, 4 * 1024);
568 size_t written = this->speaker_->play(this->speaker_buffer_, write_chunk);
569 if (written > 0) {
570 memmove(this->speaker_buffer_, this->speaker_buffer_ + written, this->speaker_buffer_size_ - written);
573 this->set_timeout(SPEAKER_IDLE_TIMEOUT_ID, 5000, [this]() { this->speaker_->stop(); });
574 } else {
575 ESP_LOGV(TAG, "Speaker buffer full, trying again next loop");
576 }
577 }
578 }
579}
580#endif
581
583 if (!subscribe) {
584 if (this->api_client_ == nullptr || client != this->api_client_) {
585 ESP_LOGE(TAG, "Client attempting to unsubscribe that is not the current API Client");
586 return;
587 }
588 this->api_client_ = nullptr;
590 return;
591 }
592
593 if (this->api_client_ != nullptr) {
594 char current_peername[socket::SOCKADDR_STR_LEN];
595 char new_peername[socket::SOCKADDR_STR_LEN];
596 ESP_LOGE(TAG,
597 "Multiple API Clients attempting to connect to Voice Assistant\n"
598 " Current client: %s (%s)\n"
599 " New client: %s (%s)",
600 this->api_client_->get_name(), this->api_client_->get_peername_to(current_peername), client->get_name(),
601 client->get_peername_to(new_peername));
602 return;
603 }
604
605 this->api_client_ = client;
607}
608
609static const LogString *voice_assistant_state_to_string(State state) {
610 switch (state) {
611 case State::IDLE:
612 return LOG_STR("IDLE");
614 return LOG_STR("START_MICROPHONE");
616 return LOG_STR("STARTING_MICROPHONE");
618 return LOG_STR("WAIT_FOR_VAD");
620 return LOG_STR("WAITING_FOR_VAD");
622 return LOG_STR("START_PIPELINE");
624 return LOG_STR("STARTING_PIPELINE");
626 return LOG_STR("STREAMING_MICROPHONE");
628 return LOG_STR("STOP_MICROPHONE");
630 return LOG_STR("STOPPING_MICROPHONE");
632 return LOG_STR("AWAITING_RESPONSE");
634 return LOG_STR("STREAMING_RESPONSE");
636 return LOG_STR("RESPONSE_FINISHED");
637 default:
638 return LOG_STR("UNKNOWN");
639 }
640};
641
643 State old_state = this->state_;
644 this->state_ = state;
645 ESP_LOGD(TAG, "State changed from %s to %s", LOG_STR_ARG(voice_assistant_state_to_string(old_state)),
646 LOG_STR_ARG(voice_assistant_state_to_string(state)));
647}
648
650 this->set_state_(state);
651 this->desired_state_ = desired_state;
652 ESP_LOGD(TAG, "Desired state set to %s", LOG_STR_ARG(voice_assistant_state_to_string(desired_state)));
653}
654
657 // loop() moves to the desired state once the microphone has stopped
658 this->desired_state_ = state;
659 ESP_LOGD(TAG, "Desired state set to %s", LOG_STR_ARG(voice_assistant_state_to_string(state)));
660 } else if (state != this->state_) {
661 this->set_state_(state, state);
662 }
663}
664
666 ESP_LOGE(TAG, "Failed to start server. See Home Assistant logs for more details.");
667 this->error_trigger_.trigger("failed-to-start", "Failed to start server. See Home Assistant logs for more details.");
669}
670
672 if (this->state_ != State::STARTING_PIPELINE) {
673 this->signal_stop_();
674 return;
675 }
676
677 ESP_LOGD(TAG, "Client started, streaming microphone");
679
680 // Both microphone channels
681 if (this->mic_source_->is_running() && (!this->mic_source2_ || this->mic_source2_->is_running())) {
683 } else {
685 }
686}
687
688void VoiceAssistant::start_streaming(struct sockaddr_storage *addr, uint16_t port) {
689 if (this->state_ != State::STARTING_PIPELINE) {
690 this->signal_stop_();
691 return;
692 }
693
694 ESP_LOGD(TAG, "Client started, streaming microphone");
696
697 if (this->mic_source2_ != nullptr) {
698 ESP_LOGW(TAG, "UDP audio mode does not support a second microphone channel; only the primary will be streamed");
699 }
700
701 memcpy(&this->dest_addr_, addr, sizeof(this->dest_addr_));
702 switch (this->dest_addr_.ss_family) {
703#if USE_NETWORK_IPV4
704 case AF_INET:
705 ((struct sockaddr_in *) &this->dest_addr_)->sin_port = htons(port);
706 break;
707#endif
708#if LWIP_IPV6
709 case AF_INET6:
710 ((struct sockaddr_in6 *) &this->dest_addr_)->sin6_port = htons(port);
711 break;
712#endif
713 default:
714 ESP_LOGW(TAG, "Unknown address family: %d", this->dest_addr_.ss_family);
715 return;
716 }
717
718 // Only primary microphone channel over UDP
719 if (this->mic_source_->is_running()) {
721 } else {
723 }
724}
725
726void VoiceAssistant::request_start(bool continuous, bool silence_detection) {
727 if (this->api_client_ == nullptr) {
728 ESP_LOGE(TAG, "No API client connected");
730 this->continuous_ = false;
731 return;
732 }
733 if (this->state_ == State::IDLE) {
734 this->continuous_ = continuous;
735 this->silence_detection_ = silence_detection;
736
738 }
739}
740
742 this->continuous_ = false;
743 this->continue_conversation_ = false;
744
745 switch (this->state_) {
746 case State::IDLE:
747 break;
754 break;
757 this->signal_stop_();
759 break;
762 // A response queued behind the microphone shutdown is cancelled like a playing one and keeps its
763 // state, so it finishes through the same path (playback timeout, stream end) once the microphone
764 // has stopped instead of leaving those behind for the next request
766 this->cancel_response_();
767 } else {
769 }
770 break;
772 this->signal_stop_();
773 break;
775 this->cancel_response_();
776 break;
778#ifdef USE_MEDIA_PLAYER
779 // loop() holds this state while our response is still being announced;
780 // stop it the same way STREAMING_RESPONSE does.
781 if ((this->media_player_ != nullptr) &&
783 this->cancel_response_();
784 }
785#endif
786 break; // Let the incoming audio stream finish then it will go to idle.
787 }
788}
789
791#ifdef USE_MEDIA_PLAYER
792 // Stop any ongoing media player announcement
793 if (this->media_player_ != nullptr) {
794 this->media_player_->make_call()
796 .set_announcement(true)
797 .perform();
798 }
799 if (this->started_streaming_tts_) {
800 // Haven't reached the TTS_END stage, so send the stop signal to HA.
801 this->signal_stop_();
802 }
803#endif
804}
805
807 memset(&this->dest_addr_, 0, sizeof(this->dest_addr_));
808 if (this->api_client_ == nullptr) {
809 return;
810 }
811 ESP_LOGD(TAG, "Signaling stop");
813 msg.start = false;
814 if (!this->api_client_->send_message(msg)) {
815 API_LOG_MSG_DROPPED(TAG, "Stop request");
816 }
817}
818
820 this->set_timeout(PLAYBACK_TIMEOUT_ID, timeout_ms, [this]() {
821 this->cancel_timeout(SPEAKER_IDLE_TIMEOUT_ID);
823
824 if (this->api_client_ == nullptr)
825 return;
827 msg.success = true;
828 if (!this->api_client_->send_message(msg)) {
829 API_LOG_MSG_DROPPED(TAG, "Announce-finished");
830 }
831 });
832}
833
835 ESP_LOGD(TAG, "Event Type: %" PRId32, msg.event_type);
836 switch (msg.event_type) {
838 ESP_LOGD(TAG, "Assist Pipeline running");
839#ifdef USE_MEDIA_PLAYER
840 this->started_streaming_tts_ = false;
841 for (const auto &arg : msg.data) {
842 if (arg.name == "url") {
843 this->tts_response_url_ = arg.value;
844 }
845 }
846#endif
847 this->defer([this]() { this->start_trigger_.trigger(); });
848 break;
850 break;
852 ESP_LOGD(TAG, "Wake word detected");
853 this->defer([this]() { this->wake_word_detected_trigger_.trigger(); });
854 break;
855 }
857 ESP_LOGD(TAG, "STT started");
858 this->defer([this]() { this->listening_trigger_.trigger(); });
859 break;
861 // Providers that detect the end of speech themselves send no STT_VAD_END, so stop the microphone here
862 // if that event didn't already
863 if (this->state_ == State::STREAMING_MICROPHONE) {
865 }
866 std::string text;
867 for (const auto &arg : msg.data) {
868 if (arg.name == "text") {
869 text = arg.value;
870 }
871 }
872 if (text.empty()) {
873 ESP_LOGW(TAG, "No text in STT_END event");
874 return;
875 } else if (text.length() > 500) {
876 text.resize(497);
877 text += "...";
878 }
879 ESP_LOGD(TAG, "Speech recognised as: \"%s\"", text.c_str());
880 this->defer([this, text]() { this->stt_end_trigger_.trigger(text); });
881 break;
882 }
884 ESP_LOGD(TAG, "Intent started");
885 this->defer([this]() { this->intent_start_trigger_.trigger(); });
886 break;
888 ESP_LOGD(TAG, "Intent progress");
889 std::string tts_url_for_trigger;
890#ifdef USE_MEDIA_PLAYER
891 if (this->media_player_ != nullptr) {
892 for (const auto &arg : msg.data) {
893 if ((arg.name == "tts_start_streaming") && (arg.value == "1") && !this->tts_response_url_.empty()) {
895
897
898 this->started_streaming_tts_ = true;
900
901 tts_url_for_trigger = this->tts_response_url_;
902 this->tts_response_url_.clear(); // Reset streaming URL
904 }
905 }
906 }
907#endif
908 this->defer([this, tts_url_for_trigger]() { this->intent_progress_trigger_.trigger(tts_url_for_trigger); });
909 break;
910 }
912 for (const auto &arg : msg.data) {
913 if (arg.name == "conversation_id") {
914 this->conversation_id_ = arg.value;
915 } else if (arg.name == "continue_conversation") {
916 this->continue_conversation_ = (arg.value == "1");
917 }
918 }
919 this->defer([this]() { this->intent_end_trigger_.trigger(); });
920 break;
921 }
923 std::string text;
924 for (const auto &arg : msg.data) {
925 if (arg.name == "text") {
926 text = arg.value;
927 }
928 }
929 if (text.empty()) {
930 ESP_LOGW(TAG, "No text in TTS_START event");
931 return;
932 }
933 if (text.length() > 500) {
934 text.resize(497);
935 text += "...";
936 }
937 ESP_LOGD(TAG, "Response: \"%s\"", text.c_str());
938 this->defer([this, text]() {
939 this->tts_start_trigger_.trigger(text);
940#ifdef USE_SPEAKER
941 if (this->speaker_ != nullptr) {
942 this->speaker_->start();
943 }
944#endif
945 });
946 break;
947 }
949 std::string url;
950 for (const auto &arg : msg.data) {
951 if (arg.name == "url") {
952 url = arg.value;
953 }
954 }
955 if (url.empty()) {
956 ESP_LOGW(TAG, "No url in TTS_END event");
957 return;
958 }
959 ESP_LOGD(TAG, "Response URL: \"%s\"", url.c_str());
960 this->defer([this, url]() {
961#ifdef USE_MEDIA_PLAYER
962 if ((this->media_player_ != nullptr) && (!this->started_streaming_tts_)) {
964
966
968 }
969 this->started_streaming_tts_ = false; // Helps indicate reaching the TTS_END stage
970#endif
971 this->tts_end_trigger_.trigger(url);
972 });
973 // The intent progress stage may already have moved to streaming response
975 break;
976 }
978 ESP_LOGD(TAG, "Assist Pipeline ended");
979 if ((this->state_ == State::START_PIPELINE) || (this->state_ == State::STARTING_PIPELINE) ||
981 // Microphone is running, stop it
983 } else if (this->state_ == State::AWAITING_RESPONSE) {
984 // No TTS start event ("nevermind")
986 } else if (this->desired_state_ == State::AWAITING_RESPONSE) {
987 // No TTS start event while the microphone is still stopping
989 }
990 this->defer([this]() { this->end_trigger_.trigger(); });
991 break;
992 }
994 std::string code;
995 std::string message;
996 for (const auto &arg : msg.data) {
997 if (arg.name == "code") {
998 code = arg.value;
999 } else if (arg.name == "message") {
1000 message = arg.value;
1001 }
1002 }
1003 if (code == "wake-word-timeout" || code == "wake_word_detection_aborted" || code == "no_wake_word") {
1004 // Don't change state here since either the "tts-end" or "run-end" events will do it.
1005 return;
1006 } else if (code == "wake-provider-missing" || code == "wake-engine-missing") {
1007 // Wake word is not set up or not ready on Home Assistant so stop and do not retry until user starts again.
1008 this->defer([this, code, message]() {
1009 this->request_stop();
1010 this->error_trigger_.trigger(code, message);
1011 });
1012 return;
1013 }
1014 ESP_LOGE(TAG, "Error: %s - %s", code.c_str(), message.c_str());
1015 if (this->state_ != State::IDLE) {
1016 this->signal_stop_();
1018 }
1019 this->defer([this, code, message]() { this->error_trigger_.trigger(code, message); });
1020 break;
1021 }
1023#ifdef USE_SPEAKER
1024 if (this->speaker_ != nullptr) {
1025 this->wait_for_stream_end_ = true;
1026 ESP_LOGD(TAG, "TTS stream start");
1027 this->defer([this] { this->tts_stream_start_trigger_.trigger(); });
1028 }
1029#endif
1030 break;
1031 }
1033#ifdef USE_SPEAKER
1034 if (this->speaker_ != nullptr) {
1035 this->stream_ended_ = true;
1036 ESP_LOGD(TAG, "TTS stream end");
1037 }
1038#endif
1039 break;
1040 }
1042 ESP_LOGD(TAG, "Starting STT by VAD");
1043 this->defer([this]() { this->stt_vad_start_trigger_.trigger(); });
1044 break;
1046 ESP_LOGD(TAG, "STT by VAD end");
1048 this->defer([this]() { this->stt_vad_end_trigger_.trigger(); });
1049 break;
1050 default:
1051 ESP_LOGD(TAG, "Unhandled event type: %" PRId32, msg.event_type);
1052 break;
1053 }
1054}
1055
1057#ifdef USE_SPEAKER // We should never get to this function if there is no speaker anyway
1058 if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) {
1059 if (this->speaker_buffer_index_ + msg.data_len <= SPEAKER_BUFFER_SIZE) {
1060 memcpy(this->speaker_buffer_ + this->speaker_buffer_index_, msg.data, msg.data_len);
1061 this->speaker_buffer_index_ += msg.data_len;
1062 this->speaker_buffer_size_ += msg.data_len;
1063 this->speaker_bytes_received_ += msg.data_len;
1064 this->write_speaker_();
1065 ESP_LOGV(TAG, "Received audio: %u bytes from API", msg.data_len);
1066 } else {
1067 ESP_LOGE(TAG, "Cannot receive audio, buffer is full");
1068 }
1069 }
1070#endif
1071}
1072
1074 // Find existing timer or add a new one
1075 auto it = this->timers_.begin();
1076 for (; it != this->timers_.end(); ++it) {
1077 if (it->id == msg.timer_id)
1078 break;
1079 }
1080 if (it == this->timers_.end()) {
1081 this->timers_.push_back({});
1082 it = this->timers_.end() - 1;
1083 }
1084 it->id = msg.timer_id;
1085 it->name = msg.name;
1086 it->total_seconds = msg.total_seconds;
1087 it->seconds_left = msg.seconds_left;
1088 it->is_active = msg.is_active;
1089
1090 char timer_buf[Timer::TO_STR_BUFFER_SIZE];
1091 ESP_LOGD(TAG,
1092 "Timer Event\n"
1093 " Type: %" PRId32 "\n"
1094 " %s",
1095 msg.event_type, it->to_str(timer_buf));
1096
1097 switch (msg.event_type) {
1099 this->timer_started_trigger_.trigger(*it);
1100 break;
1102 this->timer_updated_trigger_.trigger(*it);
1103 break;
1105 this->timer_cancelled_trigger_.trigger(*it);
1106 this->timers_.erase(it);
1107 break;
1109 this->timer_finished_trigger_.trigger(*it);
1110 this->timers_.erase(it);
1111 break;
1112 }
1113
1114 if (this->timers_.empty()) {
1115 this->cancel_interval(TIMER_TICK_INTERVAL_ID);
1116 this->timer_tick_running_ = false;
1117 } else if (!this->timer_tick_running_) {
1118 this->set_interval(TIMER_TICK_INTERVAL_ID, 1000, [this]() { this->timer_tick_(); });
1119 this->timer_tick_running_ = true;
1120 }
1121}
1122
1124 for (auto &timer : this->timers_) {
1125 if (timer.is_active && timer.seconds_left > 0) {
1126 timer.seconds_left--;
1127 }
1128 }
1129 this->timer_tick_trigger_.trigger(this->timers_);
1130}
1131
1133#ifdef USE_MEDIA_PLAYER
1134 if (this->media_player_ != nullptr) {
1135 this->tts_start_trigger_.trigger(msg.text);
1136
1138
1139 if (!msg.preannounce_media_id.empty()) {
1141 }
1142 // Enqueueing a URL with an empty playlist will still play the file immediately
1143 this->media_player_->make_call()
1146 .set_announcement(true)
1147 .perform();
1149
1150 // Waiting for playback to start here too, not for it to finish.
1152
1153 if (this->continuous_) {
1155 } else {
1157 }
1158
1160 this->end_trigger_.trigger();
1161 }
1162#endif
1163}
1164
1165void VoiceAssistant::on_set_configuration(const std::vector<std::string> &active_wake_words) {
1166#ifdef USE_MICRO_WAKE_WORD
1167 if (this->micro_wake_word_) {
1168 // Disable all wake words first
1169 for (auto &model : this->micro_wake_word_->get_wake_words()) {
1170 model->disable();
1171 }
1172
1173 // Enable only active wake words
1174 for (const auto &ww_id : active_wake_words) {
1175 for (auto &model : this->micro_wake_word_->get_wake_words()) {
1176 if (model->get_id() == ww_id) {
1177 model->enable();
1178 ESP_LOGD(TAG, "Enabled wake word: %s (id=%s)", model->get_wake_word().c_str(), model->get_id().c_str());
1179 }
1180 }
1181 }
1182 }
1183#endif
1184};
1185
1187 this->config_.available_wake_words.clear();
1188 this->config_.active_wake_words.clear();
1189
1190#ifdef USE_MICRO_WAKE_WORD
1191 if (this->micro_wake_word_) {
1193
1194 for (auto &model : this->micro_wake_word_->get_wake_words()) {
1195 if (model->is_enabled()) {
1196 this->config_.active_wake_words.push_back(model->get_id());
1197 }
1198
1199 WakeWord wake_word;
1200 wake_word.id = model->get_id();
1201 wake_word.wake_word = model->get_wake_word();
1202 for (const auto &lang : model->get_trained_languages()) {
1203 wake_word.trained_languages.push_back(lang);
1204 }
1205 this->config_.available_wake_words.push_back(std::move(wake_word));
1206 }
1207 } else {
1208#endif
1209 // No microWakeWord
1211#ifdef USE_MICRO_WAKE_WORD
1212 }
1213#endif
1214
1215 return this->config_;
1216};
1217
1218VoiceAssistant *global_voice_assistant = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
1219
1220} // namespace esphome::voice_assistant
1221
1222#endif // USE_VOICE_ASSISTANT
uint32_t IRAM_ATTR HOT get_loop_component_start_time() const
Get the cached time in milliseconds from when the current component started its loop execution.
void mark_failed()
Mark this component as failed.
bool cancel_interval(const char *name)
Cancel an interval function.
Definition component.cpp:92
void status_clear_error()
Definition component.h:315
bool cancel_timeout(const char *name)
Cancel a timeout function.
void set_timeout(const char *name, uint32_t timeout, std::function< void()> &&f)
Set a timeout function with a const char* name.
Definition component.cpp:96
void defer(const char *name, std::function< void()> &&f)
Defer a callback to the next loop() call with a const char* name.
bool status_has_error() const
Definition component.h:295
void set_interval(const char *name, uint32_t interval, std::function< void()> &&f)
Set an interval function with a const char* name.
Definition component.cpp:88
An STL allocator that uses SPI or internal RAM.
Definition helpers.h:2396
void deallocate(T *p, size_t n)
Definition helpers.h:2453
T * allocate(size_t n)
Definition helpers.h:2423
StringRef is a reference to a string owned by something else.
Definition string_ref.h:31
constexpr bool empty() const
Definition string_ref.h:81
void trigger(const Ts &...x) ESPHOME_ALWAYS_INLINE
Inform the parent automation that the event has triggered.
Definition automation.h:557
const char * get_peername_to(std::span< char, socket::SOCKADDR_STR_LEN > buf) const
Get peer name (IP address) into caller-provided buffer, returns buf for convenience.
const char * get_name() const
bool send_message(const T &msg)
Returns false as soon as the TCP buffer is full.
enums::VoiceAssistantEvent event_type
Definition api_pb2.h:3017
std::vector< VoiceAssistantEventData > data
Definition api_pb2.h:3018
VoiceAssistantAudioSettings audio_settings
Definition api_pb2.h:2963
enums::VoiceAssistantTimerEvent event_type
Definition api_pb2.h:3064
static std::unique_ptr< RingBufferAudioSource > create(std::shared_ptr< ring_buffer::RingBuffer > ring_buffer, size_t max_fill_bytes, uint8_t alignment_bytes=1)
Creates a new ring-buffer-backed audio source after validating its parameters.
MediaPlayerCall & set_media_url(const std::string &url)
MediaPlayerCall & set_announcement(bool announce)
MediaPlayerCall & set_command(MediaPlayerCommand command)
void add_on_state_callback(F &&callback)
std::vector< WakeWordModel * > get_wake_words()
void add_data_callback(F &&data_callback)
Same callbacks as Microphone::add_data_callback; stream_info is the processed format.
static std::unique_ptr< RingBuffer > create(size_t len, MemoryPreference preference=MemoryPreference::EXTERNAL_FIRST)
virtual size_t play(const uint8_t *data, size_t length)=0
Plays the provided audio data.
bool is_running() const
Definition speaker.h:68
virtual bool has_buffered_data() const =0
virtual void start()=0
virtual void stop()=0
std::unique_ptr< socket::Socket > socket_
microphone::MicrophoneSource * mic_source2_
void on_timer_event(const api::VoiceAssistantTimerEventResponse &msg)
void cancel_response_()
Stop a playing or queued response and tell Home Assistant when the TTS stream is still open.
void on_audio(const api::VoiceAssistantAudio &msg)
std::unique_ptr< audio::RingBufferAudioSource > audio_source_
std::weak_ptr< ring_buffer::RingBuffer > ring_buffer2_
std::weak_ptr< ring_buffer::RingBuffer > ring_buffer_
media_player::MediaPlayer * media_player_
void start_playback_timeout_(uint32_t timeout_ms)
void client_subscription(api::APIConnection *client, bool subscribe)
MediaPlayerResponseState media_player_response_state_
void on_event(const api::VoiceAssistantEventResponse &msg)
Trigger< std::string, std::string > error_trigger_
void set_state_after_mic_stop_(State state)
Move to state, or only make it the desired state while the microphone is still stopping.
Trigger< const std::vector< Timer > & > timer_tick_trigger_
Trigger< std::string > intent_progress_trigger_
void on_announce(const api::VoiceAssistantAnnounceRequest &msg)
void request_start(bool continuous, bool silence_detection)
void handle_channel_stall_(size_t available, size_t available2)
std::unique_ptr< audio::RingBufferAudioSource > audio_source2_
microphone::MicrophoneSource * mic_source_
micro_wake_word::MicroWakeWord * micro_wake_word_
void on_set_configuration(const std::vector< std::string > &active_wake_words)
const LogString * message
Definition component.cpp:35
uint16_t flags
bool state
Definition fan.h:2
uint32_t socklen_t
Definition headers.h:109
__int64 ssize_t
Definition httplib.h:178
@ VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD
Definition api_pb2.h:269
@ VOICE_ASSISTANT_REQUEST_USE_VAD
Definition api_pb2.h:268
@ VOICE_ASSISTANT_TIMER_UPDATED
Definition api_pb2.h:292
@ VOICE_ASSISTANT_TIMER_STARTED
Definition api_pb2.h:291
@ VOICE_ASSISTANT_TIMER_FINISHED
Definition api_pb2.h:294
@ VOICE_ASSISTANT_TIMER_CANCELLED
Definition api_pb2.h:293
@ VOICE_ASSISTANT_WAKE_WORD_START
Definition api_pb2.h:282
@ VOICE_ASSISTANT_TTS_STREAM_END
Definition api_pb2.h:287
@ VOICE_ASSISTANT_STT_VAD_START
Definition api_pb2.h:284
@ VOICE_ASSISTANT_INTENT_PROGRESS
Definition api_pb2.h:288
@ VOICE_ASSISTANT_TTS_STREAM_START
Definition api_pb2.h:286
@ VOICE_ASSISTANT_WAKE_WORD_END
Definition api_pb2.h:283
constexpr float AFTER_CONNECTION
For components that should be initialized after a data connection (API/MQTT) is connected.
Definition component.h:57
std::unique_ptr< Socket > socket_ip(int type, int protocol)
Create a socket in the newest available IP domain (IPv6 or IPv4) of the given type and protocol.
std::unique_ptr< Socket > socket(int domain, int type, int protocol)
Create a socket of the given domain, type and protocol.
socklen_t set_sockaddr_any(struct sockaddr *addr, socklen_t addrlen, uint16_t port)
Set a sockaddr to the any address and specified port for the IP version used by socket_ip().
ESPHOME_LOG_TAG(TAG, "voice_assistant")
VoiceAssistant * global_voice_assistant
int written
Definition helpers.h:1340
Application App
Global storage of Application pointer - only one Application can exist.
static void uint32_t
std::span< const uint8_t > data
Only valid during the callback.
Definition microphone.h:25
std::vector< WakeWord > available_wake_words
std::vector< std::string > active_wake_words
static constexpr size_t TO_STR_BUFFER_SIZE
Buffer size for to_str() - sufficient for typical timer names.
std::vector< std::string > trained_languages
sa_family_t ss_family
Definition headers.h:104