ESPHome 2026.11.0b6
Loading...
Searching...
No Matches
resampler_microphone.cpp
Go to the documentation of this file.
2
3#ifdef USE_ESP32
4
6#include "esphome/core/log.h"
7
8#include <algorithm>
9
11
12ESPHOME_LOG_TAG(TAG, "resampler.microphone");
13
14// Duration of audio the resampler converts per step; longer source chunks are processed in several steps
15static constexpr uint32_t BUFFER_DURATION_MS = 16;
16
18 const audio::AudioStreamInfo input_stream_info = this->source_->get_audio_stream_info();
20 input_stream_info.get_channels(), this->target_sample_rate_);
21
22 // Allocate now for the expected source format; process_audio_ only sets up again if that format changes
23 if (!this->init_resampler_(input_stream_info)) {
24 this->mark_failed();
25 return;
26 }
27
28 this->source_->add_data_callback([this](const microphone::AudioCapture &capture) { this->process_audio_(capture); });
29
30 this->disable_loop();
31}
32
34 ESP_LOGCONFIG(TAG,
35 "Resampler Microphone:\n"
36 " Target Sample Rate: %" PRIu32 " Hz\n"
37 " Taps: %u\n"
38 " Filters: %u",
39 this->target_sample_rate_, this->taps_, this->filters_);
40}
41
43 if (this->is_failed() || this->active_listeners_ == UINT8_MAX)
44 return;
45 ++this->active_listeners_;
46 this->enable_loop();
47}
48
50 if (this->active_listeners_ == 0)
51 return;
52 --this->active_listeners_;
53 this->enable_loop();
54}
55
57 if (this->active_listeners_ == 0) {
58 if (this->state_ != microphone::STATE_STOPPED) {
59 this->source_->stop();
61 }
62 this->disable_loop();
63 return;
64 }
65
66 switch (this->state_) {
68 this->source_->start();
70 break;
72 if (this->source_->is_running()) {
74 }
75 break;
77 // Follow the source if it restarts, e.g. after a driver error
78 if (!this->source_->is_running()) {
80 }
81 break;
83 break;
84 }
85}
86
88 this->resampler_.reset();
89 this->resampler_ready_ = false;
90
91 if (input_stream_info.get_sample_rate() == this->target_sample_rate_) {
92 // The source already delivers the target sample rate, so its audio is passed through unchanged
93 this->output_buffer_.reserve(input_stream_info.ms_to_bytes(BUFFER_DURATION_MS));
94 this->input_stream_info_ = input_stream_info;
95 this->resampler_ready_ = true;
96 return true;
97 }
98
99 const audio::AudioStreamInfo output_stream_info(input_stream_info.get_bits_per_sample(),
100 input_stream_info.get_channels(), this->target_sample_rate_);
101
102 auto resampler = make_unique<PositionResampler>(input_stream_info.ms_to_samples(BUFFER_DURATION_MS),
103 output_stream_info.ms_to_samples(BUFFER_DURATION_MS));
104
105 esp_audio_libs::resampler::ResamplerConfiguration resample_config = {
106 .source_sample_rate = static_cast<float>(input_stream_info.get_sample_rate()),
107 .target_sample_rate = static_cast<float>(this->target_sample_rate_),
108 .source_bits_per_sample = input_stream_info.get_bits_per_sample(),
109 .target_bits_per_sample = input_stream_info.get_bits_per_sample(),
110 .channels = input_stream_info.get_channels(),
111 // Filters out frequencies above the new Nyquist limit when downsampling, to avoid aliasing
112 .use_pre_or_post_filter = this->target_sample_rate_ < input_stream_info.get_sample_rate(),
113 .subsample_interpolate = false, // Doubles the CPU load; more filters is a better alternative
114 .number_of_taps = this->taps_,
115 .number_of_filters = this->filters_,
116 };
117
118 if (!resampler->initialize(resample_config)) {
119 ESP_LOGE(TAG, "Not enough memory to resample");
120 return false;
121 }
122
123 this->output_buffer_.reserve(output_stream_info.ms_to_bytes(BUFFER_DURATION_MS));
124 this->resampler_ = std::move(resampler);
125 // Only set on success, so a failed set up is retried with the next chunk
126 this->input_stream_info_ = input_stream_info;
127 this->resampler_ready_ = true;
128 return true;
129}
130
132 const audio::AudioStreamInfo &input_stream_info = capture.stream_info;
133 if (input_stream_info != this->input_stream_info_) {
134 this->init_resampler_(input_stream_info);
135 }
136 if (!this->resampler_ready_) {
137 return;
138 }
139
140 if (this->resampler_ == nullptr) {
141 if (this->mute_state_) {
142 // The source's buffer is shared with its other consumers, so publish zeros from our own buffer, in pieces that
143 // fit its reserved size so muting never allocates
144 const uint32_t max_frames = input_stream_info.ms_to_frames(BUFFER_DURATION_MS);
145 const uint32_t total_frames = input_stream_info.bytes_to_frames(capture.data.size());
146 for (uint32_t done = 0; done < total_frames;) {
147 const uint32_t frames = std::min(total_frames - done, max_frames);
148 this->output_buffer_.assign(input_stream_info.frames_to_bytes(frames), 0);
149 const int64_t time_us =
150 capture.capture_time_us == 0 ? 0 : capture.capture_time_us + input_stream_info.frames_to_microseconds(done);
151 this->publish_audio_(this->output_buffer_, input_stream_info, time_us);
152 done += frames;
153 }
154 } else {
155 // Needs a protected forwarding method once data_callbacks_ is made private in 2027.5.0
156 this->data_callbacks_.call(microphone::AudioCapture{capture.data, input_stream_info, capture.capture_time_us});
157 }
158 return;
159 }
160
161 const size_t input_bytes_per_frame = input_stream_info.frames_to_bytes(1);
162 const uint32_t max_input_frames = input_stream_info.ms_to_frames(BUFFER_DURATION_MS);
163 // Both limits match the sizes the resampler's internal buffers were allocated with in init_resampler_
164 const uint32_t max_output_frames = this->audio_stream_info_.ms_to_frames(BUFFER_DURATION_MS);
165
166 const uint8_t *input = capture.data.data();
167 uint32_t input_frames = input_stream_info.bytes_to_frames(capture.data.size());
168 // Input frames of this chunk the resampler has taken in so far
169 int64_t frames_consumed = 0;
170 while (input_frames > 0) {
171 // Stays within the reserved capacity, so this never reallocates
172 this->output_buffer_.resize(this->audio_stream_info_.frames_to_bytes(max_output_frames));
173
174 // The resampler's internal buffers hold at most BUFFER_DURATION_MS of audio, so feed it in steps of that size.
175 // 0 dB keeps the microphone level that downstream detectors are tuned for; overshoot saturates instead of wrapping.
176 esp_audio_libs::resampler::ResamplerResults results = this->resampler_->resample(
177 input, this->output_buffer_.data(), std::min(input_frames, max_input_frames), max_output_frames, 0.0f);
178
179 input += results.frames_used * input_bytes_per_frame;
180 input_frames -= results.frames_used;
181 frames_consumed += results.frames_used;
182
183 if (results.frames_generated > 0) {
184 int64_t output_time_us = 0;
185 if (capture.capture_time_us != 0) {
186 // The resampler holds back half its taps of input, so its next output is near the input frame that far behind
187 // the last one taken in; its position gives the exact offset. The block's first output is one block duration
188 // before that. After a gap in the source, the few frames held back from before it are timed as if there were
189 // no gap.
190 const uint32_t input_rate = input_stream_info.get_sample_rate();
191 output_time_us = capture.capture_time_us + (frames_consumed - this->taps_ / 2) * 1000000 / input_rate +
192 static_cast<int64_t>(this->resampler_->get_position() * 1000000.0f / input_rate) -
193 static_cast<int64_t>(results.frames_generated) * 1000000 / this->target_sample_rate_;
194 }
195 this->output_buffer_.resize(this->audio_stream_info_.frames_to_bytes(results.frames_generated));
196 this->publish_audio_(this->output_buffer_, this->audio_stream_info_, output_time_us);
197 } else if (results.frames_used == 0) {
198 break; // No progress; drop the rest of the chunk instead of spinning
199 }
200 }
201}
202
203} // namespace esphome::resampler
204
205#endif // USE_ESP32
void mark_failed()
Mark this component as failed.
bool is_failed() const
Definition component.h:287
void enable_loop()
Enable this component's loop.
Definition component.h:261
void disable_loop()
Disable this component's loop.
size_t ms_to_bytes(uint32_t ms) const
Converts duration to bytes.
Definition audio.h:75
size_t frames_to_bytes(uint32_t frames) const
Converts frames to bytes.
Definition audio.h:53
uint8_t get_bits_per_sample() const
Definition audio.h:28
uint32_t frames_to_microseconds(uint32_t frames) const
Computes the duration, in microseconds, the given amount of frames represents.
Definition audio.cpp:25
uint32_t ms_to_samples(uint32_t ms) const
Converts duration to samples.
Definition audio.h:68
uint32_t bytes_to_frames(size_t bytes) const
Convert bytes to frames.
Definition audio.h:43
uint8_t get_channels() const
Definition audio.h:29
uint32_t ms_to_frames(uint32_t ms) const
Converts duration to frames.
Definition audio.h:63
uint32_t get_sample_rate() const
Definition audio.h:30
void publish_audio_(std::span< uint8_t > data, const audio::AudioStreamInfo &stream_info, int64_t capture_time_us=0)
While muted, data is zeroed first, so it must be a buffer the platform owns.
Definition microphone.h:98
DataCallbackManager data_callbacks_
Definition microphone.h:113
audio::AudioStreamInfo audio_stream_info_
Definition microphone.h:110
void add_data_callback(F &&data_callback)
Same callbacks as Microphone::add_data_callback; stream_info is the processed format.
audio::AudioStreamInfo get_audio_stream_info()
Gets the AudioStreamInfo of the data after processing.
bool init_resampler_(const audio::AudioStreamInfo &input_stream_info)
Sets up the resampler for the given input format.
microphone::MicrophoneSource * source_
void process_audio_(const microphone::AudioCapture &capture)
Resamples a chunk of source audio and passes it to the data callbacks. Source microphone task only.
std::unique_ptr< PositionResampler > resampler_
ESPHOME_LOG_TAG(TAG, "resampler.microphone")
static void uint32_t
int64_t capture_time_us
esp_timer_get_time() of the first frame, or 0 if unknown
Definition microphone.h:29
audio::AudioStreamInfo stream_info
Format of data.
Definition microphone.h:27
std::span< const uint8_t > data
Only valid during the callback.
Definition microphone.h:25