ESPHome 2026.11.0b6
Loading...
Searching...
No Matches
esp32_camera_window.cpp
Go to the documentation of this file.
1#include "esp32_camera.h"
2
3#if defined(USE_ESP32) && defined(USE_ESP32_CAMERA_WINDOW)
4
5#include "esphome/core/log.h"
6
7#include <algorithm>
8
9namespace esphome::esp32_camera {
10
11ESPHOME_LOG_TAG(TAG, "esp32_camera");
12
13// OV2640 DSP-bank crop registers; a register number carries its bank in bit 8.
14static constexpr int OV2640_REG_HSIZE = 0x0051;
15static constexpr int OV2640_REG_VSIZE = 0x0052;
16static constexpr int OV2640_REG_XOFFL = 0x0053;
17static constexpr int OV2640_REG_YOFFL = 0x0054;
18static constexpr int OV2640_REG_VHYX = 0x0055;
19// A size register covers four pixels, so a crop is a multiple of four.
20static constexpr int OV2640_CROP_STEP = 4;
21// JPEG compresses whole 16x8 blocks; a frame that does not fill them is striped.
22static constexpr int OV2640_JPEG_BLOCK_WIDTH = 16;
23static constexpr int OV2640_JPEG_BLOCK_HEIGHT = 8;
24// Readout the OV2640 crops out of, in the order the driver enumerates it (not exported).
25static constexpr int OV2640_MODE_UXGA = 0;
26static constexpr int OV2640_MODE_SVGA = 1;
27static constexpr int OV2640_MODE_CIF = 2;
28
29// 16-bit registers of the OV3660 and OV5640: x is the low byte, y the high byte, hence two apart.
30static constexpr int OV_REG_START = 0x3800; // X_ADDR_ST, Y_ADDR_ST
31static constexpr int OV_REG_END = 0x3804; // X_ADDR_END, Y_ADDR_END
32static constexpr int OV_REG_TOTAL = 0x380C; // X_TOTAL_SIZE, Y_TOTAL_SIZE
33static constexpr int OV_REG_OFFSET = 0x3810; // X_OFFSET, Y_OFFSET
34static constexpr int OV_REG_PAIR_STEP = 2;
35static constexpr int OV_REG_MASK = 0xFFFF;
36
37// Paged SC window registers; set_reg() selects the page. Each axis has start, end and high bits.
38static constexpr int SC_REG_H_START = 0x0170; // H start, H end, high bits of both
39static constexpr int SC_REG_V_START = 0x0173; // V start, V end, high bits of both
40static constexpr int SC_REG_GROUP = 3; // start, end and high bits of both
41// The readout an SC window is placed in, and the high bits the sensor takes per edge.
42struct ScWindowFormat {
43 int area_width;
44 int area_height;
45 int edge_bits;
46};
47static constexpr ScWindowFormat SC101IOT_FORMAT{1280, 720, 4};
48static constexpr ScWindowFormat SC030IOT_FORMAT{640, 480, 2};
49
51
53static int round_down_at_least(int value, int step) { return std::max(value - value % step, step); }
54
56static bool is_window_supported(sensor_t *sensor) {
57 switch (sensor->id.PID) {
58 case OV2640_PID:
59 case OV3660_PID:
60 case OV5640_PID:
61 case SC101IOT_PID:
62 case SC030IOT_PID:
63 return true;
64 default:
65 return false;
66 }
67}
68
69static const char *sensor_name(sensor_t *sensor) {
70 const camera_sensor_info_t *info = esp_camera_sensor_get_info(&sensor->id);
71 return info != nullptr ? info->name : "unknown sensor";
72}
73
75static void restore_test_pattern(sensor_t *sensor) {
76 if (sensor->id.PID == OV2640_PID && sensor->status.colorbar)
77 sensor->set_colorbar(sensor, 1);
78}
79
81static bool restore_full_readout(sensor_t *sensor) {
82 const int ret = sensor->set_framesize(sensor, sensor->status.framesize);
83 restore_test_pattern(sensor);
84 if (ret != 0) {
85 ESP_LOGE(TAG, "Restoring the full readout failed with %d", ret);
86 return false;
87 }
88 return true;
89}
90
92struct SensorPair {
93 int x{0};
94 int y{0};
95};
96
98static bool read_ov_pair(sensor_t *sensor, int reg, SensorPair *value) {
99 const int x = sensor->get_reg(sensor, reg, OV_REG_MASK);
100 const int y = sensor->get_reg(sensor, reg + OV_REG_PAIR_STEP, OV_REG_MASK);
101 if (x < 0 || y < 0) {
102 ESP_LOGW(TAG, "Register 0x%04X could not be read", reg);
103 return false;
104 }
105 value->x = x;
106 value->y = y;
107 return true;
108}
109
112struct OvReadout {
113 SensorPair start; // first sensor pixel that is read out
114 SensorPair offset; // margin the driver leaves to its image pipeline on each side
115 SensorPair total; // line and frame timing the driver wrote, kept as it was
116 int span_x{0}; // sensor pixels that are read out
117 int span_y{0};
118 int factor{1}; // sensor pixels that make up one readout pixel
119
120 int visible_x() const { return this->span_x / this->factor - 2 * this->offset.x; }
121 int visible_y() const { return this->span_y / this->factor - 2 * this->offset.y; }
122};
123
125static bool read_ov_readout(sensor_t *sensor, OvReadout *readout) {
126 SensorPair end;
127 if (!read_ov_pair(sensor, OV_REG_START, &readout->start) || !read_ov_pair(sensor, OV_REG_END, &end) ||
128 !read_ov_pair(sensor, OV_REG_OFFSET, &readout->offset) || !read_ov_pair(sensor, OV_REG_TOTAL, &readout->total))
129 return false;
130
131 readout->span_x = end.x - readout->start.x + 1;
132 readout->span_y = end.y - readout->start.y + 1;
133 // A binned sensor reads two pixels as one and hands half as many of them to its image pipeline.
134 readout->factor = sensor->status.binning ? 2 : 1;
135 if (readout->span_x <= 0 || readout->span_y <= 0) {
136 ESP_LOGW(TAG, "The sensor reported a readout of %d by %d pixels", readout->span_x, readout->span_y);
137 return false;
138 }
139 return true;
140}
141
143static Window clamp_window(const Window &window, int frame_width, int frame_height) {
144 Window rect;
145 rect.width = std::min(window.width, frame_width);
146 rect.height = std::min(window.height, frame_height);
147 rect.offset_x = std::clamp(window.offset_x, 0, frame_width - rect.width);
148 rect.offset_y = std::clamp(window.offset_y, 0, frame_height - rect.height);
149 return rect;
150}
151
153static Window scale_window(const Window &window, int frame_width, int frame_height, int area_width, int area_height) {
154 Window rect;
155 rect.width = std::clamp(window.width * area_width / frame_width, 1, area_width);
156 rect.height = std::clamp(window.height * area_height / frame_height, 1, area_height);
157 // Rounding down can ask for a pixel past the far edge.
158 rect.offset_x = std::clamp(window.offset_x * area_width / frame_width, 0, area_width - rect.width);
159 rect.offset_y = std::clamp(window.offset_y * area_height / frame_height, 0, area_height - rect.height);
160 return rect;
161}
162
164static bool set_ov_window(sensor_t *sensor, const Window &window, int frame_width, int frame_height) {
165 const Window frame_window = clamp_window(window, frame_width, frame_height);
166 // A full-frame window is the full readout, which already has the timing and scaling the driver wants.
167 if (frame_window.width == frame_width && frame_window.height == frame_height)
168 return restore_full_readout(sensor);
169
170 // Start from the full readout, or the new window would nest inside the previous one.
171 if (!restore_full_readout(sensor))
172 return false;
173 OvReadout readout;
174 if (!read_ov_readout(sensor, &readout))
175 return false;
176
177 Window crop = scale_window(frame_window, frame_width, frame_height, readout.visible_x(), readout.visible_y());
178 // Mirror and flip reverse the readout, so the window is counted from the far edge of the sensor
179 if (sensor->status.hmirror)
180 crop.offset_x = readout.visible_x() - crop.offset_x - crop.width;
181 if (sensor->status.vflip)
182 crop.offset_y = readout.visible_y() - crop.offset_y - crop.height;
183
184 // The margin is part of the readout the sensor crops away again, so a crop includes one on each side.
185 const int crop_width = crop.width + 2 * readout.offset.x;
186 const int crop_height = crop.height + 2 * readout.offset.y;
187 const int start_x = readout.start.x + readout.factor * crop.offset_x;
188 const int start_y = readout.start.y + readout.factor * crop.offset_y;
189 const int end_x = start_x + readout.factor * crop_width - 1;
190 const int end_y = start_y + readout.factor * crop_height - 1;
191
192 // The sensor only scales down (OV3660 datasheet, OV5640 application note), so a crop smaller
193 // than the frame cannot fill it.
194 if (crop.width < frame_width || crop.height < frame_height) {
195 ESP_LOGE(TAG, "A %dx%d window leaves %dx%d pixels to fill the %dx%d frame; this sensor only scales down",
196 frame_window.width, frame_window.height, crop.width, crop.height, frame_width, frame_height);
197 return false;
198 }
199 // At the same size there is nothing to scale, and the driver leaves the scaler off then too.
200 const bool scale = crop.width > frame_width || crop.height > frame_height;
201
202 const int ret =
203 sensor->set_res_raw(sensor, start_x, start_y, end_x, end_y, readout.offset.x, readout.offset.y, readout.total.x,
204 readout.total.y, frame_width, frame_height, scale, sensor->status.binning);
205 if (ret != 0) {
206 ESP_LOGE(TAG, "Reading out %d,%d to %d,%d failed with %d", start_x, start_y, end_x, end_y, ret);
207 return false;
208 }
209
210 ESP_LOGI(TAG, "Window at %d,%d of %dx%d reads out %d,%d to %d,%d and fills the %dx%d frame", frame_window.offset_x,
211 frame_window.offset_y, frame_window.width, frame_window.height, start_x, start_y, end_x, end_y, frame_width,
212 frame_height);
213 return true;
214}
215
217static bool read_ov2640_crop(sensor_t *sensor, Window *crop) {
218 const int hsize = sensor->get_reg(sensor, OV2640_REG_HSIZE, 0xFF);
219 const int vsize = sensor->get_reg(sensor, OV2640_REG_VSIZE, 0xFF);
220 const int xoffl = sensor->get_reg(sensor, OV2640_REG_XOFFL, 0xFF);
221 const int yoffl = sensor->get_reg(sensor, OV2640_REG_YOFFL, 0xFF);
222 const int vhyx = sensor->get_reg(sensor, OV2640_REG_VHYX, 0xFF);
223 if (hsize < 0 || vsize < 0 || xoffl < 0 || yoffl < 0 || vhyx < 0) {
224 ESP_LOGW(TAG, "The crop registers of the OV2640 could not be read");
225 return false;
226 }
227 // The top bit of both sizes and both offsets shares VHYX.
228 crop->width = (hsize | ((vhyx & 0x08) << 5)) * OV2640_CROP_STEP;
229 crop->height = (vsize | ((vhyx & 0x80) << 1)) * OV2640_CROP_STEP;
230 crop->offset_x = xoffl | ((vhyx & 0x07) << 8);
231 crop->offset_y = yoffl | ((vhyx & 0x70) << 4);
232 if (crop->width < OV2640_CROP_STEP || crop->height < OV2640_CROP_STEP) {
233 ESP_LOGW(TAG, "The OV2640 reported a crop of %d by %d pixels", crop->width, crop->height);
234 return false;
235 }
236 return true;
237}
238
240static int ov2640_mode_for(framesize_t framesize) {
241 if (framesize <= FRAMESIZE_CIF)
242 return OV2640_MODE_CIF;
243 if (framesize <= FRAMESIZE_SVGA)
244 return OV2640_MODE_SVGA;
245 return OV2640_MODE_UXGA;
246}
247
249static bool window_output_supported(sensor_t *sensor) {
250 return sensor->id.PID != OV2640_PID || sensor->pixformat == PIXFORMAT_JPEG;
251}
252
254static bool set_ov2640_window(sensor_t *sensor, const Window &window, int frame_width, int frame_height) {
255 Window frame_window = clamp_window(window, frame_width, frame_height);
256 // Round the window itself, not just the size it is scaled to, so crop and frame stay aligned.
257 if (sensor->pixformat == PIXFORMAT_JPEG) {
258 frame_window.width = round_down_at_least(frame_window.width, OV2640_JPEG_BLOCK_WIDTH);
259 frame_window.height = round_down_at_least(frame_window.height, OV2640_JPEG_BLOCK_HEIGHT);
260 // A window smaller than one block grows to it, which can push it past the far edge.
261 frame_window = clamp_window(frame_window, frame_width, frame_height);
262 }
263 // A full-frame window is the full readout.
264 if (frame_window.width == frame_width && frame_window.height == frame_height)
265 return restore_full_readout(sensor);
266
267 // Start from the full readout, or the new window would nest inside the previous one.
268 if (!restore_full_readout(sensor))
269 return false;
270 Window crop;
271 if (!read_ov2640_crop(sensor, &crop))
272 return false;
273
274 const Window area = scale_window(frame_window, frame_width, frame_height, crop.width, crop.height);
275
276 // The crop is a multiple of four pixels, counted from the corner of the crop in use.
277 const int offset_x = std::clamp(area.offset_x - area.offset_x % OV2640_CROP_STEP, 0, crop.width - OV2640_CROP_STEP);
278 const int offset_y = std::clamp(area.offset_y - area.offset_y % OV2640_CROP_STEP, 0, crop.height - OV2640_CROP_STEP);
279 int width = std::min(area.width, crop.width - offset_x);
280 int height = std::min(area.height, crop.height - offset_y);
281 width = round_down_at_least(width, OV2640_CROP_STEP);
282 height = round_down_at_least(height, OV2640_CROP_STEP);
283 // Rounding the size down can leave it past the far edge when the remainder is small.
284 width = std::min(width, crop.width - offset_x);
285 height = std::min(height, crop.height - offset_y);
286
287 // The crop is in readout pixels; the pipeline scales it to outputX/outputY (the window size),
288 // so a crop larger than the frame cannot overrun the buffers sized for it.
289 const int output_x = round_down_at_least(frame_window.width, OV2640_CROP_STEP);
290 const int output_y = round_down_at_least(frame_window.height, OV2640_CROP_STEP);
291 const int ret =
292 sensor->set_res_raw(sensor, ov2640_mode_for(sensor->status.framesize), 0, 0, 0, crop.offset_x + offset_x,
293 crop.offset_y + offset_y, width, height, output_x, output_y, false, false);
294 restore_test_pattern(sensor);
295 if (ret != 0) {
296 ESP_LOGE(TAG, "Cropping to offset %d,%d size %dx%d failed with %d", crop.offset_x + offset_x,
297 crop.offset_y + offset_y, width, height, ret);
298 return false;
299 }
300
301 ESP_LOGI(TAG, "Window at %d,%d of %dx%d crops to offset %d,%d size %dx%d of the %dx%d readout and is scaled to %dx%d",
302 frame_window.offset_x, frame_window.offset_y, frame_window.width, frame_window.height,
303 crop.offset_x + offset_x, crop.offset_y + offset_y, width, height, crop.width, crop.height, output_x,
304 output_y);
305 return true;
306}
307
311static bool write_sc_axis(sensor_t *sensor, int reg, int first, int last, int edge_bits) {
312 const int mask = (1 << edge_bits) - 1;
313 // The high bits of both edges share the third register of the group.
314 const int values[SC_REG_GROUP] = {first & 0xFF, last & 0xFF, ((first >> 8) & mask) | (((last >> 8) & mask) << 4)};
315 for (int offset = 0; offset < SC_REG_GROUP; offset++) {
316 const int ret = sensor->set_reg(sensor, reg + offset, 0xFF, values[offset]);
317 if (ret != 0) {
318 ESP_LOGE(TAG, "Writing register 0x%04X failed with %d", reg + offset, ret);
319 return false;
320 }
321 }
322 return true;
323}
324
326static bool set_sc_window(sensor_t *sensor, const Window &window, const ScWindowFormat &format) {
327 const Window area = clamp_window(window, format.area_width, format.area_height);
328
329 if (!write_sc_axis(sensor, SC_REG_H_START, area.offset_x, area.offset_x + area.width, format.edge_bits) ||
330 !write_sc_axis(sensor, SC_REG_V_START, area.offset_y, area.offset_y + area.height, format.edge_bits))
331 return false;
332
333 ESP_LOGI(TAG, "Window at %d,%d of %dx%d reads out %dx%d at %d,%d of the %dx%d sensor", window.offset_x,
334 window.offset_y, window.width, window.height, area.width, area.height, area.offset_x, area.offset_y,
335 format.area_width, format.area_height);
336 return true;
337}
338
340 if (this->window_.width == 0)
341 return;
342 this->set_window(this->window_.offset_x, this->window_.offset_y, this->window_.width, this->window_.height);
343}
344
345void ESP32Camera::dump_window_config_(sensor_t *sensor) {
346 ESP_LOGCONFIG(TAG,
347 " Sensor: %s\n"
348 " Window support: %s",
349 sensor_name(sensor), YESNO(this->supports_window()));
350 if (this->window_.width != 0) {
351 ESP_LOGCONFIG(TAG, " Window: offset %d,%d size %dx%d", this->window_.offset_x, this->window_.offset_y,
352 this->window_.width, this->window_.height);
353 }
354}
355
357static bool apply_window(sensor_t *sensor, const Window &window, const resolution_info_t &frame) {
358 switch (sensor->id.PID) {
359 case OV2640_PID:
360 return set_ov2640_window(sensor, window, frame.width, frame.height);
361 case OV3660_PID:
362 case OV5640_PID:
363 return set_ov_window(sensor, window, frame.width, frame.height);
364 case SC101IOT_PID:
365 case SC030IOT_PID: {
366 // These sensors read a window out as it is, so a window of another size would be read out
367 // at that size and no longer fill the frame the camera is set to.
368 if (window.width != frame.width || window.height != frame.height) {
369 ESP_LOGE(TAG,
370 "%s reads out a window as it is and cannot scale it, so a window has to be the size of "
371 "the %dx%d frame; %dx%d was asked for",
372 sensor_name(sensor), frame.width, frame.height, window.width, window.height);
373 return false;
374 }
375 const ScWindowFormat &format = sensor->id.PID == SC101IOT_PID ? SC101IOT_FORMAT : SC030IOT_FORMAT;
376 return set_sc_window(sensor, window, format);
377 }
378 default:
379 // window_sensor_() already rejected every other sensor.
380 return false;
381 }
382}
383
384bool ESP32Camera::set_window(int offset_x, int offset_y, int width, int height) {
385 sensor_t *sensor = this->window_sensor_();
386 if (sensor == nullptr)
387 return false;
388 // Lambda values skip the schema's range checks
389 if (width <= 0 || height <= 0 || offset_x < 0 || offset_y < 0) {
390 ESP_LOGE(TAG, "A window at %d,%d of %dx%d is not valid", offset_x, offset_y, width, height);
391 return false;
392 }
393 if (sensor->status.framesize >= FRAMESIZE_INVALID) {
394 ESP_LOGE(TAG, "The camera is not set to a valid frame size");
395 return false;
396 }
397 const resolution_info_t frame = resolution[sensor->status.framesize];
398 if (!window_output_supported(sensor)) {
399 ESP_LOGE(TAG, "%s can only read out a window with JPEG output; set pixel_format to JPEG", sensor_name(sensor));
400 return false;
401 }
402 const Window window{offset_x, offset_y, width, height};
403 if (apply_window(sensor, window, frame)) {
404 this->window_ = window;
405 return true;
406 }
407 // A failed window can leave the sensor half set up; put the previous window back, or the full readout
408 if (this->window_.width == 0 || !apply_window(sensor, this->window_, frame)) {
409 this->window_ = {};
410 restore_full_readout(sensor);
411 }
412 return false;
413}
414
416 sensor_t *sensor = this->window_sensor_();
417 if (sensor == nullptr)
418 return false;
419
420 // set_framesize() programs the full readout again; the next captures drain stale frames on their own.
421 if (!restore_full_readout(sensor))
422 return false;
423 this->window_ = {};
424
425 ESP_LOGI(TAG, "Window cleared, reading out the whole sensor");
426 return true;
427}
428
430 sensor_t *sensor = esp_camera_sensor_get();
431 return sensor != nullptr && is_window_supported(sensor) && window_output_supported(sensor);
432}
433
435 sensor_t *sensor = esp_camera_sensor_get();
436 if (sensor == nullptr) {
437 ESP_LOGE(TAG, "No sensor; the camera has not finished setting up");
438 return nullptr;
439 }
440 if (!is_window_supported(sensor)) {
441 ESP_LOGE(TAG, "%s does not support windows", sensor_name(sensor));
442 return nullptr;
443 }
444 return sensor;
445}
446
447} // namespace esphome::esp32_camera
448
449#endif // USE_ESP32 && USE_ESP32_CAMERA_WINDOW
bool set_window(int offset_x, int offset_y, int width, int height)
Read out the given rectangle of the frame, replacing any previous window.
bool reset_window()
Read out the whole sensor again; changing the frame size does the same.
bool supports_window() const
True when the sensor takes a window in the configured pixel format; meant for lambdas.
int ret
const char * format
Resolution resolution
Definition msa3xx.h:1
ESP32Camera::Window Window
ESPHOME_LOG_TAG(TAG, "esp32_camera")
uint32_t value
Definition hal.cpp:41
A rectangle, with (0, 0) at the top left, in the pixels of whatever it is measured against.
uint8_t end[39]
Definition sun_gtil2.cpp:17
uint16_t x
Definition tt21100.cpp:5
uint16_t y
Definition tt21100.cpp:6