diff --git a/components/i2c/include/i2c.hpp b/components/i2c/include/i2c.hpp index d34edf695..546f416c4 100644 --- a/components/i2c/include/i2c.hpp +++ b/components/i2c/include/i2c.hpp @@ -578,6 +578,13 @@ class I2c : public BaseComponent { /// \return The active configuration. const Config &config() const { return config_; } + /// Get the native ESP-IDF I2C master bus handle. + /// \return The underlying i2c_master_bus_handle_t (nullptr if not initialized). + /// \note Provided so the bus can be shared with ESP-IDF drivers that require + /// the native handle (e.g. camera SCCB via esp_video). The bus remains + /// owned by this object. Only available with the new I2C master API. + i2c_master_bus_handle_t native_bus_handle() const { return master_bus_.native_handle(); } + /// Initialize the I2C bus. /// \param ec Error code populated on failure. void init(std::error_code &ec) { diff --git a/components/i2c/include/i2c_master.hpp b/components/i2c/include/i2c_master.hpp index b38311e19..a44f55ed8 100644 --- a/components/i2c/include/i2c_master.hpp +++ b/components/i2c/include/i2c_master.hpp @@ -265,6 +265,13 @@ class I2cMasterBus : public BaseComponent { /// @return True if device responds bool probe(uint16_t device_address, int32_t timeout_ms, std::error_code &ec); + /// @brief Get the native ESP-IDF I2C master bus handle + /// @return The underlying i2c_master_bus_handle_t (nullptr if not initialized) + /// @note This is provided so the bus can be shared with ESP-IDF drivers that + /// require the native handle (e.g. camera SCCB via esp_video). The bus + /// remains owned by this object. + i2c_master_bus_handle_t native_handle() const { return bus_handle_; } + protected: Config config_; i2c_master_bus_handle_t bus_handle_ = nullptr; diff --git a/components/m5stack-tab5/CMakeLists.txt b/components/m5stack-tab5/CMakeLists.txt index f0aeae866..7d91d55ad 100755 --- a/components/m5stack-tab5/CMakeLists.txt +++ b/components/m5stack-tab5/CMakeLists.txt @@ -1,6 +1,6 @@ idf_component_register( INCLUDE_DIRS "include" SRC_DIRS "src" - REQUIRES driver esp_driver_i2s esp_driver_ppa esp_driver_sdmmc esp_driver_spi esp_lcd fatfs base_component bmi270 codec display display_drivers gt911 i2c ina226 input_drivers interrupt pi4ioe5v rx8130ce st7123touch task touch + REQUIRES driver esp_driver_i2s esp_driver_ppa esp_driver_sdmmc esp_driver_spi esp_lcd esp_video esp_cam_sensor fatfs base_component bmi270 codec display display_drivers gt911 i2c ina226 input_drivers interrupt pi4ioe5v rx8130ce st7123touch task touch REQUIRED_IDF_TARGETS "esp32p4" ) diff --git a/components/m5stack-tab5/README.md b/components/m5stack-tab5/README.md index fa9ed6cba..021393552 100644 --- a/components/m5stack-tab5/README.md +++ b/components/m5stack-tab5/README.md @@ -21,9 +21,12 @@ The `espp::M5StackTab5` component provides a singleton hardware abstraction for - Hi-Fi recording and playback capabilities ### Camera -- SC2356 2MP camera (1600 × 1200) via MIPI-CSI -- HD video recording and image processing -- Support for edge-AI applications +- MIPI-CSI camera (SC202CS) brought up via Espressif's `esp_video` (V4L2) + pipeline (CSI receiver + ISP), with the ISP converting the sensor's RAW8 + output to RGB565 +- `initialize_camera(callback)` streams frames to the callback from a capture + task; the sensor's SCCB shares the internal I2C bus +- See the example's Camera tab for a live-feed display ### Sensors & IMU - BMI270 6-axis sensor (accelerometer + gyroscope) diff --git a/components/m5stack-tab5/example/CMakeLists.txt b/components/m5stack-tab5/example/CMakeLists.txt index ca930545a..7ac93c829 100644 --- a/components/m5stack-tab5/example/CMakeLists.txt +++ b/components/m5stack-tab5/example/CMakeLists.txt @@ -2,7 +2,15 @@ # in this exact order for cmake to work correctly cmake_minimum_required(VERSION 3.20) -set(ENV{IDF_COMPONENT_MANAGER} "0") +# The component manager is needed to pull the managed camera components +# (espressif/esp_video + espressif/esp_cam_sensor). They are declared as +# dependencies of the m5stack-tab5 BSP (components/m5stack-tab5/idf_component.yml) +# and fetched transitively, so they do not need to be listed here. The espp +# components are provided locally via EXTRA_COMPONENT_DIRS below. +# +# Enable the component manager explicitly so this example configures even if the +# build environment defaults it off (it is on by default in a normal ESP-IDF). +set(ENV{IDF_COMPONENT_MANAGER} "1") include($ENV{IDF_PATH}/tools/cmake/project.cmake) # add the component directories that we want to use diff --git a/components/m5stack-tab5/example/README.md b/components/m5stack-tab5/example/README.md index 49f7b9a0a..a9a164f41 100644 --- a/components/m5stack-tab5/example/README.md +++ b/components/m5stack-tab5/example/README.md @@ -18,6 +18,31 @@ This example demonstrates the comprehensive functionality of the M5Stack Tab5 de before playback, and the per-channel peak amplitude is logged so you can tell a silent capture (peak near 0) from a playback issue. - **BMI270 6-axis IMU**: Real-time motion sensing +- **MIPI-CSI Camera (SC202CS)**: The **Camera** tab shows the live camera feed. + The BSP brings up the camera via Espressif's `esp_video` (V4L2) pipeline + (CSI + ISP, RAW8 -> RGB565) and streams frames to a capture task; each frame is + downscaled and rotated to the current display orientation by the PPA, then the + example copies it into an `lv_canvas` on the Camera tab. The managed + `espressif/esp_video` + `espressif/esp_cam_sensor` components are declared as + dependencies of the `m5stack-tab5` BSP (see + `components/m5stack-tab5/idf_component.yml`) and fetched transitively, so this + example builds with the IDF component manager enabled. The ISP pipeline + controller (auto exposure / white balance) is enabled so the feed is correctly + white-balanced. + - **Known upstream quirks (both benign, image unaffected):** + - The stock SC202CS auto color-correction sometimes computes a CCM value + beyond the ESP32-P4 ISP's `+/-4.0` limit under certain lighting. The ISP + safely rejects it and keeps the previous CCM, but the pipeline logs the + rejection every frame - enough blocking UART logging to drop frames - so + the BSP's `initialize_camera()` silences those ISP/CCM log tags. See the + comment in `components/m5stack-tab5/src/camera.cpp`. + - You may also see occasional `ISP: gamma xcoord error` lines (a few at + startup, then rarely on big scene-brightness changes). The precompiled + auto-enhancement (AEN) hands the ISP a gamma table the hardware rejects; + the ISP keeps the previous gamma, so the picture is fine and no frames are + dropped. It is logged from an ISR (`ESP_EARLY_LOGE`), which ignores the + runtime log-level controls, so it cannot be silenced the way the CCM spam + is without also hiding real errors - it is left as-is. - **Battery Management**: INA226 power monitoring with charging control ### Communication Interfaces diff --git a/components/m5stack-tab5/example/main/gui.cpp b/components/m5stack-tab5/example/main/gui.cpp index 04d2b7379..6cf1fe203 100644 --- a/components/m5stack-tab5/example/main/gui.cpp +++ b/components/m5stack-tab5/example/main/gui.cpp @@ -1,6 +1,9 @@ #include +#include #include +#include + #include "gui.hpp" void Gui::init_ui() { @@ -9,12 +12,38 @@ void Gui::init_ui() { init_draw_tab(); init_status_tab(); init_audio_tab(); + init_camera_tab(); init_circle_layer(); + ui_ready_ = true; } void Gui::deinit_ui() { std::lock_guard lock(mutex_); + // Mark the UI down first so a camera frame arriving mid-teardown (the camera + // task runs independently) is dropped rather than touching freed objects. + ui_ready_ = false; lv_obj_clean(lv_screen_active()); + // lv_obj_clean() deleted every object under the screen; null the pointers we + // hold so nothing dereferences a freed object. It does not free the canvas's + // external PSRAM buffer (a member we own), so release that too. + tabview_ = nullptr; + draw_tab_ = nullptr; + status_tab_ = nullptr; + audio_tab_ = nullptr; + camera_tab_ = nullptr; + camera_canvas_ = nullptr; + camera_settings_btn_ = nullptr; + camera_panel_ = nullptr; + camera_scale_dd_ = nullptr; + camera_mirror_sw_ = nullptr; + camera_flip_sw_ = nullptr; + camera_label_ = nullptr; + if (camera_buf_) { + heap_caps_free(camera_buf_); + camera_buf_ = nullptr; + } + camera_w_ = 0; + camera_h_ = 0; } void Gui::init_tabview() { @@ -28,6 +57,7 @@ void Gui::init_tabview() { draw_tab_ = lv_tabview_add_tab(tabview_, "Draw"); status_tab_ = lv_tabview_add_tab(tabview_, "Status"); audio_tab_ = lv_tabview_add_tab(tabview_, "Audio"); + camera_tab_ = lv_tabview_add_tab(tabview_, "Camera"); // switching tabs is done with the tab buttons only: disable swipe // scrolling of the content so drawing on the Draw tab cannot accidentally // change pages @@ -166,6 +196,211 @@ void Gui::update_audio_label() { LV_SYMBOL_PLUS); } +void Gui::init_camera_tab() { + // Center the camera feed; show a placeholder label until the first frame + // arrives. The canvas that displays the live feed is created lazily in + // set_camera_frame() once the true frame size is known. The settings controls + // float on top of the feed (see build_camera_controls). + lv_obj_set_flex_flow(camera_tab_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_flex_align(camera_tab_, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, + LV_FLEX_ALIGN_CENTER); + // No padding, so a full-size feed can sit flush in the top-left corner. + lv_obj_set_style_pad_all(camera_tab_, 0, 0); + camera_label_ = lv_label_create(camera_tab_); + lv_label_set_text(camera_label_, "Waiting for camera..."); + build_camera_controls(); +} + +void Gui::build_camera_controls() { + // A gear button in the top-right corner, floating (not part of the tab's flex + // layout) so it overlays the feed and is always visible. Tapping it toggles + // the settings panel. + camera_settings_btn_ = lv_btn_create(camera_tab_); + lv_obj_add_flag(camera_settings_btn_, LV_OBJ_FLAG_FLOATING); + lv_obj_set_size(camera_settings_btn_, 48, 48); + lv_obj_align(camera_settings_btn_, LV_ALIGN_TOP_RIGHT, -8, 8); + lv_obj_t *gear = lv_label_create(camera_settings_btn_); + lv_label_set_text(gear, LV_SYMBOL_SETTINGS); + lv_obj_center(gear); + lv_obj_add_event_cb(camera_settings_btn_, camera_settings_toggle_cb, LV_EVENT_CLICKED, this); + + // The panel: floating, semi-transparent, sized to its content so it stays + // compact. Starts collapsed (hidden). + camera_panel_ = lv_obj_create(camera_tab_); + lv_obj_add_flag(camera_panel_, LV_OBJ_FLAG_FLOATING); + lv_obj_set_size(camera_panel_, 240, LV_SIZE_CONTENT); + lv_obj_align(camera_panel_, LV_ALIGN_TOP_RIGHT, -8, 64); + lv_obj_set_flex_flow(camera_panel_, LV_FLEX_FLOW_COLUMN); + lv_obj_set_style_pad_row(camera_panel_, 8, 0); + lv_obj_set_style_bg_opa(camera_panel_, LV_OPA_80, 0); + lv_obj_add_flag(camera_panel_, LV_OBJ_FLAG_HIDDEN); + + lv_obj_t *title = lv_label_create(camera_panel_); + lv_label_set_text(title, "Camera settings"); + + // Image size dropdown. + lv_obj_t *size_label = lv_label_create(camera_panel_); + lv_label_set_text(size_label, "Image size"); + camera_scale_dd_ = lv_dropdown_create(camera_panel_); + lv_dropdown_set_options(camera_scale_dd_, "Full\nHalf\nQuarter"); + lv_dropdown_set_selected(camera_scale_dd_, 1); // Half (the BSP default) + lv_obj_set_width(camera_scale_dd_, lv_pct(100)); + lv_obj_add_event_cb(camera_scale_dd_, camera_control_event_cb, LV_EVENT_VALUE_CHANGED, this); + + // A labeled switch row helper. + auto add_switch = [&](const char *text, bool on) { + lv_obj_t *row = lv_obj_create(camera_panel_); + lv_obj_remove_style_all(row); + lv_obj_set_size(row, lv_pct(100), LV_SIZE_CONTENT); + lv_obj_set_flex_flow(row, LV_FLEX_FLOW_ROW); + lv_obj_set_flex_align(row, LV_FLEX_ALIGN_SPACE_BETWEEN, LV_FLEX_ALIGN_CENTER, + LV_FLEX_ALIGN_CENTER); + lv_obj_t *l = lv_label_create(row); + lv_label_set_text(l, text); + lv_obj_t *sw = lv_switch_create(row); + if (on) { + lv_obj_add_state(sw, LV_STATE_CHECKED); + } + lv_obj_add_event_cb(sw, camera_control_event_cb, LV_EVENT_VALUE_CHANGED, this); + return sw; + }; + camera_mirror_sw_ = add_switch("Mirror", false); + camera_flip_sw_ = add_switch("Flip", false); +} + +void Gui::camera_settings_toggle_cb(lv_event_t *e) { + auto *gui = static_cast(lv_event_get_user_data(e)); + if (!gui || !gui->camera_panel_) { + return; + } + if (lv_obj_has_flag(gui->camera_panel_, LV_OBJ_FLAG_HIDDEN)) { + lv_obj_clear_flag(gui->camera_panel_, LV_OBJ_FLAG_HIDDEN); + lv_obj_move_foreground(gui->camera_panel_); + } else { + lv_obj_add_flag(gui->camera_panel_, LV_OBJ_FLAG_HIDDEN); + } +} + +void Gui::camera_control_event_cb(lv_event_t *e) { + auto *gui = static_cast(lv_event_get_user_data(e)); + if (gui) { + gui->sync_camera_controls(); + } +} + +void Gui::sync_camera_controls() { + // Read every widget into camera_controls_ and push it to the BSP. (Called + // from an LVGL event, i.e. already under the GUI mutex via lv_task_handler.) + using Scale = espp::M5StackTab5::CameraScale; + uint32_t scale_idx = lv_dropdown_get_selected(camera_scale_dd_); + camera_controls_.scale = (scale_idx == 0) ? Scale::FULL + : (scale_idx == 2) ? Scale::QUARTER + : Scale::HALF; + camera_controls_.hmirror = lv_obj_has_state(camera_mirror_sw_, LV_STATE_CHECKED); + camera_controls_.vflip = lv_obj_has_state(camera_flip_sw_, LV_STATE_CHECKED); + espp::M5StackTab5::get().set_camera_controls(camera_controls_); +} + +void Gui::raise_camera_controls() { + // Keep the overlay above the camera canvas, which is (re)created lazily and + // would otherwise cover the controls. + if (camera_settings_btn_) { + lv_obj_move_foreground(camera_settings_btn_); + } + if (camera_panel_) { + lv_obj_move_foreground(camera_panel_); + } +} + +void Gui::camera_canvas_drag_cb(lv_event_t *e) { + auto *canvas = static_cast(lv_event_get_target(e)); + lv_indev_t *indev = lv_indev_active(); + if (!canvas || !indev) { + return; + } + // Move the feed by the drag delta since the last event. + lv_point_t vect; + lv_indev_get_vect(indev, &vect); + lv_obj_t *parent = lv_obj_get_parent(canvas); + int32_t x = lv_obj_get_x(canvas) + vect.x; + int32_t y = lv_obj_get_y(canvas) + vect.y; + // Keep the feed within the tab; if it is larger than the tab, the range goes + // negative so its far edges can still be panned into view. + int32_t range_x = lv_obj_get_width(parent) - lv_obj_get_width(canvas); + int32_t range_y = lv_obj_get_height(parent) - lv_obj_get_height(canvas); + int32_t lo_x = LV_MIN(0, range_x), hi_x = LV_MAX(0, range_x); + int32_t lo_y = LV_MIN(0, range_y), hi_y = LV_MAX(0, range_y); + x = x < lo_x ? lo_x : (x > hi_x ? hi_x : x); + y = y < lo_y ? lo_y : (y > hi_y ? hi_y : y); + lv_obj_set_pos(canvas, x, y); +} + +void Gui::set_camera_frame(const uint8_t *rgb565, int w, int h) { + if (!rgb565 || w <= 0 || h <= 0) { + return; + } + std::lock_guard lock(mutex_); + // The camera task may still deliver a frame after the UI is torn down; drop it + // rather than create objects under a freed camera_tab_. + if (!ui_ready_ || !camera_tab_) { + return; + } + // (Re)allocate the canvas buffer and (re)create the canvas on the first frame + // or whenever the frame size changes. The buffer must outlive the canvas, so + // it is a member kept in PSRAM. + if (camera_buf_ == nullptr || w != camera_w_ || h != camera_h_) { + // Allocate the new buffer BEFORE freeing the old one / deleting the canvas, + // so that on OOM we keep showing the previous frame instead of blanking the + // tab. + const size_t bytes = static_cast(w) * static_cast(h) * 2; + auto *new_buf = + static_cast(heap_caps_malloc(bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (!new_buf) { + return; // out of memory; keep the current canvas / preview + } + if (camera_canvas_) { + lv_obj_del(camera_canvas_); + camera_canvas_ = nullptr; + } + if (camera_buf_) { + heap_caps_free(camera_buf_); + } + camera_buf_ = new_buf; + camera_w_ = w; + camera_h_ = h; + camera_canvas_ = lv_canvas_create(camera_tab_); + lv_canvas_set_buffer(camera_canvas_, camera_buf_, w, h, LV_COLOR_FORMAT_RGB565); + // Frame the feed (a border + rounded corners so it reads as a distinct + // element when it is smaller than the tab), and make it draggable within the + // tab. LV_OBJ_FLAG_FLOATING takes it out of the tab's flex layout so it can + // be freely positioned; the border brightens while it is pressed to hint + // that it can be moved. + lv_obj_add_flag(camera_canvas_, LV_OBJ_FLAG_FLOATING); + lv_obj_add_flag(camera_canvas_, LV_OBJ_FLAG_CLICKABLE); + lv_obj_set_style_border_width(camera_canvas_, 2, 0); + lv_obj_set_style_border_color(camera_canvas_, lv_palette_main(LV_PALETTE_GREY), 0); + lv_obj_set_style_border_color(camera_canvas_, lv_palette_main(LV_PALETTE_BLUE), + LV_STATE_PRESSED); + lv_obj_set_style_radius(camera_canvas_, 6, 0); + lv_obj_add_event_cb(camera_canvas_, camera_canvas_drag_cb, LV_EVENT_PRESSING, this); + // Center the feed when it fits within the tab; otherwise pin it to the + // top-left so it fills from the corner (drag to pan the overflow). + lv_obj_update_layout(camera_tab_); + const int32_t tab_w = lv_obj_get_width(camera_tab_); + const int32_t tab_h = lv_obj_get_height(camera_tab_); + const bool fits = (w <= tab_w) && (h <= tab_h); + lv_obj_set_pos(camera_canvas_, fits ? (tab_w - w) / 2 : 0, fits ? (tab_h - h) / 2 : 0); + if (camera_label_) { + lv_obj_add_flag(camera_label_, LV_OBJ_FLAG_HIDDEN); + } + // the canvas was just created on top of the settings overlay; restore the + // overlay to the foreground so the gear button / panel stay tappable. + raise_camera_controls(); + } + std::memcpy(camera_buf_, rgb565, static_cast(w) * static_cast(h) * 2); + lv_obj_invalidate(camera_canvas_); +} + void Gui::init_circle_layer() { // a transparent, click-through overlay above the tabview which shows the // touch trail (only populated while the Draw tab is active) diff --git a/components/m5stack-tab5/example/main/gui.hpp b/components/m5stack-tab5/example/main/gui.hpp index 099ee9511..81981117c 100644 --- a/components/m5stack-tab5/example/main/gui.hpp +++ b/components/m5stack-tab5/example/main/gui.hpp @@ -66,6 +66,14 @@ class Gui { /// @param text The text to display void set_status_text(std::string_view text); + /// Show a camera frame on the Camera tab. Thread-safe. + /// @param rgb565 The frame pixel data (RGB565, w*h*2 bytes) + /// @param w The frame width in pixels + /// @param h The frame height in pixels + /// @note The data is copied, so it need not outlive the call. The first call + /// (allocating the canvas buffer for the given size) sizes the display. + void set_camera_frame(const uint8_t *rgb565, int w, int h); + /// Whether the Draw tab is currently active (used by the example to only /// draw circles / play clicks for touches on that tab). Thread-safe. /// @return True if the Draw tab is the active tab @@ -148,6 +156,7 @@ class Gui { void init_draw_tab(); void init_status_tab(); void init_audio_tab(); + void init_camera_tab(); void init_circle_layer(); // update the audio volume label from the BSP's current volumes; called @@ -168,6 +177,18 @@ class Gui { void on_pressed(lv_event_t *e); void on_tab_changed(lv_event_t *e); + // Camera controls overlay: build the collapsible settings panel, toggle it, + // read the widgets into a CameraControls and push it to the BSP, and + // show/hide the manual sliders depending on the auto toggle. + void build_camera_controls(); + static void camera_settings_toggle_cb(lv_event_t *e); + static void camera_control_event_cb(lv_event_t *e); + void sync_camera_controls(); + // keep the settings overlay above the (lazily (re)created) camera canvas + void raise_camera_controls(); + // drag the camera feed around within the tab (LV_EVENT_PRESSING handler) + static void camera_canvas_drag_cb(lv_event_t *e); + // custom drawing of the circle layer static void draw_circle_layer(lv_event_t *e); void draw_circles(lv_event_t *e) const; @@ -181,6 +202,23 @@ class Gui { lv_obj_t *draw_tab_{nullptr}; lv_obj_t *status_tab_{nullptr}; lv_obj_t *audio_tab_{nullptr}; + lv_obj_t *camera_tab_{nullptr}; + + // Camera-feed widgets: a canvas bound to a PSRAM RGB565 buffer, (re)allocated + // on the first frame (or a size change) once the true frame size is known. + lv_obj_t *camera_canvas_{nullptr}; + lv_obj_t *camera_label_{nullptr}; + uint8_t *camera_buf_{nullptr}; + int camera_w_{0}; + int camera_h_{0}; + // Camera controls overlay (a gear button + a collapsible panel floating over + // the feed) and the widgets inside it. + lv_obj_t *camera_settings_btn_{nullptr}; + lv_obj_t *camera_panel_{nullptr}; + lv_obj_t *camera_scale_dd_{nullptr}; + lv_obj_t *camera_mirror_sw_{nullptr}; + lv_obj_t *camera_flip_sw_{nullptr}; + espp::M5StackTab5::CameraControls camera_controls_{}; lv_obj_t *label_{nullptr}; lv_obj_t *status_label_{nullptr}; lv_obj_t *kalman_line_{nullptr}; @@ -227,4 +265,7 @@ class Gui { .name = "gui", .stack_size_bytes = 12 * 1024, .priority = 20, .core_id = 1}}}; espp::Logger logger_; std::recursive_mutex mutex_; + // True between init_ui() and deinit_ui(). Guards set_camera_frame() (called + // from the camera task) against touching the LVGL tree after teardown. + bool ui_ready_{false}; }; diff --git a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp index b3f0106a8..4d2f1bb85 100644 --- a/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp +++ b/components/m5stack-tab5/example/main/m5stack_tab5_example.cpp @@ -472,6 +472,16 @@ extern "C" void app_main(void) { }}); imu_task.start(); + // Initialize the on-board MIPI-CSI camera and stream its frames to the Camera + // tab. The BSP runs a capture task that hands each RGB565 frame to this + // callback; forward it to the thread-safe GUI. Non-fatal: the rest of the + // example still runs if the camera is unavailable. + logger.info("Initializing camera..."); + if (!tab5.initialize_camera( + [&](const uint8_t *data, int w, int h, size_t) { gui.set_camera_frame(data, w, h); })) { + logger.warn("Failed to initialize camera; the Camera tab will stay blank"); + } + // Main loop: stream any active playback to the speaker and notice when a // recording stops (either button press or the buffer filling up) size_t play_offset = 0; diff --git a/components/m5stack-tab5/example/sdkconfig.defaults b/components/m5stack-tab5/example/sdkconfig.defaults index afef7c936..d70bf7978 100644 --- a/components/m5stack-tab5/example/sdkconfig.defaults +++ b/components/m5stack-tab5/example/sdkconfig.defaults @@ -62,3 +62,22 @@ CONFIG_LV_USE_THEME_DEFAULT=y CONFIG_LV_THEME_DEFAULT_DARK=y CONFIG_LV_THEME_DEFAULT_GROW=y CONFIG_LV_THEME_DEFAULT_TRANSITION_TIME=30 + +# On-board MIPI-CSI camera: esp_video (V4L2) capture pipeline + SC202CS sensor. +# Enabling the MIPI-CSI video device auto-selects the ISP (RAW8 -> RGB565). The +# SC202CS is auto-detected on the shared SCCB/I2C bus at startup. +CONFIG_ESP_VIDEO_ENABLE_MIPI_CSI_VIDEO_DEVICE=y +CONFIG_CAMERA_SC202CS=y +CONFIG_CAMERA_SC202CS_AUTO_DETECT_MIPI_INTERFACE_SENSOR=y +# Enable the ISP pipeline controller (isp_task): auto exposure / white balance. +# Without it the ISP runs with static gains and the feed has a heavy green cast. +# NOTE: the stock SC202CS IPA config is used. Its auto color-correction (ACC) +# algorithm occasionally computes a CCM coefficient beyond the ESP32-P4 ISP's +# +/-4.0 hardware limit under some illuminants; the ISP safely rejects it and +# keeps the previous CCM (the image is unaffected), but the stock pipeline logs +# the rejection several times per frame. That per-frame blocking UART logging is +# what dropped frames, so initialize_camera() silences those specific tags. See +# the BSP camera.cpp for details. (A custom clamped JSON via +# CONFIG_CAMERA_SC202CS_CUSTOMIZED_IPA_JSON_CONFIGURATION_FILE cannot fully bound +# the runtime CCM - the amplification happens inside the precompiled esp_ipa.) +CONFIG_ESP_VIDEO_ENABLE_ISP_PIPELINE_CONTROLLER=y diff --git a/components/m5stack-tab5/idf_component.yml b/components/m5stack-tab5/idf_component.yml index 0566c70cb..937bc6b9c 100644 --- a/components/m5stack-tab5/idf_component.yml +++ b/components/m5stack-tab5/idf_component.yml @@ -30,5 +30,9 @@ dependencies: espp/rx8130ce: ">=1.0" espp/st7123touch: ">=1.0" espp/task: ">=1.0" + # MIPI-CSI camera pipeline: esp_video provides the V4L2 capture framework + # (CSI + ISP) and esp_cam_sensor provides the SC202CS sensor driver. + espressif/esp_video: ">=2.0" + espressif/esp_cam_sensor: ">=2.0" targets: - esp32p4 diff --git a/components/m5stack-tab5/include/m5stack-tab5.hpp b/components/m5stack-tab5/include/m5stack-tab5.hpp old mode 100755 new mode 100644 index e9c34c7c9..c428a7fb2 --- a/components/m5stack-tab5/include/m5stack-tab5.hpp +++ b/components/m5stack-tab5/include/m5stack-tab5.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -52,7 +53,7 @@ namespace espp { /// - 5" 720p MIPI-DSI Display with GT911 multi-touch /// - Dual audio codecs (ES8388 + ES7210 AEC) /// - BMI270 6-axis IMU sensor -/// - SC2356 2MP camera via MIPI-CSI (not yet implemented) +/// - MIPI-CSI camera (SC202CS) via the esp_video (V4L2) pipeline /// - ESP32-C6 wireless module (Wi-Fi 6, Thread, ZigBee) /// - USB-A Host and USB-C OTG ports /// - RS-485 industrial interface (not yet implemented) @@ -129,6 +130,13 @@ class M5StackTab5 : public BaseComponent { /// Alias for the touch callback when touch events are received using touch_callback_t = std::function; + /// Alias for the camera frame callback. Called from the camera task with each + /// captured frame: \p data is the pixel buffer (RGB565, \p width x \p height), + /// valid only for the duration of the callback, and \p length is its size in + /// bytes. Copy the data if it needs to outlive the call. + using camera_frame_callback_t = + std::function; + /// Mount point for the uSD card on the TDeck. static constexpr char mount_point[] = "/sdcard"; @@ -352,6 +360,76 @@ class M5StackTab5 : public BaseComponent { /// \return The microphone volume as a percentage (0 - 100) float microphone_volume() const; + ///////////////////////////////////////////////////////////////////////////// + // Camera (MIPI-CSI, SC202CS) + ///////////////////////////////////////////////////////////////////////////// + + /// Initialize the on-board MIPI-CSI camera and start streaming frames. + /// + /// Brings up the ESP32-P4 camera pipeline (MIPI-CSI receiver + ISP + sensor) + /// via the esp_video (V4L2) framework, configures the ISP to output RGB565, + /// and starts a task that delivers each captured frame to \p callback. The + /// camera sensor's SCCB shares the internal I2C bus. + /// + /// \param callback Function called from the camera task with each RGB565 + /// frame (see camera_frame_callback_t). Keep it quick and non-blocking. + /// \param task_config The configuration for the camera task + /// \return true if the camera was successfully initialized and streaming + /// started, false otherwise + /// \note The internal I2C bus must be initialized first (the sensor is on it); + /// initialize_io_expanders() must also have run (camera reset is on an + /// IO expander). The callback runs in the camera task's context. + bool initialize_camera(const camera_frame_callback_t &callback, + const espp::Task::BaseConfig &task_config = {.name = "tab5_camera", + .stack_size_bytes = 6 * 1024, + .priority = 5, + .core_id = 0}); + + /// Stop the camera stream and release the camera pipeline. + void stop_camera(); + + /// Get the width of the captured camera frames, in pixels + /// \return The camera frame width (0 if the camera is not initialized) + uint16_t camera_width() const; + + /// Get the height of the captured camera frames, in pixels + /// \return The camera frame height (0 if the camera is not initialized) + uint16_t camera_height() const; + + /// Assert or release the camera reset line (IO expander 0x43 P6, active-low) + /// \param assert_reset True to hold the camera in reset, false to release it + /// \return true if the IO expander write succeeded, false otherwise + bool camera_reset(bool assert_reset); + + /// Camera preview scale: the PPA downscales each frame by this factor before + /// it is delivered, trading resolution for lower per-frame CPU / PSRAM cost. + enum class CameraScale { + FULL, ///< No downscale (native, e.g. 1280x720) - sharpest, heaviest + HALF, ///< 1/2 (e.g. 640x360) - the default + QUARTER, ///< 1/4 (e.g. 320x180) - lightest + }; + + /// Adjustable camera controls, applied by the camera task. + /// + /// Only exposes the controls that are effective on the Tab5: the preview + /// scale and the mirror / flip (both done in the PPA). Exposure / white + /// balance / color are driven by the ISP auto pipeline and are not manually + /// overridable here. + struct CameraControls { + CameraScale scale{CameraScale::HALF}; + bool hmirror{false}; ///< Horizontal mirror + bool vflip{false}; ///< Vertical flip + }; + + /// Update the camera controls. Thread-safe: the change is applied by the + /// camera task on its next iteration, so this is safe to call from the GUI. + /// \param controls The desired controls + void set_camera_controls(const CameraControls &controls); + + /// Get the current (last requested) camera controls. + /// \return The camera controls + CameraControls camera_controls() const; + ///////////////////////////////////////////////////////////////////////////// // IMU & Sensors ///////////////////////////////////////////////////////////////////////////// @@ -549,6 +627,7 @@ class M5StackTab5 : public BaseComponent { static constexpr uint8_t IO43_BIT_SPK_EN = 1; // P1 static constexpr uint8_t IO43_BIT_LCD_RST = 4; // P4 static constexpr uint8_t IO43_BIT_TP_RST = 5; // P5 + static constexpr uint8_t IO43_BIT_CAM_RST = 6; // P6 static constexpr uint8_t IO44_BIT_CHG_EN = 7; // P7 static constexpr uint8_t IO44_BIT_CHG_STAT = 6; // P6 @@ -772,6 +851,49 @@ class M5StackTab5 : public BaseComponent { std::atomic recording_{false}; std::function audio_rx_callback_{nullptr}; + // Camera (MIPI-CSI via esp_video / V4L2) + bool camera_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + std::atomic camera_initialized_{false}; + camera_frame_callback_t camera_callback_{nullptr}; + std::unique_ptr camera_task_{nullptr}; + int camera_fd_{-1}; // MIPI-CSI capture device (/dev/video0) + bool camera_video_inited_{false}; // esp_video_init() succeeded (needs deinit) + bool camera_logs_silenced_{false}; // ISP log tags muted (restore on stop) + int camera_prev_log_levels_[4]{}; // their prior levels (esp_log_level_t) + uint16_t camera_width_{0}; + uint16_t camera_height_{0}; + static constexpr int CAMERA_BUFFER_COUNT = 2; + void *camera_buffers_[CAMERA_BUFFER_COUNT]{nullptr, nullptr}; + size_t camera_buffer_sizes_[CAMERA_BUFFER_COUNT]{0, 0}; + int camera_buffer_count_{0}; // buffers VIDIOC_REQBUFS actually allocated + // Each captured frame is run through the PPA (Pixel Processing Accelerator) + // in one hardware pass to downscale it (a full-resolution frame is expensive + // to re-render every frame) and rotate it to match the current display + // orientation. The callback receives this preview buffer, not the raw frame. + ppa_client_handle_t camera_ppa_client_{nullptr}; + uint8_t *camera_preview_buffer_{nullptr}; + size_t camera_preview_bytes_{0}; + uint16_t camera_preview_width_{0}; + uint16_t camera_preview_height_{0}; + // Adjustable controls: the GUI (any thread) posts a desired state here; the + // camera task applies it on its next iteration so all V4L2 ioctls, the PPA + // scale and the preview-buffer (re)allocation happen in one thread. + mutable std::mutex camera_controls_mutex_; + CameraControls camera_controls_{}; + bool camera_controls_dirty_{false}; + bool camera_scale_alloc_warned_{false}; // rate-limit the OOM-on-scale warning + CameraScale camera_active_scale_{CameraScale::HALF}; + // Mirror / flip are done in the PPA pass; the task reads these when building + // the PPA operation. + bool camera_active_hmirror_{false}; + bool camera_active_vflip_{false}; + // Current display rotation (LVGL enum value 0..3), cached by the display + // flush (which runs on the GUI thread and already reads it) so the camera + // task can read the rotation without calling LVGL from its own thread. + std::atomic camera_display_rotation_{0}; + void apply_camera_controls(); + bool allocate_camera_preview_buffer(CameraScale scale); + // Power management std::atomic battery_monitoring_initialized_{false}; BatteryStatus battery_status_; diff --git a/components/m5stack-tab5/src/camera.cpp b/components/m5stack-tab5/src/camera.cpp new file mode 100644 index 000000000..276cae81e --- /dev/null +++ b/components/m5stack-tab5/src/camera.cpp @@ -0,0 +1,511 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "linux/videodev2.h" + +#include "esp_video_device.h" +#include "esp_video_init.h" + +#include "m5stack-tab5.hpp" + +using namespace espp; + +// ISP / esp_video log tags silenced during streaming (see initialize_camera). +// Kept in one place so stop_camera() can restore their prior levels. +static constexpr const char *kCameraSilencedLogTags[] = {"ISP_CCM", "ISP", "isp_video", + "esp_video"}; + +//////////////////////// +// Camera Functions // +//////////////////////// + +bool M5StackTab5::camera_reset(bool assert_reset) { + // CAM_RST is on IO expander 0x43 pin P6 and is active-low: drive the output + // LOW to hold the sensor in reset, HIGH to release it. set_io_expander_output + // takes the desired output level, so invert the "assert" request. + return set_io_expander_output(0x43, IO43_BIT_CAM_RST, !assert_reset); +} + +bool M5StackTab5::initialize_camera(const camera_frame_callback_t &callback, + const espp::Task::BaseConfig &task_config) { + logger_.info("Initializing camera (MIPI-CSI, SC202CS)"); + if (camera_initialized_) { + logger_.warn("Camera already initialized"); + return false; + } + if (!callback) { + logger_.error("A callback is required to receive camera frames"); + return false; + } + camera_callback_ = callback; + + // Pulse the (active-low) camera reset for a clean sensor power-up, then + // release it and give the sensor a moment before esp_video probes it over + // SCCB. The IO expander defaults this pin HIGH, so the camera is normally + // out of reset already; the pulse just guarantees a known start state. The + // reset goes through the IO expander, so a write failure here (e.g. the IO + // expander was not initialized) means the sensor state is unknown - fail + // rather than let esp_video probe a possibly-held-in-reset sensor. + if (!camera_reset(true)) { + logger_.error("Camera reset (assert) failed; is the IO expander initialized?"); + camera_callback_ = nullptr; + return false; + } + vTaskDelay(pdMS_TO_TICKS(10)); + if (!camera_reset(false)) { + logger_.error("Camera reset (release) failed"); + camera_callback_ = nullptr; + return false; + } + vTaskDelay(pdMS_TO_TICKS(20)); + + // Bring up the CSI receiver + ISP + sensor. The SC202CS's SCCB shares the + // internal I2C bus (same GPIO32/31), so hand esp_video that existing bus + // handle rather than letting it create a second master on the same pins. + // Reset / power-down are NC here: the sensor reset is on the IO expander and + // was handled above. + esp_video_init_csi_config_t csi_config = {}; + csi_config.sccb_config.init_sccb = false; + csi_config.sccb_config.i2c_handle = internal_i2c_.native_bus_handle(); + // Run the sensor SCCB at 400 kHz rather than 100 kHz. The SCCB shares the + // internal I2C bus with the touch controller (and IMU / RTC / power monitor), + // and the ISP's auto-exposure writes the sensor over SCCB every frame; at + // 100 kHz each of those writes holds the shared bus ~4x longer than needed, + // starving the (deliberately short, fail-fast) touch reads and making touch + // feel laggy. 400 kHz is a safe SCCB speed for this sensor and cuts that + // bus-hold time. (The bus itself runs the other devices at 1 MHz; the I2C + // master reprograms the clock per transaction.) + csi_config.sccb_config.freq = 400000; + csi_config.reset_pin = GPIO_NUM_NC; + csi_config.pwdn_pin = GPIO_NUM_NC; + csi_config.dont_init_ldo = false; + + esp_video_init_config_t video_config = {}; + video_config.csi = &csi_config; + + esp_err_t err = esp_video_init(&video_config); + if (err != ESP_OK) { + logger_.error("esp_video_init failed: {}", esp_err_to_name(err)); + camera_callback_ = nullptr; + return false; + } + camera_video_inited_ = true; + + // Open the MIPI-CSI capture device. Non-blocking so the capture task can + // observe a stop request even when no frame is ready. From here on, failures + // go through stop_camera() so the esp_video pipeline is torn down (otherwise a + // later initialize_camera() retry would fail). + camera_fd_ = open(ESP_VIDEO_MIPI_CSI_DEVICE_NAME, O_RDWR | O_NONBLOCK); + if (camera_fd_ < 0) { + logger_.error("Could not open camera device {} (errno {}: {})", ESP_VIDEO_MIPI_CSI_DEVICE_NAME, + errno, strerror(errno)); + stop_camera(); + return false; + } + + struct v4l2_capability capability = {}; + if (ioctl(camera_fd_, VIDIOC_QUERYCAP, &capability) != 0) { + logger_.error("VIDIOC_QUERYCAP failed (errno {}: {})", errno, strerror(errno)); + stop_camera(); + return false; + } + + // Choose a capture resolution: enumerate the RGB565 discrete frame sizes and + // pick the largest that fits the panel (<= 1280 wide, the landscape width), + // falling back to 1280x720 (the SC202CS's HD mode). VIDIOC_S_FMT below + // adjusts to the sensor's actual size and we read the real values back, so + // this is only a hint. + uint32_t want_w = 1280, want_h = 720; + { + struct v4l2_frmsizeenum frmsize = {}; + frmsize.pixel_format = V4L2_PIX_FMT_RGB565; + uint32_t best_w = 0, best_h = 0; + for (frmsize.index = 0; ioctl(camera_fd_, VIDIOC_ENUM_FRAMESIZES, &frmsize) == 0; + ++frmsize.index) { + if (frmsize.type != V4L2_FRMSIZE_TYPE_DISCRETE) { + break; + } + uint32_t w = frmsize.discrete.width; + uint32_t h = frmsize.discrete.height; + if (w <= 1280 && (w > best_w || (w == best_w && h > best_h))) { + best_w = w; + best_h = h; + } + } + if (best_w > 0) { + want_w = best_w; + want_h = best_h; + } + } + + struct v4l2_format format = {}; + format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + format.fmt.pix.width = want_w; + format.fmt.pix.height = want_h; + format.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB565; + if (ioctl(camera_fd_, VIDIOC_S_FMT, &format) != 0) { + logger_.error("VIDIOC_S_FMT (RGB565 {}x{}) failed (errno {}: {})", want_w, want_h, errno, + strerror(errno)); + stop_camera(); + return false; + } + camera_width_ = static_cast(format.fmt.pix.width); + camera_height_ = static_cast(format.fmt.pix.height); + logger_.info("Camera format: {}x{} RGB565", camera_width_, camera_height_); + + // Request and memory-map the capture buffers. REQBUFS may hand back fewer + // buffers than requested; use the count it actually allocated (and require at + // least one, capped at our array size). + struct v4l2_requestbuffers req = {}; + req.count = CAMERA_BUFFER_COUNT; + req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + req.memory = V4L2_MEMORY_MMAP; + if (ioctl(camera_fd_, VIDIOC_REQBUFS, &req) != 0) { + logger_.error("VIDIOC_REQBUFS failed (errno {}: {})", errno, strerror(errno)); + stop_camera(); + return false; + } + if (req.count < 1) { + logger_.error("VIDIOC_REQBUFS allocated 0 buffers"); + stop_camera(); + return false; + } + camera_buffer_count_ = std::min(req.count, CAMERA_BUFFER_COUNT); + if (req.count < CAMERA_BUFFER_COUNT) { + logger_.warn("VIDIOC_REQBUFS allocated {} of {} requested buffers", req.count, + CAMERA_BUFFER_COUNT); + } + for (int i = 0; i < camera_buffer_count_; ++i) { + struct v4l2_buffer buf = {}; + buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.memory = V4L2_MEMORY_MMAP; + buf.index = i; + if (ioctl(camera_fd_, VIDIOC_QUERYBUF, &buf) != 0) { + logger_.error("VIDIOC_QUERYBUF {} failed (errno {}: {})", i, errno, strerror(errno)); + stop_camera(); + return false; + } + camera_buffer_sizes_[i] = buf.length; + camera_buffers_[i] = + mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, camera_fd_, buf.m.offset); + if (camera_buffers_[i] == MAP_FAILED) { + camera_buffers_[i] = nullptr; + logger_.error("mmap of camera buffer {} failed (errno {}: {})", i, errno, strerror(errno)); + stop_camera(); + return false; + } + if (ioctl(camera_fd_, VIDIOC_QBUF, &buf) != 0) { + logger_.error("VIDIOC_QBUF {} failed (errno {}: {})", i, errno, strerror(errno)); + stop_camera(); + return false; + } + } + + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (ioctl(camera_fd_, VIDIOC_STREAMON, &type) != 0) { + logger_.error("VIDIOC_STREAMON failed (errno {}: {})", errno, strerror(errno)); + stop_camera(); + return false; + } + + // Silence the ISP CCM-rejection spam. The stock SC202CS IPA config's auto + // color-correction (esp_ipa ACC) periodically computes a CCM coefficient + // beyond the ESP32-P4 ISP's +/-4.0 hardware limit under some illuminants. The + // ISP driver safely rejects it and keeps the previous CCM - the image is + // unaffected - but esp_video's ISP pipeline logs the rejection several times + // per frame (ISP_CCM / ISP / isp_video / esp_video, all at ERROR level). That + // per-frame *blocking* UART logging both floods the console and starves the + // capture pipeline enough to drop frames. The out-of-range CCM is produced + // inside the precompiled esp_ipa and cannot be fully bounded from the JSON + // config, so silence just these tags (done after esp_video_init so its + // bring-up diagnostics are still printed). A genuine CSI/ISP failure still + // surfaces as a missing feed. The prior levels are cached so stop_camera() + // can restore them - muting must not outlive the camera. + for (size_t i = 0; i < std::size(kCameraSilencedLogTags); ++i) { + camera_prev_log_levels_[i] = esp_log_level_get(kCameraSilencedLogTags[i]); + esp_log_level_set(kCameraSilencedLogTags[i], ESP_LOG_NONE); + } + camera_logs_silenced_ = true; + + // Register a PPA client and allocate a downscaled preview buffer. Each frame + // is run through the PPA to (1) halve its size - a full-resolution frame is + // expensive to re-render and rotate every frame, which is what made the feed + // slow in the rotated display orientations - and (2) swap the RGB565 byte + // order the ISP emits so it matches the LVGL canvas. Both happen in one + // hardware pass. The callback receives this preview buffer. + ppa_client_config_t ppa_cfg = {}; + ppa_cfg.oper_type = PPA_OPERATION_SRM; + if (ppa_register_client(&ppa_cfg, &camera_ppa_client_) != ESP_OK) { + logger_.error("Could not register the camera PPA client"); + stop_camera(); + return false; + } + { + std::lock_guard lock(camera_controls_mutex_); + camera_active_scale_ = camera_controls_.scale; + } + if (!allocate_camera_preview_buffer(camera_active_scale_)) { + stop_camera(); + return false; + } + logger_.info("Camera preview: {}x{} RGB565 (PPA downscaled)", camera_preview_width_, + camera_preview_height_); + + // Apply whatever controls were requested before streaming started. + { + std::lock_guard lock(camera_controls_mutex_); + camera_controls_dirty_ = true; + } + apply_camera_controls(); + + using namespace std::placeholders; + camera_task_ = espp::Task::make_unique({ + .callback = std::bind(&M5StackTab5::camera_task_callback, this, _1, _2, _3), + .task_config = task_config, + }); + if (!camera_task_->start()) { + logger_.error("Could not start the camera task"); + stop_camera(); + return false; + } + camera_initialized_ = true; + return true; +} + +bool M5StackTab5::allocate_camera_preview_buffer(CameraScale scale) { + const int factor = (scale == CameraScale::FULL) ? 1 : (scale == CameraScale::QUARTER) ? 4 : 2; + const uint16_t w = static_cast((camera_width_ / factor) & ~1u); + const uint16_t h = static_cast((camera_height_ / factor) & ~1u); + static constexpr size_t kCacheAlign = 128; // PPA output buffer alignment (L2 line) + const size_t data_bytes = static_cast(w) * h * 2; + const size_t needed = (data_bytes + kCacheAlign - 1) / kCacheAlign * kCacheAlign; + // Reuse the buffer if the size is unchanged; otherwise allocate a new one + // first so we don't lose the current preview on OOM. + if (camera_preview_buffer_ == nullptr || camera_preview_bytes_ != needed) { + auto *new_buf = static_cast( + heap_caps_aligned_alloc(kCacheAlign, needed, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (new_buf == nullptr) { + logger_.error("Could not allocate the camera preview buffer ({} bytes)", needed); + return false; + } + if (camera_preview_buffer_ != nullptr) { + heap_caps_free(camera_preview_buffer_); + } + camera_preview_buffer_ = new_buf; + camera_preview_bytes_ = needed; + } + camera_preview_width_ = w; + camera_preview_height_ = h; + camera_active_scale_ = scale; + return true; +} + +void M5StackTab5::apply_camera_controls() { + CameraControls c; + { + std::lock_guard lock(camera_controls_mutex_); + if (!camera_controls_dirty_) { + return; + } + camera_controls_dirty_ = false; + c = camera_controls_; + } + + // Mirror / flip are applied by the PPA pass (the sensor rejects V4L2_CID_HFLIP + // / VFLIP on the capture device); just record the requested state here. These + // cannot fail. + camera_active_hmirror_ = c.hmirror; + camera_active_vflip_ = c.vflip; + + // Scale: reallocate the preview buffer if the requested scale changed. On + // failure (OOM) keep the current buffer and re-queue the request so a later + // iteration retries when memory frees, rather than silently dropping it. The + // warning is rate-limited so a sustained OOM does not spam. + if (c.scale != camera_active_scale_) { + if (allocate_camera_preview_buffer(c.scale)) { + camera_scale_alloc_warned_ = false; + } else { + if (!camera_scale_alloc_warned_) { + logger_.warn("Could not apply camera scale change (out of memory); will retry"); + camera_scale_alloc_warned_ = true; + } + std::lock_guard lock(camera_controls_mutex_); + camera_controls_dirty_ = true; + } + } +} + +void M5StackTab5::set_camera_controls(const CameraControls &controls) { + std::lock_guard lock(camera_controls_mutex_); + camera_controls_ = controls; + camera_controls_dirty_ = true; +} + +M5StackTab5::CameraControls M5StackTab5::camera_controls() const { + std::lock_guard lock(camera_controls_mutex_); + return camera_controls_; +} + +bool M5StackTab5::camera_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + // Apply any pending control changes (scale / mirror / flip) here so the PPA + // scale and the preview-buffer realloc happen in this one thread. + apply_camera_controls(); + // Dequeue a filled frame, hand it to the callback (valid only for the call), + // then requeue the buffer for reuse. The fd is non-blocking, so if no frame + // is ready yet the DQBUF fails and we wait briefly - this keeps the task + // returning regularly so a stop request is observed promptly. + struct v4l2_buffer buf = {}; + buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.memory = V4L2_MEMORY_MMAP; + if (ioctl(camera_fd_, VIDIOC_DQBUF, &buf) == 0) { + if (camera_callback_ && buf.index < CAMERA_BUFFER_COUNT && camera_buffers_[buf.index] && + camera_ppa_client_ && camera_preview_buffer_) { + // Rotate the frame to match the current display orientation. The camera is + // fixed to the tablet, so as the display is rotated the captured scene + // tilts; rotating here keeps the scene upright in the (rotated) UI. Two + // parts: (1) follow the display rotation using the same LVGL->step mapping + // as the display flush (LVGL 90/180/270 -> 1/2/3 quarter-turns), and (2) + // add a fixed mount offset, because the sensor is mounted turned 90 deg + // clockwise relative to the panel. If the feed is still turned, adjust + // kCameraMountSteps (0..3, each step = 90 deg CCW). For a net 90/270 turn + // the output width/height are swapped. + static constexpr int kCameraMountSteps = 3; // +270 CCW == 90 deg clockwise + // Read the display rotation from the atomic cached by the display flush; + // the LVGL rotation enum values are 0/1/2/3 quarter-turns. Do not call + // LVGL from this (non-GUI) thread. + int disp_steps = camera_display_rotation_.load(std::memory_order_relaxed) & 3; + int steps = (disp_steps + kCameraMountSteps) & 3; + ppa_srm_rotation_angle_t angle = PPA_SRM_ROTATION_ANGLE_0; + uint16_t out_w = camera_preview_width_; + uint16_t out_h = camera_preview_height_; + if (steps == 1) { + angle = PPA_SRM_ROTATION_ANGLE_90; + out_w = camera_preview_height_; + out_h = camera_preview_width_; + } else if (steps == 2) { + angle = PPA_SRM_ROTATION_ANGLE_180; + } else if (steps == 3) { + angle = PPA_SRM_ROTATION_ANGLE_270; + out_w = camera_preview_height_; + out_h = camera_preview_width_; + } + // Downscale + rotate the frame into the preview buffer in one PPA pass. + ppa_srm_oper_config_t srm = {}; + srm.in.buffer = camera_buffers_[buf.index]; + srm.in.pic_w = camera_width_; + srm.in.pic_h = camera_height_; + srm.in.block_w = camera_width_; + srm.in.block_h = camera_height_; + srm.in.srm_cm = PPA_SRM_COLOR_MODE_RGB565; + srm.out.buffer = camera_preview_buffer_; + srm.out.buffer_size = camera_preview_bytes_; + srm.out.pic_w = out_w; + srm.out.pic_h = out_h; + srm.out.srm_cm = PPA_SRM_COLOR_MODE_RGB565; + srm.rotation_angle = angle; + // scale is applied in the input axes (before rotation), so it is the same + // regardless of the rotation angle. + srm.scale_x = static_cast(camera_preview_width_) / static_cast(camera_width_); + srm.scale_y = static_cast(camera_preview_height_) / static_cast(camera_height_); + // Mirror / flip in the same hardware pass (the sensor rejects flip on the + // capture device). If a flip comes out on the wrong axis in a rotated + // orientation, swap these two. + srm.mirror_x = camera_active_hmirror_; + srm.mirror_y = camera_active_vflip_; + srm.mode = PPA_TRANS_MODE_BLOCKING; + if (ppa_do_scale_rotate_mirror(camera_ppa_client_, &srm) == ESP_OK) { + const size_t preview_len = static_cast(out_w) * out_h * 2; + camera_callback_(camera_preview_buffer_, out_w, out_h, preview_len); + } + } + // Requeue the buffer. If this fails the capture queue drains and the stream + // stalls, so treat it as fatal to the task rather than spinning silently. + if (ioctl(camera_fd_, VIDIOC_QBUF, &buf) != 0) { + logger_.error("VIDIOC_QBUF failed (errno {}); stopping the camera task", errno); + return true; // stop the task; the owner can stop_camera() / re-init + } + } else if (errno == EAGAIN) { + // No frame ready yet on the non-blocking fd; wait briefly and retry. + vTaskDelay(pdMS_TO_TICKS(5)); + } else { + // A real capture error (device/stream/driver): surface it and stop the task + // instead of spinning forever with no diagnostics. + logger_.error("VIDIOC_DQBUF failed (errno {}); stopping the camera task", errno); + return true; + } + // honor a stop request per the Task contract: check/clear notified under m + std::unique_lock lock(m); + if (task_notified) { + task_notified = false; + return true; // stop the task + } + return false; // keep running +} + +void M5StackTab5::stop_camera() { + if (camera_task_) { + camera_task_->stop(); + camera_task_.reset(); + } + if (camera_fd_ >= 0) { + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + ioctl(camera_fd_, VIDIOC_STREAMOFF, &type); + } + for (int i = 0; i < CAMERA_BUFFER_COUNT; ++i) { + if (camera_buffers_[i]) { + munmap(camera_buffers_[i], camera_buffer_sizes_[i]); + camera_buffers_[i] = nullptr; + camera_buffer_sizes_[i] = 0; + } + } + if (camera_fd_ >= 0) { + close(camera_fd_); + camera_fd_ = -1; + } + if (camera_ppa_client_) { + ppa_unregister_client(camera_ppa_client_); + camera_ppa_client_ = nullptr; + } + if (camera_preview_buffer_) { + heap_caps_free(camera_preview_buffer_); + camera_preview_buffer_ = nullptr; + camera_preview_bytes_ = 0; + } + // Restore the ISP log tags we muted while streaming BEFORE deinit, so any + // teardown diagnostics esp_video_deinit() emits under those tags are visible. + if (camera_logs_silenced_) { + for (size_t i = 0; i < std::size(kCameraSilencedLogTags); ++i) { + esp_log_level_set(kCameraSilencedLogTags[i], + static_cast(camera_prev_log_levels_[i])); + } + camera_logs_silenced_ = false; + } + if (camera_video_inited_) { + esp_video_deinit(); + camera_video_inited_ = false; + } + // Reset the reported dimensions so camera_width()/height() honor their + // documented "0 when not initialized" contract after a stop or a failure. + camera_buffer_count_ = 0; + camera_width_ = 0; + camera_height_ = 0; + camera_preview_width_ = 0; + camera_preview_height_ = 0; + camera_initialized_ = false; + camera_callback_ = nullptr; +} + +uint16_t M5StackTab5::camera_width() const { return camera_width_; } + +uint16_t M5StackTab5::camera_height() const { return camera_height_; } diff --git a/components/m5stack-tab5/src/video.cpp b/components/m5stack-tab5/src/video.cpp index 6ffd982d2..678aa2168 100755 --- a/components/m5stack-tab5/src/video.cpp +++ b/components/m5stack-tab5/src/video.cpp @@ -437,6 +437,10 @@ void M5StackTab5::flush(lv_display_t *disp, const lv_area_t *area, uint8_t *px_m int offsety2 = area->y2; auto rotation = lv_display_get_rotation(lv_display_get_default()); + // Cache the rotation for the camera task, which must not call LVGL from its + // own thread. flush() runs on the LVGL (GUI) thread, so reading it here is + // safe; the camera task reads the cached atomic instead. + camera_display_rotation_.store(static_cast(rotation), std::memory_order_relaxed); if (rotation > LV_DISPLAY_ROTATION_0 && third_buffer != nullptr) { int32_t ww = lv_area_get_width(area); int32_t hh = lv_area_get_height(area);