[nrf52, ota] ble and serial OTA based on mcumgr (#11932)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: J. Nick Koston <nick@koston.org>
Co-authored-by: J. Nick Koston <nick+github@koston.org>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

CODEOWNERS

@@ -587,6 +587,7 @@ esphome/components/xl9535/* @mreditor97
 esphome/components/xpt2046/touchscreen/* @nielsnl68 @numo68
 esphome/components/xxtea/* @clydebarrow
 esphome/components/zephyr/* @tomaszduda23
+esphome/components/zephyr_mcumgr/ota/* @tomaszduda23
 esphome/components/zhlt01/* @cfeenstra1024
 esphome/components/zigbee/* @tomaszduda23
 esphome/components/zio_ultrasonic/* @kahrendt

esphome/components/nrf52/__init__.py

@@ -27,10 +27,15 @@ from esphome.components.zephyr.const import (
 )
 import esphome.config_validation as cv
 from esphome.const import (
+    CONF_ADVANCED,
     CONF_BOARD,
+    CONF_DISABLED,
+    CONF_ENABLE_OTA_ROLLBACK,
     CONF_FRAMEWORK,
     CONF_ID,
+    CONF_OTA,
     CONF_RESET_PIN,
+    CONF_SAFE_MODE,
     CONF_VERSION,
     CONF_VOLTAGE,
     KEY_CORE,
@@ -41,6 +46,7 @@ from esphome.const import (
     ThreadModel,
 )
 from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority
+import esphome.final_validate as fv
 from esphome.storage_json import StorageJSON
 from esphome.types import ConfigType
 
@@ -133,6 +139,7 @@ CONF_UICR_ERASE = "uicr_erase"
 
 VOLTAGE_LEVELS = [1.8, 2.1, 2.4, 2.7, 3.0, 3.3]
 
+
 CONFIG_SCHEMA = cv.All(
     _detect_bootloader,
     set_core_data,
@@ -156,9 +163,19 @@ CONFIG_SCHEMA = cv.All(
                     cv.Optional(CONF_UICR_ERASE, default=False): cv.boolean,
                 }
             ),
-            cv.Optional(CONF_FRAMEWORK, default={CONF_VERSION: "2.6.1-a"}): cv.Schema(
+            cv.Optional(
+                CONF_FRAMEWORK,
+                default={},
+            ): cv.Schema(
                 {
-                    cv.Required(CONF_VERSION): cv.string_strict,
+                    cv.Optional(CONF_VERSION, default="2.6.1-a"): cv.string_strict,
+                    cv.Optional(CONF_ADVANCED, default={}): cv.Schema(
+                        {
+                            cv.Optional(
+                                CONF_ENABLE_OTA_ROLLBACK, default=True
+                            ): cv.boolean,
+                        }
+                    ),
                 }
             ),
             cv.GenerateID(CONF_CDC_ACM): cv.declare_id(CdcAcm),
@@ -181,6 +198,24 @@ def _final_validate(config):
         _LOGGER.warning(
             "Selected generic Adafruit bootloader. The board might crash. Consider settings `bootloader:`"
         )
+    full_config = fv.full_config.get()
+    conf = config[CONF_FRAMEWORK]
+    advanced = conf[CONF_ADVANCED]
+
+    if advanced[CONF_ENABLE_OTA_ROLLBACK]:
+        # "disabled: false" means safe mode *is* enabled.
+        safe_mode_config = full_config.get(CONF_SAFE_MODE, {CONF_DISABLED: True})
+        safe_mode_enabled = not safe_mode_config[CONF_DISABLED]
+        ota_enabled = CONF_OTA in full_config
+        # Both need to be enabled for rollback to work
+        if not (ota_enabled and safe_mode_enabled):
+            # But only warn if ota is even possible
+            if ota_enabled:
+                _LOGGER.warning(
+                    "OTA rollback requires safe_mode, disabling rollback support"
+                )
+            # disable the rollback feature anyway since it can't be used.
+            advanced[CONF_ENABLE_OTA_ROLLBACK] = False
 
 
 FINAL_VALIDATE_SCHEMA = _final_validate
@@ -247,6 +282,11 @@ async def to_code(config: ConfigType) -> None:
         if reg0_config[CONF_UICR_ERASE]:
             cg.add_define("USE_NRF52_UICR_ERASE")
 
+    conf = config[CONF_FRAMEWORK]
+    advanced = conf[CONF_ADVANCED]
+    # Enable OTA rollback support
+    if advanced[CONF_ENABLE_OTA_ROLLBACK]:
+        cg.add_define("USE_OTA_ROLLBACK")
     # c++ support
     if framework_ver < cv.Version(2, 9, 2):
         zephyr_add_prj_conf("CPLUSPLUS", True)
@@ -259,7 +299,7 @@ async def to_code(config: ConfigType) -> None:
     zephyr_add_prj_conf("WDT_DISABLE_AT_BOOT", False)
     # disable console
     zephyr_add_prj_conf("UART_CONSOLE", False)
-    zephyr_add_prj_conf("CONSOLE", False)
+    zephyr_add_prj_conf("CONSOLE", False, False)
     # use NFC pins as GPIO
     if framework_ver < cv.Version(2, 9, 2):
         zephyr_add_prj_conf("NFCT_PINS_AS_GPIOS", True)

esphome/components/ota/__init__.py

@@ -18,7 +18,14 @@ from esphome.coroutine import CoroPriority
 OTA_STATE_LISTENER_KEY = "ota_state_listener"
 
 CODEOWNERS = ["@esphome/core"]
-AUTO_LOAD = ["md5", "safe_mode"]
+
+
+def AUTO_LOAD() -> list[str]:
+    components = ["safe_mode"]
+    if not CORE.using_zephyr:
+        components.extend(["md5"])
+    return components
+
 
 IS_PLATFORM_COMPONENT = True
 

esphome/components/safe_mode/safe_mode.cpp

@@ -9,10 +9,14 @@
 #include <cinttypes>
 #include <cstdio>
 
-#if defined(USE_ESP32) && defined(USE_OTA_ROLLBACK)
+#ifdef USE_OTA_ROLLBACK
+#ifdef USE_ZEPHYR
+#include <zephyr/dfu/mcuboot.h>
+#elif defined(USE_ESP32)
 #include <esp_ota_ops.h>
 #include <esp_system.h>
 #endif
+#endif
 
 namespace esphome::safe_mode {
 
@@ -66,9 +70,16 @@ float SafeModeComponent::get_setup_priority() const { return setup_priority::AFT
 void SafeModeComponent::mark_successful() {
   this->clean_rtc();
   this->boot_successful_ = true;
-#if defined(USE_ESP32) && defined(USE_OTA_ROLLBACK)
+#if defined(USE_OTA_ROLLBACK)
+// Mark OTA partition as valid to prevent rollback
+#if defined(USE_ZEPHYR)
+  if (!boot_is_img_confirmed()) {
+    boot_write_img_confirmed();
+  }
+#elif defined(USE_ESP32)
   // Mark OTA partition as valid to prevent rollback
   esp_ota_mark_app_valid_cancel_rollback();
+#endif
 #endif
   // Disable loop since we no longer need to check
   this->disable_loop();

esphome/components/zephyr_mcumgr/ota/__init__.py

@@ -0,0 +1,141 @@
+import esphome.codegen as cg
+from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code
+from esphome.components.zephyr import (
+    zephyr_add_cdc_acm,
+    zephyr_add_overlay,
+    zephyr_add_prj_conf,
+    zephyr_data,
+)
+from esphome.components.zephyr.const import BOOTLOADER_MCUBOOT, KEY_BOOTLOADER
+import esphome.config_validation as cv
+from esphome.const import CONF_HARDWARE_UART, CONF_ID, Framework
+from esphome.core import CORE, coroutine_with_priority
+from esphome.coroutine import CoroPriority
+from esphome.types import ConfigType
+
+CODEOWNERS = ["@tomaszduda23"]
+DEPENDENCIES = ["zephyr"]
+
+ZephyrMcumgrOTAComponent = cg.esphome_ns.namespace("zephyr_mcumgr").class_(
+    "OTAComponent", OTAComponent
+)
+
+CONF_BLE = "ble"
+CONF_TRANSPORT = "transport"
+
+
+def _validate_transport(conf: ConfigType) -> ConfigType:
+    transport = conf[CONF_TRANSPORT]
+    if transport[CONF_BLE] or CONF_HARDWARE_UART in transport:
+        return conf
+    raise cv.Invalid(
+        f"At least one transport protocol has to be enabled. Set '{CONF_BLE}: true' or '{CONF_HARDWARE_UART}'"
+    )
+
+
+UARTS = {
+    "CDC": ("cdc_acm_uart0", 0),
+    "CDC1": ("cdc_acm_uart1", 1),
+    "UART0": ("uart0", -1),
+    "UART1": ("uart1", -1),
+}
+
+
+CONFIG_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(ZephyrMcumgrOTAComponent),
+            cv.Optional(CONF_TRANSPORT, default={CONF_BLE: True}): cv.Schema(
+                {
+                    cv.Optional(CONF_BLE, default=False): cv.boolean,
+                    cv.Optional(
+                        CONF_HARDWARE_UART,
+                    ): cv.one_of(*UARTS, upper=True),
+                }
+            ),
+        }
+    )
+    .extend(BASE_OTA_SCHEMA)
+    .extend(cv.COMPONENT_SCHEMA),
+    _validate_transport,
+    cv.only_with_framework(Framework.ZEPHYR),
+)
+
+
+def _validate_mcumgr_bootloader(config: ConfigType) -> None:
+    bootloader = zephyr_data()[KEY_BOOTLOADER]
+    if bootloader != BOOTLOADER_MCUBOOT:
+        raise cv.Invalid(f"'{bootloader}' bootloader does not support OTA")
+
+
+KEY_ZEPHYR_BLE_SERVER = "zephyr_ble_server"
+
+
+def _validate_ble_server(config: ConfigType) -> None:
+    if (
+        config[CONF_TRANSPORT][CONF_BLE]
+        and KEY_ZEPHYR_BLE_SERVER not in CORE.loaded_integrations
+    ):
+        raise cv.Invalid(f"'{KEY_ZEPHYR_BLE_SERVER}' component is required for BLE OTA")
+
+
+def _final_validate(config: ConfigType) -> None:
+    _validate_mcumgr_bootloader(config)
+    _validate_ble_server(config)
+
+
+FINAL_VALIDATE_SCHEMA = _final_validate
+
+
+@coroutine_with_priority(CoroPriority.OTA_UPDATES)
+async def to_code(config: ConfigType) -> None:
+    var = cg.new_Pvariable(config[CONF_ID])
+    await ota_to_code(var, config)
+
+    await cg.register_component(var, config)
+
+    zephyr_add_prj_conf("NET_BUF", True)
+    zephyr_add_prj_conf("ZCBOR", True)
+    zephyr_add_prj_conf("MCUMGR", True)
+
+    zephyr_add_prj_conf("MCUMGR_GRP_IMG", True)
+
+    zephyr_add_prj_conf("IMG_MANAGER", True)
+    zephyr_add_prj_conf("STREAM_FLASH", True)
+    zephyr_add_prj_conf("FLASH_MAP", True)
+    zephyr_add_prj_conf("FLASH", True)
+
+    zephyr_add_prj_conf("IMG_ERASE_PROGRESSIVELY", True)
+
+    zephyr_add_prj_conf("BOOTLOADER_MCUBOOT", True)
+
+    zephyr_add_prj_conf("MCUMGR_MGMT_NOTIFICATION_HOOKS", True)
+    zephyr_add_prj_conf("MCUMGR_GRP_IMG_STATUS_HOOKS", True)
+    zephyr_add_prj_conf("MCUMGR_GRP_IMG_UPLOAD_CHECK_HOOK", True)
+    transport = config[CONF_TRANSPORT]
+    if transport[CONF_BLE]:
+        zephyr_add_prj_conf("MCUMGR_TRANSPORT_BT", True)
+        zephyr_add_prj_conf("MCUMGR_TRANSPORT_BT_REASSEMBLY", True)
+
+        zephyr_add_prj_conf("MCUMGR_GRP_OS", True)
+        zephyr_add_prj_conf("MCUMGR_GRP_OS_MCUMGR_PARAMS", True)
+
+        zephyr_add_prj_conf("NCS_SAMPLE_MCUMGR_BT_OTA_DFU_SPEEDUP", True)
+    if CONF_HARDWARE_UART in transport:
+        uart = UARTS[transport[CONF_HARDWARE_UART]]
+        uart_name = uart[0]
+        cdc_id = uart[1]
+        if cdc_id >= 0:
+            zephyr_add_cdc_acm(config, cdc_id)
+        zephyr_add_prj_conf("MCUMGR_TRANSPORT_UART", True)
+        zephyr_add_prj_conf("BASE64", True)
+        zephyr_add_prj_conf("CONSOLE", True)
+        zephyr_add_overlay(
+            f"""
+                / {{
+                    chosen {{
+                        zephyr,uart-mcumgr = &{uart_name};
+                    }};
+                }};
+                """
+        )

esphome/components/zephyr_mcumgr/ota/ota_zephyr_mcumgr.cpp

@@ -0,0 +1,143 @@
+#ifdef USE_ZEPHYR
+#include "ota_zephyr_mcumgr.h"
+#include "esphome/core/log.h"
+#include "esphome/core/hal.h"
+#include <zephyr/sys/math_extras.h>
+#include <zephyr/usb/usb_device.h>
+#include <zephyr/dfu/mcuboot.h>
+
+// It should be from below header but there is problem with internal includes.
+// #include <zephyr/mgmt/mcumgr/grp/img_mgmt/img_mgmt.h>
+// NOLINTBEGIN(readability-identifier-naming,google-runtime-int)
+struct img_mgmt_upload_action {
+  /** The total size of the image. */
+  unsigned long long size;
+};
+
+struct img_mgmt_upload_req {
+  uint32_t image; /* 0 by default */
+  size_t off;     /* SIZE_MAX if unspecified */
+};
+// NOLINTEND(readability-identifier-naming,google-runtime-int)
+
+namespace esphome::zephyr_mcumgr {
+
+static_assert(sizeof(struct img_mgmt_upload_action) == 8, "ABI mismatch");
+static_assert(sizeof(struct img_mgmt_upload_req) == 8, "ABI mismatch");
+static_assert(offsetof(struct img_mgmt_upload_req, image) == 0, "ABI mismatch");
+static_assert(offsetof(struct img_mgmt_upload_req, off) == 4, "ABI mismatch");
+
+static const char *const TAG = "zephyr_mcumgr";
+static OTAComponent *global_ota_component;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+
+static enum mgmt_cb_return mcumgr_img_mgmt_cb(uint32_t event, enum mgmt_cb_return prev_status, int32_t *rc,
+                                              uint16_t *group, bool *abort_more, void *data, size_t data_size) {
+  if (MGMT_EVT_OP_IMG_MGMT_DFU_CHUNK == event) {
+    const img_mgmt_upload_check &upload = *static_cast<img_mgmt_upload_check *>(data);
+    global_ota_component->update_chunk(upload);
+  } else if (MGMT_EVT_OP_IMG_MGMT_DFU_STARTED == event) {
+    global_ota_component->update_started();
+  } else if (MGMT_EVT_OP_IMG_MGMT_DFU_CHUNK_WRITE_COMPLETE == event) {
+    global_ota_component->update_chunk_wrote();
+  } else if (MGMT_EVT_OP_IMG_MGMT_DFU_PENDING == event) {
+    global_ota_component->update_pending();
+  } else if (MGMT_EVT_OP_IMG_MGMT_DFU_STOPPED == event) {
+    global_ota_component->update_stopped();
+  } else {
+    ESP_LOGD(TAG, "MCUmgr Image Management Event with the %d ID", u32_count_trailing_zeros(MGMT_EVT_GET_ID(event)));
+  }
+  return MGMT_CB_OK;
+}
+
+OTAComponent::OTAComponent() { global_ota_component = this; }
+
+void OTAComponent::setup() {
+  this->img_mgmt_callback_.callback = mcumgr_img_mgmt_cb;
+  this->img_mgmt_callback_.event_id = MGMT_EVT_OP_IMG_MGMT_ALL;
+  mgmt_callback_register(&this->img_mgmt_callback_);
+#ifdef CONFIG_USB_DEVICE_STACK
+  usb_enable(nullptr);
+#endif
+// Handle OTA rollback: mark partition valid immediately unless USE_OTA_ROLLBACK is enabled,
+// in which case safe_mode will mark it valid after confirming successful boot.
+#ifndef USE_OTA_ROLLBACK
+  if (!boot_is_img_confirmed()) {
+    boot_write_img_confirmed();
+  }
+#endif
+}
+
+#ifdef ESPHOME_LOG_HAS_CONFIG
+static const char *swap_type_str(uint8_t type) {
+  switch (type) {
+    case BOOT_SWAP_TYPE_NONE:
+      return "none";
+    case BOOT_SWAP_TYPE_TEST:
+      return "test";
+    case BOOT_SWAP_TYPE_PERM:
+      return "perm";
+    case BOOT_SWAP_TYPE_REVERT:
+      return "revert";
+    case BOOT_SWAP_TYPE_FAIL:
+      return "fail";
+  }
+
+  return "unknown";
+}
+#endif
+
+void OTAComponent::dump_config() {
+  ESP_LOGCONFIG(TAG,
+                "Over-The-Air Updates:\n"
+                "  swap type after reboot: %s\n"
+                "  image confirmed: %s",
+                swap_type_str(mcuboot_swap_type()), YESNO(boot_is_img_confirmed()));
+}
+
+void OTAComponent::update_chunk(const img_mgmt_upload_check &upload) {
+  float percentage = (upload.req->off * 100.0f) / upload.action->size;
+  this->defer([this, percentage]() { this->percentage_ = percentage; });
+}
+
+void OTAComponent::update_started() {
+  this->defer([this]() {
+    ESP_LOGD(TAG, "Starting update");
+#ifdef USE_OTA_STATE_LISTENER
+    this->notify_state_(ota::OTA_STARTED, 0.0f, 0);
+#endif
+  });
+}
+
+void OTAComponent::update_chunk_wrote() {
+  uint32_t now = millis();
+  if (now - this->last_progress_ > 1000) {
+    this->last_progress_ = now;
+    this->defer([this]() {
+      ESP_LOGD(TAG, "OTA in progress: %0.1f%%", this->percentage_);
+#ifdef USE_OTA_STATE_LISTENER
+      this->notify_state_(ota::OTA_IN_PROGRESS, this->percentage_, 0);
+#endif
+    });
+  }
+}
+
+void OTAComponent::update_pending() {
+  this->defer([this]() {
+    ESP_LOGD(TAG, "OTA pending");
+#ifdef USE_OTA_STATE_LISTENER
+    this->notify_state_(ota::OTA_COMPLETED, 100.0f, 0);
+#endif
+  });
+}
+
+void OTAComponent::update_stopped() {
+  this->defer([this]() {
+    ESP_LOGD(TAG, "OTA stopped");
+#ifdef USE_OTA_STATE_LISTENER
+    this->notify_state_(ota::OTA_ERROR, 0.0f, static_cast<uint8_t>(ota::OTA_RESPONSE_ERROR_UNKNOWN));
+#endif
+  });
+}
+
+}  // namespace esphome::zephyr_mcumgr
+#endif

esphome/components/zephyr_mcumgr/ota/ota_zephyr_mcumgr.h

@@ -0,0 +1,29 @@
+#pragma once
+#include "esphome/core/defines.h"
+#ifdef USE_ZEPHYR
+#include "esphome/components/ota/ota_backend.h"
+#include <zephyr/mgmt/mcumgr/mgmt/callbacks.h>
+
+struct img_mgmt_upload_check;
+
+namespace esphome::zephyr_mcumgr {
+
+class OTAComponent : public ota::OTAComponent {
+ public:
+  OTAComponent();
+  void setup() override;
+  void dump_config() override;
+  void update_chunk(const img_mgmt_upload_check &upload);
+  void update_started();
+  void update_chunk_wrote();
+  void update_pending();
+  void update_stopped();
+
+ protected:
+  uint32_t last_progress_ = 0;
+  float percentage_ = 0;
+  mgmt_callback img_mgmt_callback_{};
+};
+
+}  // namespace esphome::zephyr_mcumgr
+#endif

esphome/core/defines.h

@@ -367,6 +367,7 @@
 #define USE_NRF52_DFU
 #define USE_NRF52_REG0_VOUT 5
 #define USE_NRF52_UICR_ERASE
+#define USE_OTA_ROLLBACK
 #define USE_SOFTDEVICE_ID 7
 #define USE_SOFTDEVICE_VERSION 1
 #define USE_ZIGBEE

script/helpers_zephyr.py

@@ -28,6 +28,22 @@ extern "C" void zboss_signal_handler() {};
 CONFIG_NEWLIB_LIBC=y
 CONFIG_BT=y
 CONFIG_ADC=y
+#mcumgr begin
+CONFIG_NET_BUF=y
+CONFIG_ZCBOR=y
+CONFIG_MCUMGR=y
+CONFIG_MCUMGR_GRP_IMG=y
+CONFIG_IMG_MANAGER=y
+CONFIG_STREAM_FLASH=y
+CONFIG_FLASH_MAP=y
+CONFIG_FLASH=y
+CONFIG_IMG_ERASE_PROGRESSIVELY=y
+CONFIG_BOOTLOADER_MCUBOOT=y
+CONFIG_MCUMGR_MGMT_NOTIFICATION_HOOKS=y
+CONFIG_MCUMGR_GRP_IMG_STATUS_HOOKS=y
+CONFIG_MCUMGR_GRP_IMG_UPLOAD_CHECK_HOOK=y
+CONFIG_MCUMGR_TRANSPORT_UART=y
+#mcumgr end
 #zigbee begin
 CONFIG_ZIGBEE=y
 CONFIG_CRYPTO=y

tests/components/ota/test.nrf52-mcumgr.yaml

@@ -0,0 +1,27 @@
+zephyr_ble_server:
+
+ota:
+  - platform: zephyr_mcumgr
+    transport:
+      ble: true
+      hardware_uart: CDC
+    on_begin:
+      then:
+        - logger.log: "OTA start"
+    on_progress:
+      then:
+        - logger.log:
+            format: "OTA progress %0.1f%%"
+            args: ["x"]
+    on_end:
+      then:
+        - logger.log: "OTA end"
+    on_error:
+      then:
+        - logger.log:
+            format: "OTA update error %d"
+            args: ["x"]
+    on_state_change:
+      then:
+        lambda: >-
+          ESP_LOGD("ota", "State %d", state);