From 690a1973460f5492e67bc06ff4606afc5f778c69 Mon Sep 17 00:00:00 2001 From: Diorcet Yann Date: Mon, 4 May 2026 21:07:31 +0200 Subject: [PATCH 01/60] [main] Move stacktrace handling out of platformio_api and FlashImage into platform components/util (#16186) --- esphome/__main__.py | 26 ++- esphome/components/api/client.py | 10 +- esphome/components/esp32/__init__.py | 76 +++++++++ esphome/components/esp8266/__init__.py | 115 ++++++++++++++ esphome/platformio_api.py | 150 +----------------- esphome/util.py | 6 + .../unit_tests/components/api/test_client.py | 23 +-- .../components/test_esp_stacktrace.py | 109 +++++++++++++ tests/unit_tests/conftest.py | 13 +- tests/unit_tests/test_main.py | 23 +-- tests/unit_tests/test_platformio_api.py | 100 +----------- 11 files changed, 359 insertions(+), 292 deletions(-) create mode 100644 tests/unit_tests/components/test_esp_stacktrace.py diff --git a/esphome/__main__.py b/esphome/__main__.py index 9ab2dee189..222e299f6d 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -63,6 +63,7 @@ from esphome.log import AnsiFore, color, setup_log from esphome.types import ConfigType from esphome.util import ( PICOTOOL_PACKAGE, + FlashImage, detect_rp2040_bootsel, get_picotool_path, get_serial_ports, @@ -586,8 +587,6 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: from aioesphomeapi import LogParser import serial - from esphome import platformio_api - if CONF_LOGGER not in config: _LOGGER.info("Logger is not enabled. Not starting UART logs.") return 1 @@ -602,8 +601,11 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: try: module = importlib.import_module("esphome.components." + CORE.target_platform) process_stacktrace = getattr(module, "process_stacktrace") - except AttributeError: - pass + except (AttributeError, ImportError): + _LOGGER.info( + 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', + CORE.target_platform, + ) backtrace_state = False ser = serial.Serial() @@ -646,14 +648,10 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: ) safe_print(parser.parse_line(line, time_str)) - if process_stacktrace: + if process_stacktrace is not None: backtrace_state = process_stacktrace( config, line, backtrace_state ) - else: - backtrace_state = platformio_api.process_stacktrace( - config, line, backtrace_state=backtrace_state - ) except serial.SerialException: _LOGGER.error("Serial port closed!") return 0 @@ -843,22 +841,20 @@ def _make_crystal_freq_callback( def upload_using_esptool( config: ConfigType, port: str, file: str, speed: int ) -> str | int: - from esphome import platformio_api - first_baudrate = speed or config[CONF_ESPHOME][CONF_PLATFORMIO_OPTIONS].get( "upload_speed", os.getenv("ESPHOME_UPLOAD_SPEED", "460800") ) if file is not None: - flash_images = [platformio_api.FlashImage(path=file, offset="0x0")] + flash_images = [FlashImage(path=file, offset="0x0")] else: + from esphome import platformio_api + idedata = platformio_api.get_idedata(config) firmware_offset = "0x10000" if CORE.is_esp32 else "0x0" flash_images = [ - platformio_api.FlashImage( - path=idedata.firmware_bin_path, offset=firmware_offset - ), + FlashImage(path=idedata.firmware_bin_path, offset=firmware_offset), ] for image in idedata.extra_flash_images: if not image.path.is_file(): diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index d5214ccbf6..0a5370cb30 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -19,7 +19,6 @@ import contextlib from esphome.const import CONF_KEY, CONF_PORT, __version__ from esphome.core import CORE, EsphomeError -from esphome.platformio_api import process_stacktrace from . import CONF_ENCRYPTION @@ -61,10 +60,6 @@ class _LogLineProcessor: self.backtrace_state = self._platform_handler( self._config, raw_line, self.backtrace_state ) - else: - self.backtrace_state = process_stacktrace( - self._config, raw_line, backtrace_state=self.backtrace_state - ) except EsphomeError as exc: self._decode_enabled = False self.backtrace_state = False @@ -114,7 +109,10 @@ async def async_run_logs( module = importlib.import_module("esphome.components." + CORE.target_platform) platform_process_stacktrace = getattr(module, "process_stacktrace") except (AttributeError, ImportError): - pass + _LOGGER.info( + 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', + CORE.target_platform, + ) processor = _LogLineProcessor(config, platform_process_stacktrace) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index b60dab3634..582721ef73 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -5,6 +5,7 @@ import logging import os from pathlib import Path import re +import subprocess from esphome import yaml_util import esphome.codegen as cg @@ -2515,3 +2516,78 @@ def copy_files(): CORE.relative_build_path(name).write_bytes(content) else: copy_file_if_changed(path, CORE.relative_build_path(name)) + + +def _decode_pc(config, addr): + from esphome import platformio_api + + idedata = platformio_api.get_idedata(config) + if not idedata.addr2line_path or not idedata.firmware_elf_path: + _LOGGER.debug("decode_pc no addr2line") + return + command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + try: + translation = subprocess.check_output(command, close_fds=False).decode().strip() + except Exception: # pylint: disable=broad-except + _LOGGER.debug("Caught exception for command %s", command, exc_info=1) + return + + if "?? ??:0" in translation: + # Nothing useful + return + translation = translation.replace(" at ??:?", "").replace(":?", "") + _LOGGER.warning("Decoded %s", translation) + + +def _parse_register(config, regex, line): + match = regex.match(line) + if match is not None: + _decode_pc(config, match.group(1)) + + +STACKTRACE_ESP32_PC_RE = re.compile(r".*PC\s*:\s*(?:0x)?(4[0-9a-fA-F]{7}).*") +STACKTRACE_ESP32_EXCVADDR_RE = re.compile(r"EXCVADDR\s*:\s*(?:0x)?(4[0-9a-fA-F]{7})") +STACKTRACE_ESP32_C3_PC_RE = re.compile(r"MEPC\s*:\s*(?:0x)?(4[0-9a-fA-F]{7})") +STACKTRACE_ESP32_C3_RA_RE = re.compile(r"RA\s*:\s*(?:0x)?(4[0-9a-fA-F]{7})") +STACKTRACE_BAD_ALLOC_RE = re.compile( + r"^last failed alloc call: (4[0-9a-fA-F]{7})\((\d+)\)$" +) +STACKTRACE_ESP32_BACKTRACE_RE = re.compile( + r"Backtrace:(?:\s*0x[0-9a-fA-F]{8}:0x[0-9a-fA-F]{8})+" +) +STACKTRACE_ESP32_BACKTRACE_PC_RE = re.compile(r"4[0-9a-f]{7}") +# ESP32 crash handler (stored backtrace from previous boot) +STACKTRACE_ESP32_CRASH_BT_RE = re.compile(r"BT\d+:\s*0x([0-9a-fA-F]{8})") + + +def process_stacktrace(config, line, backtrace_state): + line = line.strip() + + # ESP32 PC/EXCVADDR + _parse_register(config, STACKTRACE_ESP32_PC_RE, line) + _parse_register(config, STACKTRACE_ESP32_EXCVADDR_RE, line) + # ESP32-C3 PC/RA + _parse_register(config, STACKTRACE_ESP32_C3_PC_RE, line) + _parse_register(config, STACKTRACE_ESP32_C3_RA_RE, line) + + # bad alloc + match = re.match(STACKTRACE_BAD_ALLOC_RE, line) + if match is not None: + _LOGGER.warning( + "Memory allocation of %s bytes failed at %s", match.group(2), match.group(1) + ) + _decode_pc(config, match.group(1)) + + # ESP32 crash handler backtrace (from previous boot) + match = re.search(STACKTRACE_ESP32_CRASH_BT_RE, line) + if match is not None: + _decode_pc(config, match.group(1)) + + # ESP32 single-line backtrace + match = re.match(STACKTRACE_ESP32_BACKTRACE_RE, line) + if match is not None: + _LOGGER.warning("Found stack trace! Trying to decode it") + for addr in re.finditer(STACKTRACE_ESP32_BACKTRACE_PC_RE, line): + _decode_pc(config, addr.group()) + + return backtrace_state diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 3c2806a307..b6383653f4 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -1,6 +1,7 @@ import logging from pathlib import Path import re +import subprocess import esphome.codegen as cg import esphome.config_validation as cv @@ -419,3 +420,117 @@ def copy_files() -> None: remove_float_scanf_file, CORE.relative_build_path("remove_float_scanf.py"), ) + + +# ESP logs stack trace decoder, based on https://github.com/me-no-dev/EspExceptionDecoder +ESP8266_EXCEPTION_CODES = { + 0: "Illegal instruction (Is the flash damaged?)", + 1: "SYSCALL instruction", + 2: "InstructionFetchError: Processor internal physical address or data error during " + "instruction fetch", + 3: "LoadStoreError: Processor internal physical address or data error during load or store", + 4: "Level1Interrupt: Level-1 interrupt as indicated by set level-1 bits in the INTERRUPT " + "register", + 5: "Alloca: MOVSP instruction, if caller's registers are not in the register file", + 6: "Integer Divide By Zero", + 7: "reserved", + 8: "Privileged: Attempt to execute a privileged operation when CRING ? 0", + 9: "LoadStoreAlignmentCause: Load or store to an unaligned address", + 10: "reserved", + 11: "reserved", + 12: "InstrPIFDataError: PIF data error during instruction fetch", + 13: "LoadStorePIFDataError: Synchronous PIF data error during LoadStore access", + 14: "InstrPIFAddrError: PIF address error during instruction fetch", + 15: "LoadStorePIFAddrError: Synchronous PIF address error during LoadStore access", + 16: "InstTLBMiss: Error during Instruction TLB refill", + 17: "InstTLBMultiHit: Multiple instruction TLB entries matched", + 18: "InstFetchPrivilege: An instruction fetch referenced a virtual address at a ring level " + "less than CRING", + 19: "reserved", + 20: "InstFetchProhibited: An instruction fetch referenced a page mapped with an attribute " + "that does not permit instruction fetch", + 21: "reserved", + 22: "reserved", + 23: "reserved", + 24: "LoadStoreTLBMiss: Error during TLB refill for a load or store", + 25: "LoadStoreTLBMultiHit: Multiple TLB entries matched for a load or store", + 26: "LoadStorePrivilege: A load or store referenced a virtual address at a ring level less " + "than ", + 27: "reserved", + 28: "Access to invalid address: LOAD (wild pointer?)", + 29: "Access to invalid address: STORE (wild pointer?)", +} + + +def _decode_pc(config, addr): + from esphome import platformio_api + + idedata = platformio_api.get_idedata(config) + if not idedata.addr2line_path or not idedata.firmware_elf_path: + _LOGGER.debug("decode_pc no addr2line") + return + command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + try: + translation = subprocess.check_output(command, close_fds=False).decode().strip() + except Exception: # pylint: disable=broad-except + _LOGGER.debug("Caught exception for command %s", command, exc_info=1) + return + + if "?? ??:0" in translation: + # Nothing useful + return + translation = translation.replace(" at ??:?", "").replace(":?", "") + _LOGGER.warning("Decoded %s", translation) + + +def _parse_register(config, regex, line): + match = regex.match(line) + if match is not None: + _decode_pc(config, match.group(1)) + + +STACKTRACE_ESP8266_EXCEPTION_TYPE_RE = re.compile(r"[eE]xception \((\d+)\):") +STACKTRACE_ESP8266_PC_RE = re.compile(r"epc1=0x(4[0-9a-fA-F]{7})") +STACKTRACE_ESP8266_EXCVADDR_RE = re.compile(r"excvaddr=0x(4[0-9a-fA-F]{7})") +STACKTRACE_BAD_ALLOC_RE = re.compile( + r"^last failed alloc call: (4[0-9a-fA-F]{7})\((\d+)\)$" +) +STACKTRACE_ESP8266_BACKTRACE_PC_RE = re.compile(r"4[0-9a-f]{7}") + + +def process_stacktrace(config, line, backtrace_state): + line = line.strip() + # ESP8266 Exception type + match = re.match(STACKTRACE_ESP8266_EXCEPTION_TYPE_RE, line) + if match is not None: + code = int(match.group(1)) + _LOGGER.warning( + "Exception type: %s", ESP8266_EXCEPTION_CODES.get(code, "unknown") + ) + + # ESP8266 PC/EXCVADDR + _parse_register(config, STACKTRACE_ESP8266_PC_RE, line) + _parse_register(config, STACKTRACE_ESP8266_EXCVADDR_RE, line) + + # bad alloc + match = re.match(STACKTRACE_BAD_ALLOC_RE, line) + if match is not None: + _LOGGER.warning( + "Memory allocation of %s bytes failed at %s", match.group(2), match.group(1) + ) + _decode_pc(config, match.group(1)) + + # ESP8266 multi-line backtrace + if ">>>stack>>>" in line: + # Start of backtrace + backtrace_state = True + _LOGGER.warning("Found stack trace! Trying to decode it") + elif "<< "IDEData": return idedata -# ESP logs stack trace decoder, based on https://github.com/me-no-dev/EspExceptionDecoder -ESP8266_EXCEPTION_CODES = { - 0: "Illegal instruction (Is the flash damaged?)", - 1: "SYSCALL instruction", - 2: "InstructionFetchError: Processor internal physical address or data error during " - "instruction fetch", - 3: "LoadStoreError: Processor internal physical address or data error during load or store", - 4: "Level1Interrupt: Level-1 interrupt as indicated by set level-1 bits in the INTERRUPT " - "register", - 5: "Alloca: MOVSP instruction, if caller's registers are not in the register file", - 6: "Integer Divide By Zero", - 7: "reserved", - 8: "Privileged: Attempt to execute a privileged operation when CRING ? 0", - 9: "LoadStoreAlignmentCause: Load or store to an unaligned address", - 10: "reserved", - 11: "reserved", - 12: "InstrPIFDataError: PIF data error during instruction fetch", - 13: "LoadStorePIFDataError: Synchronous PIF data error during LoadStore access", - 14: "InstrPIFAddrError: PIF address error during instruction fetch", - 15: "LoadStorePIFAddrError: Synchronous PIF address error during LoadStore access", - 16: "InstTLBMiss: Error during Instruction TLB refill", - 17: "InstTLBMultiHit: Multiple instruction TLB entries matched", - 18: "InstFetchPrivilege: An instruction fetch referenced a virtual address at a ring level " - "less than CRING", - 19: "reserved", - 20: "InstFetchProhibited: An instruction fetch referenced a page mapped with an attribute " - "that does not permit instruction fetch", - 21: "reserved", - 22: "reserved", - 23: "reserved", - 24: "LoadStoreTLBMiss: Error during TLB refill for a load or store", - 25: "LoadStoreTLBMultiHit: Multiple TLB entries matched for a load or store", - 26: "LoadStorePrivilege: A load or store referenced a virtual address at a ring level less " - "than ", - 27: "reserved", - 28: "Access to invalid address: LOAD (wild pointer?)", - 29: "Access to invalid address: STORE (wild pointer?)", -} - - -def _decode_pc(config, addr): - idedata = get_idedata(config) - if not idedata.addr2line_path or not idedata.firmware_elf_path: - _LOGGER.debug("decode_pc no addr2line") - return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] - try: - translation = subprocess.check_output(command, close_fds=False).decode().strip() - except Exception: # pylint: disable=broad-except - _LOGGER.debug("Caught exception for command %s", command, exc_info=1) - return - - if "?? ??:0" in translation: - # Nothing useful - return - translation = translation.replace(" at ??:?", "").replace(":?", "") - _LOGGER.warning("Decoded %s", translation) - - -def _parse_register(config, regex, line): - match = regex.match(line) - if match is not None: - _decode_pc(config, match.group(1)) - - -STACKTRACE_ESP8266_EXCEPTION_TYPE_RE = re.compile(r"[eE]xception \((\d+)\):") -STACKTRACE_ESP8266_PC_RE = re.compile(r"epc1=0x(4[0-9a-fA-F]{7})") -STACKTRACE_ESP8266_EXCVADDR_RE = re.compile(r"excvaddr=0x(4[0-9a-fA-F]{7})") -STACKTRACE_ESP32_PC_RE = re.compile(r".*PC\s*:\s*(?:0x)?(4[0-9a-fA-F]{7}).*") -STACKTRACE_ESP32_EXCVADDR_RE = re.compile(r"EXCVADDR\s*:\s*(?:0x)?(4[0-9a-fA-F]{7})") -STACKTRACE_ESP32_C3_PC_RE = re.compile(r"MEPC\s*:\s*(?:0x)?(4[0-9a-fA-F]{7})") -STACKTRACE_ESP32_C3_RA_RE = re.compile(r"RA\s*:\s*(?:0x)?(4[0-9a-fA-F]{7})") -STACKTRACE_BAD_ALLOC_RE = re.compile( - r"^last failed alloc call: (4[0-9a-fA-F]{7})\((\d+)\)$" -) -STACKTRACE_ESP32_BACKTRACE_RE = re.compile( - r"Backtrace:(?:\s*0x[0-9a-fA-F]{8}:0x[0-9a-fA-F]{8})+" -) -STACKTRACE_ESP32_BACKTRACE_PC_RE = re.compile(r"4[0-9a-f]{7}") -# ESP32 crash handler (stored backtrace from previous boot) -STACKTRACE_ESP32_CRASH_BT_RE = re.compile(r"BT\d+:\s*0x([0-9a-fA-F]{8})") -STACKTRACE_ESP8266_BACKTRACE_PC_RE = re.compile(r"4[0-9a-f]{7}") - - -def process_stacktrace(config, line, backtrace_state): - line = line.strip() - # ESP8266 Exception type - match = re.match(STACKTRACE_ESP8266_EXCEPTION_TYPE_RE, line) - if match is not None: - code = int(match.group(1)) - _LOGGER.warning( - "Exception type: %s", ESP8266_EXCEPTION_CODES.get(code, "unknown") - ) - - # ESP8266 PC/EXCVADDR - _parse_register(config, STACKTRACE_ESP8266_PC_RE, line) - _parse_register(config, STACKTRACE_ESP8266_EXCVADDR_RE, line) - # ESP32 PC/EXCVADDR - _parse_register(config, STACKTRACE_ESP32_PC_RE, line) - _parse_register(config, STACKTRACE_ESP32_EXCVADDR_RE, line) - # ESP32-C3 PC/RA - _parse_register(config, STACKTRACE_ESP32_C3_PC_RE, line) - _parse_register(config, STACKTRACE_ESP32_C3_RA_RE, line) - - # bad alloc - match = re.match(STACKTRACE_BAD_ALLOC_RE, line) - if match is not None: - _LOGGER.warning( - "Memory allocation of %s bytes failed at %s", match.group(2), match.group(1) - ) - _decode_pc(config, match.group(1)) - - # ESP32 crash handler backtrace (from previous boot) - match = re.search(STACKTRACE_ESP32_CRASH_BT_RE, line) - if match is not None: - _decode_pc(config, match.group(1)) - - # ESP32 single-line backtrace - match = re.match(STACKTRACE_ESP32_BACKTRACE_RE, line) - if match is not None: - _LOGGER.warning("Found stack trace! Trying to decode it") - for addr in re.finditer(STACKTRACE_ESP32_BACKTRACE_PC_RE, line): - _decode_pc(config, addr.group()) - - # ESP8266 multi-line backtrace - if ">>>stack>>>" in line: - # Start of backtrace - backtrace_state = True - _LOGGER.warning("Found stack trace! Trying to decode it") - elif "<< str | None: "https://esphome.io/guides/esp32_arduino_to_idf/\n\n", ) ) + + +@dataclass +class FlashImage: + path: Path + offset: str diff --git a/tests/unit_tests/components/api/test_client.py b/tests/unit_tests/components/api/test_client.py index 3970d1ce8b..333ef70b22 100644 --- a/tests/unit_tests/components/api/test_client.py +++ b/tests/unit_tests/components/api/test_client.py @@ -4,6 +4,7 @@ from __future__ import annotations from unittest.mock import patch +from esphome.components import esp32 from esphome.components.api import client as api_client from esphome.core import EsphomeError @@ -18,11 +19,11 @@ def test_decoder_swallows_esphome_error() -> None: reconnect. """ config = {"esphome": {"name": "test"}} - processor = api_client._LogLineProcessor(config, None) with patch.object( - api_client, "process_stacktrace", side_effect=EsphomeError("no idedata") + esp32, "process_stacktrace", side_effect=EsphomeError("no idedata") ) as mock_process: + processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) processor.process_line("PC: 0x4010496e") assert mock_process.called @@ -47,9 +48,9 @@ def test_decoder_warning_uses_fallback_for_empty_error(caplog) -> None: must show a useful explanation rather than empty parens. """ config = {"esphome": {"name": "test"}} - processor = api_client._LogLineProcessor(config, None) - with patch.object(api_client, "process_stacktrace", side_effect=EsphomeError()): + with patch.object(esp32, "process_stacktrace", side_effect=EsphomeError()): + processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) processor.process_line("PC: 0x4010496e") warnings = [r.message for r in caplog.records if r.levelname == "WARNING"] @@ -65,11 +66,11 @@ def test_decoder_short_circuits_after_failure() -> None: stall log streaming. """ config = {"esphome": {"name": "test"}} - processor = api_client._LogLineProcessor(config, None) with patch.object( - api_client, "process_stacktrace", side_effect=EsphomeError("no idedata") + esp32, "process_stacktrace", side_effect=EsphomeError("no idedata") ) as mock_process: + processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) processor.process_line("PC: 0x4010496e") processor.process_line("BT0: 0x4010496e") processor.process_line("BT1: 0x401049aa") @@ -80,18 +81,18 @@ def test_decoder_short_circuits_after_failure() -> None: def test_decoder_threads_backtrace_state() -> None: """When decoding succeeds, backtrace_state is threaded across calls.""" config = {"esphome": {"name": "test"}} - processor = api_client._LogLineProcessor(config, None) with patch.object( - api_client, "process_stacktrace", side_effect=[True, False] + esp32, "process_stacktrace", side_effect=[True, False] ) as mock_process: + processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) processor.process_line(">>>stack>>>") assert processor.backtrace_state is True processor.process_line("<< None: @@ -105,7 +106,7 @@ def test_decoder_uses_platform_handler_when_provided() -> None: processor = api_client._LogLineProcessor(config, platform_handler) - with patch.object(api_client, "process_stacktrace") as mock_generic: + with patch.object(esp32, "process_stacktrace") as mock_generic: processor.process_line("BT0: 0x4010496e") assert calls == [(config, "BT0: 0x4010496e", False)] diff --git a/tests/unit_tests/components/test_esp_stacktrace.py b/tests/unit_tests/components/test_esp_stacktrace.py new file mode 100644 index 0000000000..5235f313d6 --- /dev/null +++ b/tests/unit_tests/components/test_esp_stacktrace.py @@ -0,0 +1,109 @@ +"""Tests for ESP32 component.""" + +from pathlib import Path +from unittest.mock import Mock + + +def test_process_stacktrace_esp8266_exception(setup_core: Path, caplog) -> None: + """Test process_stacktrace handles ESP8266 exceptions.""" + from esphome.components.esp8266 import process_stacktrace + + config = {"name": "test"} + + # Test exception type parsing + line = "Exception (28):" + backtrace_state = False + + result = process_stacktrace(config, line, backtrace_state) + + assert "Access to invalid address: LOAD (wild pointer?)" in caplog.text + assert result is False + + +def test_process_stacktrace_esp8266_backtrace( + setup_core: Path, mock_esp8266_decode_pc: Mock +) -> None: + """Test process_stacktrace handles ESP8266 multi-line backtrace.""" + from esphome.components.esp8266 import process_stacktrace + + config = {"name": "test"} + + # Start of backtrace + line1 = ">>>stack>>>" + state = process_stacktrace(config, line1, False) + assert state is True + + # Backtrace content with addresses + line2 = "40201234 40205678" + state = process_stacktrace(config, line2, state) + assert state is True + assert mock_esp8266_decode_pc.call_count == 2 + + # End of backtrace + line3 = "<< None: + """Test process_stacktrace handles ESP32 single-line backtrace.""" + from esphome.components.esp32 import process_stacktrace + + config = {"name": "test"} + + line = "Backtrace: 0x40081234:0x3ffb1234 0x40085678:0x3ffb5678" + state = process_stacktrace(config, line, False) + + # Should decode both addresses + assert mock_esp32_decode_pc.call_count == 2 + mock_esp32_decode_pc.assert_any_call(config, "40081234") + mock_esp32_decode_pc.assert_any_call(config, "40085678") + assert state is False + + +def test_process_stacktrace_bad_alloc( + setup_core: Path, mock_esp32_decode_pc: Mock, caplog +) -> None: + """Test process_stacktrace handles bad alloc messages.""" + from esphome.components.esp32 import process_stacktrace + + config = {"name": "test"} + + line = "last failed alloc call: 40201234(512)" + state = process_stacktrace(config, line, False) + + assert "Memory allocation of 512 bytes failed at 40201234" in caplog.text + mock_esp32_decode_pc.assert_called_once_with(config, "40201234") + assert state is False + + +def test_process_stacktrace_esp32_crash_handler( + setup_core: Path, mock_esp32_decode_pc: Mock +) -> None: + """Test process_stacktrace handles ESP32 crash handler backtrace lines.""" + from esphome.components.esp32 import process_stacktrace + + config = {"name": "test"} + + # Simulate crash handler log lines as they appear from the API/serial + line_pc = "[E][esp32.crash:078]: PC: 0x400D1234 (fault location)" + state = process_stacktrace(config, line_pc, False) + # PC line is matched by existing STACKTRACE_ESP32_PC_RE + mock_esp32_decode_pc.assert_called_with(config, "400D1234") + assert state is False + + mock_esp32_decode_pc.reset_mock() + + line_bt0 = "[E][esp32.crash:080]: BT0: 0x400D5678 (backtrace)" + state = process_stacktrace(config, line_bt0, False) + mock_esp32_decode_pc.assert_called_once_with(config, "400D5678") + assert state is False + + mock_esp32_decode_pc.reset_mock() + + line_bt1 = "[E][esp32.crash:080]: BT1: 0x42005ABC (backtrace)" + state = process_stacktrace(config, line_bt1, False) + mock_esp32_decode_pc.assert_called_once_with(config, "42005ABC") + assert state is False diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index dfd4305c4d..626f4168a6 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -77,9 +77,16 @@ def mock_run_platformio_cli_run() -> Generator[Mock, None, None]: @pytest.fixture -def mock_decode_pc() -> Generator[Mock, None, None]: - """Mock _decode_pc for platformio_api.""" - with patch("esphome.platformio_api._decode_pc") as mock: +def mock_esp32_decode_pc() -> Generator[Mock, None, None]: + """Mock _decode_pc for esp32.""" + with patch("esphome.components.esp32._decode_pc") as mock: + yield mock + + +@pytest.fixture +def mock_esp8266_decode_pc() -> Generator[Mock, None, None]: + """Mock _decode_pc for esp8266.""" + with patch("esphome.components.esp8266._decode_pc") as mock: yield mock diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 798a43a4ce..f8a3ea888e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -54,6 +54,7 @@ from esphome.__main__ import ( ) from esphome.address_cache import AddressCache from esphome.bundle import BUNDLE_EXTENSION, BundleFile, BundleResult +from esphome.components import esp32 from esphome.components.esp32 import KEY_ESP32, KEY_VARIANT, VARIANT_ESP32 from esphome.const import ( CONF_API, @@ -85,7 +86,7 @@ from esphome.const import ( ) from esphome.core import CORE, EsphomeError from esphome.espota2 import OTA_TYPE_UPDATE_APP, OTA_TYPE_UPDATE_PARTITION_TABLE -from esphome.util import BootselResult +from esphome.util import BootselResult, FlashImage from esphome.zeroconf import _await_discovery, discover_mdns_devices @@ -1181,8 +1182,8 @@ def test_upload_using_esptool_path_conversion( mock_idedata = MagicMock(spec=platformio_api.IDEData) mock_idedata.firmware_bin_path = tmp_path / "firmware.bin" mock_idedata.extra_flash_images = [ - platformio_api.FlashImage(path=tmp_path / "bootloader.bin", offset="0x1000"), - platformio_api.FlashImage(path=tmp_path / "partitions.bin", offset="0x8000"), + FlashImage(path=tmp_path / "bootloader.bin", offset="0x1000"), + FlashImage(path=tmp_path / "partitions.bin", offset="0x8000"), ] mock_get_idedata.return_value = mock_idedata @@ -1259,8 +1260,8 @@ def test_upload_using_esptool_skips_missing_extra_flash_images( mock_idedata = MagicMock(spec=platformio_api.IDEData) mock_idedata.firmware_bin_path = tmp_path / "firmware.bin" mock_idedata.extra_flash_images = [ - platformio_api.FlashImage(path=tmp_path / "bootloader.bin", offset="0x1000"), - platformio_api.FlashImage(path=missing_path, offset="0x2d0000"), + FlashImage(path=tmp_path / "bootloader.bin", offset="0x1000"), + FlashImage(path=missing_path, offset="0x2d0000"), ] mock_get_idedata.return_value = mock_idedata @@ -4225,7 +4226,7 @@ def test_run_miniterm_batches_lines_with_same_timestamp( with ( patch("serial.Serial", return_value=mock_serial), - patch.object(platformio_api, "process_stacktrace") as mock_bt, + patch.object(esp32, "process_stacktrace") as mock_bt, ): mock_bt.return_value = False result = run_miniterm(config, "/dev/ttyUSB0", args) @@ -4264,7 +4265,7 @@ def test_run_miniterm_different_chunks_different_timestamps( with ( patch("serial.Serial", return_value=mock_serial), - patch.object(platformio_api, "process_stacktrace") as mock_bt, + patch.object(esp32, "process_stacktrace") as mock_bt, ): mock_bt.return_value = False result = run_miniterm(config, "/dev/ttyUSB0", args) @@ -4295,7 +4296,7 @@ def test_run_miniterm_handles_split_lines() -> None: with ( patch("serial.Serial", return_value=mock_serial), - patch.object(platformio_api, "process_stacktrace") as mock_bt, + patch.object(esp32, "process_stacktrace") as mock_bt, patch("esphome.__main__.safe_print") as mock_print, ): mock_bt.return_value = False @@ -4349,7 +4350,7 @@ def test_run_miniterm_backtrace_state_maintained() -> None: with ( patch("serial.Serial", return_value=mock_serial), patch.object( - platformio_api, + esp32, "process_stacktrace", side_effect=track_backtrace_state, ), @@ -4400,7 +4401,7 @@ def test_run_miniterm_handles_empty_reads( with ( patch("serial.Serial", return_value=mock_serial), - patch.object(platformio_api, "process_stacktrace") as mock_bt, + patch.object(esp32, "process_stacktrace") as mock_bt, ): mock_bt.return_value = False result = run_miniterm(config, "/dev/ttyUSB0", args) @@ -4473,7 +4474,7 @@ def test_run_miniterm_buffer_limit_prevents_unbounded_growth() -> None: with ( patch("serial.Serial", return_value=mock_serial), - patch.object(platformio_api, "process_stacktrace") as mock_bt, + patch.object(esp32, "process_stacktrace") as mock_bt, patch("esphome.__main__.safe_print") as mock_print, patch("esphome.__main__.SERIAL_BUFFER_MAX_SIZE", test_buffer_limit), ): diff --git a/tests/unit_tests/test_platformio_api.py b/tests/unit_tests/test_platformio_api.py index b241622f89..7a88ec4d9e 100644 --- a/tests/unit_tests/test_platformio_api.py +++ b/tests/unit_tests/test_platformio_api.py @@ -13,6 +13,7 @@ import pytest from esphome import platformio_api, platformio_runner from esphome.core import CORE, EsphomeError +from esphome.util import FlashImage def test_idedata_firmware_elf_path(setup_core: Path) -> None: @@ -70,7 +71,7 @@ def test_idedata_extra_flash_images(setup_core: Path) -> None: images = idedata.extra_flash_images assert len(images) == 2 - assert all(isinstance(img, platformio_api.FlashImage) for img in images) + assert all(isinstance(img, FlashImage) for img in images) assert images[0].path == Path("/path/to/bootloader.bin") assert images[0].offset == "0x1000" assert images[1].path == Path("/path/to/partition.bin") @@ -106,7 +107,7 @@ def test_idedata_cc_path(setup_core: Path) -> None: def test_flash_image_dataclass() -> None: """Test FlashImage dataclass stores path and offset correctly.""" - image = platformio_api.FlashImage(path=Path("/path/to/image.bin"), offset="0x10000") + image = FlashImage(path=Path("/path/to/image.bin"), offset="0x10000") assert image.path == Path("/path/to/image.bin") assert image.offset == "0x10000" @@ -708,101 +709,6 @@ def test_patched_clean_build_dir_creates_missing(setup_core: Path) -> None: assert build_dir.exists() -def test_process_stacktrace_esp8266_exception(setup_core: Path, caplog) -> None: - """Test process_stacktrace handles ESP8266 exceptions.""" - config = {"name": "test"} - - # Test exception type parsing - line = "Exception (28):" - backtrace_state = False - - result = platformio_api.process_stacktrace(config, line, backtrace_state) - - assert "Access to invalid address: LOAD (wild pointer?)" in caplog.text - assert result is False - - -def test_process_stacktrace_esp8266_backtrace( - setup_core: Path, mock_decode_pc: Mock -) -> None: - """Test process_stacktrace handles ESP8266 multi-line backtrace.""" - config = {"name": "test"} - - # Start of backtrace - line1 = ">>>stack>>>" - state = platformio_api.process_stacktrace(config, line1, False) - assert state is True - - # Backtrace content with addresses - line2 = "40201234 40205678" - state = platformio_api.process_stacktrace(config, line2, state) - assert state is True - assert mock_decode_pc.call_count == 2 - - # End of backtrace - line3 = "<< None: - """Test process_stacktrace handles ESP32 single-line backtrace.""" - config = {"name": "test"} - - line = "Backtrace: 0x40081234:0x3ffb1234 0x40085678:0x3ffb5678" - state = platformio_api.process_stacktrace(config, line, False) - - # Should decode both addresses - assert mock_decode_pc.call_count == 2 - mock_decode_pc.assert_any_call(config, "40081234") - mock_decode_pc.assert_any_call(config, "40085678") - assert state is False - - -def test_process_stacktrace_bad_alloc( - setup_core: Path, mock_decode_pc: Mock, caplog -) -> None: - """Test process_stacktrace handles bad alloc messages.""" - config = {"name": "test"} - - line = "last failed alloc call: 40201234(512)" - state = platformio_api.process_stacktrace(config, line, False) - - assert "Memory allocation of 512 bytes failed at 40201234" in caplog.text - mock_decode_pc.assert_called_once_with(config, "40201234") - assert state is False - - -def test_process_stacktrace_esp32_crash_handler( - setup_core: Path, mock_decode_pc: Mock -) -> None: - """Test process_stacktrace handles ESP32 crash handler backtrace lines.""" - config = {"name": "test"} - - # Simulate crash handler log lines as they appear from the API/serial - line_pc = "[E][esp32.crash:078]: PC: 0x400D1234 (fault location)" - state = platformio_api.process_stacktrace(config, line_pc, False) - # PC line is matched by existing STACKTRACE_ESP32_PC_RE - mock_decode_pc.assert_called_with(config, "400D1234") - assert state is False - - mock_decode_pc.reset_mock() - - line_bt0 = "[E][esp32.crash:080]: BT0: 0x400D5678 (backtrace)" - state = platformio_api.process_stacktrace(config, line_bt0, False) - mock_decode_pc.assert_called_once_with(config, "400D5678") - assert state is False - - mock_decode_pc.reset_mock() - - line_bt1 = "[E][esp32.crash:080]: BT1: 0x42005ABC (backtrace)" - state = platformio_api.process_stacktrace(config, line_bt1, False) - mock_decode_pc.assert_called_once_with(config, "42005ABC") - assert state is False - - def test_patch_file_downloader_succeeds_first_try() -> None: """Test patch_file_downloader succeeds on first attempt.""" mock_exception_cls = type("PackageException", (Exception,), {}) From 7c2a63bf82d7d5c34c475ba4413cc74e2876bae3 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 5 May 2026 08:12:20 +1200 Subject: [PATCH 02/60] [api] Use safe_print for log output and fix safe_print bytes-repr fallback (#16160) --- esphome/components/api/client.py | 8 ++- esphome/util.py | 26 +++++-- tests/unit_tests/test_util.py | 116 +++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index 0a5370cb30..d6150fbd29 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -19,6 +19,7 @@ import contextlib from esphome.const import CONF_KEY, CONF_PORT, __version__ from esphome.core import CORE, EsphomeError +from esphome.util import safe_print from . import CONF_ENCRYPTION @@ -101,7 +102,6 @@ async def async_run_logs( noise_psk=noise_psk, addresses=addresses, # Pass all addresses for automatic retry ) - dashboard = CORE.dashboard # Try platform-specific stacktrace handler first, fall back to generic platform_process_stacktrace = None @@ -126,7 +126,11 @@ async def async_run_logs( f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}.{nanoseconds:03}]" ) for parsed_msg in parse_log_message(text, timestamp): - print(parsed_msg.replace("\033", "\\033") if dashboard else parsed_msg) + # safe_print handles the dashboard \033 escaping and falls back + # to backslashreplace encoding on stdouts that can't represent + # the wifi signal-bar block characters (Windows redirected + # cp1252 pipe). + safe_print(parsed_msg) for raw_line in text.splitlines(): processor.process_line(raw_line) diff --git a/esphome/util.py b/esphome/util.py index 9d6e995f1f..39ce7c0963 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -94,13 +94,29 @@ def safe_print(message="", end="\n"): except UnicodeEncodeError: pass + # Fall back to the stream's actual encoding (e.g. cp1252 on Windows + # redirected pipes). Use "backslashreplace" so unencodable code points + # like the wifi signal-bar block characters (U+2582..U+2588) become + # readable ``\uXXXX`` escapes, and decode back to ``str`` so ``print`` + # never receives a ``bytes`` object (which would render as a ``b'...'`` + # repr). + encoding = sys.stdout.encoding or "ascii" try: - print(message.encode("utf-8", "backslashreplace"), end=end) + print( + message.encode(encoding, "backslashreplace").decode(encoding), + end=end, + ) + return except UnicodeEncodeError: - try: - print(message.encode("ascii", "backslashreplace"), end=end) - except UnicodeEncodeError: - print("Cannot print line because of invalid locale!") + pass + + try: + print( + message.encode("ascii", "backslashreplace").decode("ascii"), + end=end, + ) + except UnicodeEncodeError: + print("Cannot print line because of invalid locale!") def safe_input(prompt=""): diff --git a/tests/unit_tests/test_util.py b/tests/unit_tests/test_util.py index ff58fb1394..581b1aca99 100644 --- a/tests/unit_tests/test_util.py +++ b/tests/unit_tests/test_util.py @@ -709,3 +709,119 @@ def test_detect_rp2040_bootsel_timeout() -> None: result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 0 assert result.permission_error is False + + +class TestSafePrint: + """Tests for ``safe_print`` and its UnicodeEncodeError fallback chain.""" + + @pytest.fixture(autouse=True) + def _no_dashboard(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Default ``CORE.dashboard`` to False so each test starts hermetic.""" + from esphome.core import CORE + + monkeypatch.setattr(CORE, "dashboard", False) + + def test_prints_plain_message(self, capsys: pytest.CaptureFixture[str]) -> None: + """ASCII-only messages take the fast path through native ``print``.""" + util.safe_print("hello world") + assert capsys.readouterr().out == "hello world\n" + + def test_prints_unicode_on_utf8_stdout( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + """Non-ASCII goes straight through when stdout can encode it.""" + util.safe_print("bars: \u2582\u2584\u2586\u2588") + assert capsys.readouterr().out == "bars: \u2582\u2584\u2586\u2588\n" + + def test_dashboard_escapes_esc_byte( + self, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + r"""Dashboard mode escapes raw ``\033`` ESC bytes to literal ``\\033``.""" + from esphome.core import CORE + + monkeypatch.setattr(CORE, "dashboard", True) + util.safe_print("\033[0;32mhi\033[0m") + assert capsys.readouterr().out == "\\033[0;32mhi\\033[0m\n" + + def test_fallback_writes_string_not_bytes_repr( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Regression: cp1252 fallback must produce a printable str, not ``b'...'``. + + On Windows, when stdout is a redirected pipe (e.g. the dashboard), + Python uses cp1252, which cannot encode the wifi signal-bar block + characters (U+2582..U+2588). The previous fallback path called + ``print(message.encode(...))`` with a ``bytes`` object, which + Python's ``print`` rendered as a literal ``b'...'`` repr — visible + in the user's dashboard output. The fix re-encodes through the + stream's encoding with ``backslashreplace`` and decodes back to + ``str``. + """ + buf = io.BytesIO() + cp1252_stream = io.TextIOWrapper(buf, encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", cp1252_stream) + + util.safe_print("bars: \u2582\u2584\u2586\u2588 done") + cp1252_stream.flush() + output = buf.getvalue().decode("cp1252") + + # Output is a clean line, not the bytes repr. + assert not output.startswith("b'") + assert "b'bars" not in output + # Unencodable codepoints become readable backslash escapes. + assert "\\u2582\\u2584\\u2586\\u2588" in output + # Encodable parts survive unchanged. + assert "bars: " in output + assert " done" in output + assert output.endswith("\n") + + def test_fallback_with_dashboard_escaped_message( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Dashboard ESC escaping + cp1252 fallback compose correctly.""" + from esphome.core import CORE + + monkeypatch.setattr(CORE, "dashboard", True) + buf = io.BytesIO() + cp1252_stream = io.TextIOWrapper(buf, encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", cp1252_stream) + + util.safe_print("\033[0;32m\u2582\u2584\u2586\u2588\033[0m") + cp1252_stream.flush() + output = buf.getvalue().decode("cp1252") + + # Dashboard escaping turned ESC into literal "\033" (5 chars), which + # cp1252 can encode, so it survives the round-trip verbatim. + assert "\\033[0;32m" in output + assert "\\033[0m" in output + # Block characters became backslash escapes via backslashreplace. + assert "\\u2582\\u2584\\u2586\\u2588" in output + + def test_final_message_when_locale_is_invalid( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + """If every encoding path fails, surface the locale-error sentinel.""" + original_print = print + call_count = 0 + + def fake_print(*args: Any, **kwargs: Any) -> None: + nonlocal call_count + call_count += 1 + # The first three calls are: native print, stream-encoding + # fallback, ASCII fallback. Make all three raise so we reach + # the final sentinel "Cannot print line..." which is expected + # to succeed (no encoding required). + if call_count <= 3: + raise UnicodeEncodeError("ascii", "x", 0, 1, "boom") + original_print(*args, **kwargs) + + monkeypatch.setattr("builtins.print", fake_print) + util.safe_print("x") + assert call_count == 4 + assert ( + capsys.readouterr().out == "Cannot print line because of invalid locale!\n" + ) From a460f5343cf8a350d34e25f159a1b54295a85a65 Mon Sep 17 00:00:00 2001 From: Brandon Harvey <8107750+bharvey88@users.noreply.github.com> Date: Tue, 28 Apr 2026 05:27:20 -0500 Subject: [PATCH 03/60] [automation] Fix codegen type for component.resume update_interval (#16069) Co-authored-by: Claude Opus 4.7 (1M context) --- esphome/automation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/automation.py b/esphome/automation.py index b4dcc41995..bfbfd58d8a 100644 --- a/esphome/automation.py +++ b/esphome/automation.py @@ -598,7 +598,7 @@ async def component_resume_action_to_code( comp = await cg.get_variable(config[CONF_ID]) var = cg.new_Pvariable(action_id, template_arg, comp) if CONF_UPDATE_INTERVAL in config: - template_ = await cg.templatable(config[CONF_UPDATE_INTERVAL], args, int) + template_ = await cg.templatable(config[CONF_UPDATE_INTERVAL], args, cg.uint32) cg.add(var.set_update_interval(template_)) return var From ce466c6b60fa086e323c8b7b65b2ffccb4a5070d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 29 Apr 2026 18:14:07 -0500 Subject: [PATCH 04/60] [mcp23xxx_base] Reject unsupported interrupt_pin options (inverted, allow_other_uses) (#16149) --- esphome/components/mcp23xxx_base/__init__.py | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/esphome/components/mcp23xxx_base/__init__.py b/esphome/components/mcp23xxx_base/__init__.py index cd952099c0..76a3aabe3f 100644 --- a/esphome/components/mcp23xxx_base/__init__.py +++ b/esphome/components/mcp23xxx_base/__init__.py @@ -2,6 +2,7 @@ from esphome import pins import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import ( + CONF_ALLOW_OTHER_USES, CONF_ID, CONF_INPUT, CONF_INTERRUPT, @@ -30,10 +31,29 @@ MCP23XXX_INTERRUPT_MODES = { "FALLING": MCP23XXXInterruptMode.MCP23XXX_FALLING, } + +def _validate_interrupt_pin(value): + # The MCP component owns INT polarity (active-low, hardcoded falling-edge ISR) + # and installs a single ISR per GPIO, so neither inversion nor sharing is supported. + value = pins.internal_gpio_input_pin_schema(value) + if value.get(CONF_INVERTED): + raise cv.Invalid( + f"'{CONF_INVERTED}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " + "the MCP23xxx INT line is fixed active-low" + ) + if value.get(CONF_ALLOW_OTHER_USES): + raise cv.Invalid( + f"'{CONF_ALLOW_OTHER_USES}: true' is not supported on '{CONF_INTERRUPT_PIN}'; " + "sharing the interrupt pin between multiple MCP23xxx (or other components) " + "is not implemented. Remove the interrupt_pin to fall back to polling." + ) + return value + + MCP23XXX_CONFIG_SCHEMA = cv.Schema( { cv.Optional(CONF_OPEN_DRAIN_INTERRUPT, default=False): cv.boolean, - cv.Optional(CONF_INTERRUPT_PIN): pins.internal_gpio_input_pin_schema, + cv.Optional(CONF_INTERRUPT_PIN): _validate_interrupt_pin, } ).extend(cv.COMPONENT_SCHEMA) From 9371ec319a507bd0dfb185fbe65181370711e7b4 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:04:52 +1200 Subject: [PATCH 05/60] [core] Strip \\?\ prefix from sys.executable for PlatformIO subprocess (#16158) --- esphome/platformio_api.py | 44 ++++++++++- tests/unit_tests/test_platformio_api.py | 99 +++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index fc21977fdd..544bca3b94 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -53,6 +53,37 @@ FILTER_PLATFORMIO_LINES = [ ] +def _strip_win_long_path_prefix(path: str) -> str: + r"""Strip the Windows extended-length path prefix from ``path``. + + Handles both forms documented at + https://learn.microsoft.com/windows/win32/fileio/naming-a-file: + + * ``\\?\C:\path\to\file`` -> ``C:\path\to\file`` + * ``\\?\UNC\server\share\path`` -> ``\\server\share\path`` + + The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with + ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates + into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from + the environment, falling back to ``os.path.normpath(sys.executable)``) + and ends up baked into SCons-emitted command lines for build steps such + as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand + the ``\\?\`` prefix, so the build fails with + "The system cannot find the path specified." Stripping the prefix early + keeps the path shell-quotable. + + No-op on non-Windows platforms. + """ + if sys.platform != "win32": + return path + if path.startswith("\\\\?\\UNC\\"): + # \\?\UNC\server\share\... -> \\server\share\... + return "\\\\" + path[len("\\\\?\\UNC\\") :] + if path.startswith("\\\\?\\"): + return path[len("\\\\?\\") :] + return path + + def run_platformio_cli(*args, **kwargs) -> str | int: os.environ["PLATFORMIO_FORCE_COLOR"] = "true" os.environ["PLATFORMIO_BUILD_DIR"] = str(CORE.relative_pioenvs_path().absolute()) @@ -63,7 +94,18 @@ def run_platformio_cli(*args, **kwargs) -> str | int: os.environ.setdefault("PYTHONWARNINGS", "ignore::SyntaxWarning") # Increase uv retry count to handle transient network errors (default is 3) os.environ.setdefault("UV_HTTP_RETRIES", "10") - cmd = [sys.executable, "-m", "esphome.platformio_runner"] + list(args) + # Strip the Windows extended-length path prefix from sys.executable so it + # doesn't propagate into PlatformIO's $PYTHONEXE and break SCons-emitted + # command lines run through cmd.exe. + python_exe = _strip_win_long_path_prefix(sys.executable) + if python_exe != sys.executable: + # Only override PYTHONEXEPATH when we actually stripped a prefix. + # PlatformIO's get_pythonexe_path() reads this and falls back to + # sys.executable otherwise; setting it unconditionally would clobber + # a user-provided value (or the unmodified path on platforms that + # don't need the strip). + os.environ["PYTHONEXEPATH"] = python_exe + cmd = [python_exe, "-m", "esphome.platformio_runner"] + list(args) return run_external_process(*cmd, **kwargs) diff --git a/tests/unit_tests/test_platformio_api.py b/tests/unit_tests/test_platformio_api.py index ddc4e45c84..a92e00167d 100644 --- a/tests/unit_tests/test_platformio_api.py +++ b/tests/unit_tests/test_platformio_api.py @@ -311,6 +311,105 @@ def test_run_platformio_cli_sets_environment_variables( assert "arg" in args +@pytest.mark.parametrize( + ("platform", "input_path", "expected"), + [ + # win32: drive-letter extended-length prefix is stripped + ( + "win32", + "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + ), + # win32: UNC extended-length prefix is translated to a regular UNC path + ( + "win32", + "\\\\?\\UNC\\server\\share\\python.exe", + "\\\\server\\share\\python.exe", + ), + # win32: paths without the prefix are returned unchanged + ( + "win32", + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe", + ), + # non-win32: prefix is left alone (no-op) + ("linux", "\\\\?\\C:\\python.exe", "\\\\?\\C:\\python.exe"), + ("darwin", "/usr/bin/python3", "/usr/bin/python3"), + ], +) +def test_strip_win_long_path_prefix( + platform: str, input_path: str, expected: str +) -> None: + r"""``\\?\`` and ``\\?\UNC\`` prefixes are stripped only on win32.""" + with patch("esphome.platformio_api.sys.platform", platform): + assert platformio_api._strip_win_long_path_prefix(input_path) == expected + + +def test_run_platformio_cli_strips_win_long_path_prefix( + setup_core: Path, mock_run_external_process: Mock +) -> None: + r"""Windows ``\\?\`` prefix on sys.executable does not leak into the subprocess. + + The NSIS-installed esphome.exe launcher starts Python with + ``sys.executable`` already prefixed by the extended-length path marker. + That prefix would otherwise propagate into PlatformIO's ``PYTHONEXE`` and + break SCons-emitted command lines run through ``cmd.exe``. + """ + CORE.build_path = str(setup_core / "build" / "test") + prefixed_exe = ( + "\\\\?\\C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe" + ) + stripped_exe = ( + "C:\\Users\\jesse\\AppData\\Local\\ESPHome Builder\\python\\python.exe" + ) + + with ( + patch.dict(os.environ, {}, clear=False), + patch("esphome.platformio_api.sys.platform", "win32"), + patch("esphome.platformio_api.sys.executable", prefixed_exe), + ): + # Pop any pre-existing PYTHONEXEPATH so the assertion below reflects + # what run_platformio_cli set, not whatever the test runner's + # environment happened to contain. + os.environ.pop("PYTHONEXEPATH", None) + mock_run_external_process.return_value = 0 + platformio_api.run_platformio_cli("test", "arg") + + # The subprocess is invoked with the stripped executable path. + mock_run_external_process.assert_called_once() + args = mock_run_external_process.call_args[0] + assert args[0] == stripped_exe + # PYTHONEXEPATH is exported with the stripped path so PlatformIO's + # get_pythonexe_path() picks it up in the subprocess. + assert os.environ["PYTHONEXEPATH"] == stripped_exe + + +def test_run_platformio_cli_does_not_set_pythonexepath_without_strip( + setup_core: Path, mock_run_external_process: Mock +) -> None: + r"""PYTHONEXEPATH is not touched when sys.executable has no ``\\?\`` prefix. + + Setting it unconditionally would clobber a user-provided value (or + interfere with non-Windows tooling that has no prefix to strip). + """ + CORE.build_path = str(setup_core / "build" / "test") + plain_exe = "/usr/bin/python3" + + with ( + patch.dict(os.environ, {}, clear=False), + patch("esphome.platformio_api.sys.platform", "linux"), + patch("esphome.platformio_api.sys.executable", plain_exe), + ): + os.environ.pop("PYTHONEXEPATH", None) + mock_run_external_process.return_value = 0 + platformio_api.run_platformio_cli("test", "arg") + + mock_run_external_process.assert_called_once() + args = mock_run_external_process.call_args[0] + assert args[0] == plain_exe + assert "PYTHONEXEPATH" not in os.environ + + def test_run_platformio_cli_run_builds_command( setup_core: Path, mock_run_platformio_cli: Mock ) -> None: From 60a94fd10972681f28772716103d0ca46177104d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 30 Apr 2026 12:27:54 -0500 Subject: [PATCH 06/60] [esp32] Replace 512B stack buffer in printf wraps with picolibc cookie FILE (#16170) --- esphome/components/esp32/printf_stubs.cpp | 77 +++++++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/esphome/components/esp32/printf_stubs.cpp b/esphome/components/esp32/printf_stubs.cpp index 386fbbd79d..908b4023ea 100644 --- a/esphome/components/esp32/printf_stubs.cpp +++ b/esphome/components/esp32/printf_stubs.cpp @@ -13,14 +13,21 @@ * and printf() calls in SDK components are only in debug/assert paths * (gpio_dump_io_configuration, ringbuf diagnostics) that are either * GC'd or never called. Crash backtraces and panic output are - * unaffected — they use esp_rom_printf() which is a ROM function + * unaffected; they use esp_rom_printf() which is a ROM function * and does not go through libc. * - * These stubs redirect through vsnprintf() (which uses _svfprintf_r - * already in the binary) and fwrite(), allowing the linker to - * dead-code eliminate _vfprintf_r. + * On picolibc (default for IDF >= 5 on RISC-V, IDF >= 6 everywhere) we + * route output through a stack-allocated cookie FILE that forwards each + * byte to the real target stream via fputc(). Picolibc's tinystdio + * vfprintf walks the FILE::put callback one character at a time, so this + * costs ~32 bytes of stack for the cookie struct vs. a 512-byte format + * buffer. The buffered path overflows the loopTask stack on IDF 6. * - * Saves ~11 KB of flash. + * On newlib (IDF <= 5 on Xtensa) we keep the original snprintf-then-fwrite + * path because that loopTask stack budget has plenty of headroom for the + * 512-byte buffer; the picolibc-only crash above does not affect it. + * + * Saves ~11 KB of flash on newlib, ~2.8 KB on picolibc. * * To disable these wraps, set enable_full_printf: true in the esp32 * advanced config section. @@ -30,10 +37,55 @@ #include #include +#ifndef __PICOLIBC__ #include "esp_system.h" +#endif namespace esphome::esp32 {} +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +extern "C" { + +#ifdef __PICOLIBC__ + +#include +#include + +extern int __real_vfprintf(FILE *stream, const char *fmt, va_list ap); + +namespace { + +struct CookieFile { + FILE base; + FILE *target; +}; + +// cookie_put() recovers CookieFile* from FILE* via reinterpret_cast, which is +// only well-defined when FILE is the first member at offset 0 and CookieFile +// is standard-layout. +static_assert(offsetof(CookieFile, base) == 0, "FILE must be the first member of CookieFile"); +static_assert(std::is_standard_layout::value, "CookieFile must be standard-layout"); + +int cookie_put(char c, FILE *stream) { + auto *cookie = reinterpret_cast(stream); + return fputc(static_cast(c), cookie->target); +} + +const FILE COOKIE_FILE_TEMPLATE = FDEV_SETUP_STREAM(cookie_put, nullptr, nullptr, _FDEV_SETUP_WRITE); + +} // namespace + +int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { + CookieFile cookie; + cookie.base = COOKIE_FILE_TEMPLATE; + cookie.target = stream; + return __real_vfprintf(&cookie.base, fmt, ap); +} + +int __wrap_vprintf(const char *fmt, va_list ap) { return __wrap_vfprintf(stdout, fmt, ap); } + +#else // !__PICOLIBC__ + static constexpr size_t PRINTF_BUFFER_SIZE = 512; // These stubs are essentially dead code at runtime — ESPHome replaces the @@ -55,14 +107,18 @@ static int write_printf_buffer(FILE *stream, char *buf, int len) { return len; } -// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) -extern "C" { - int __wrap_vprintf(const char *fmt, va_list ap) { char buf[PRINTF_BUFFER_SIZE]; return write_printf_buffer(stdout, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); } +int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { + char buf[PRINTF_BUFFER_SIZE]; + return write_printf_buffer(stream, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); +} + +#endif // __PICOLIBC__ + int __wrap_printf(const char *fmt, ...) { va_list ap; va_start(ap, fmt); @@ -71,11 +127,6 @@ int __wrap_printf(const char *fmt, ...) { return len; } -int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { - char buf[PRINTF_BUFFER_SIZE]; - return write_printf_buffer(stream, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); -} - int __wrap_fprintf(FILE *stream, const char *fmt, ...) { va_list ap; va_start(ap, fmt); From d9c22d6b56d80adbd9cf767244dc98b0dc4af4de Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 1 May 2026 12:23:14 +1000 Subject: [PATCH 07/60] [lvgl] Clamp values for meter line indicators (#16180) --- esphome/components/lvgl/lvgl_esphome.cpp | 6 ++++-- esphome/components/lvgl/lvgl_esphome.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index d8248e4aa4..722a7a1b02 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -454,10 +454,12 @@ void LVTouchListener::update(const touchscreen::TouchPoints_t &tpoints) { #ifdef USE_LVGL_METER -int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int value) { +int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int32_t value) { auto *scale = lv_obj_get_parent(obj); auto min_value = lv_scale_get_range_min_value(scale); - return ((value - min_value) * lv_scale_get_angle_range(scale) / (lv_scale_get_range_max_value(scale) - min_value) + + auto max_value = lv_scale_get_range_max_value(scale); + value = clamp(value, min_value, max_value); + return ((value - min_value) * lv_scale_get_angle_range(scale) / (max_value - min_value) + lv_scale_get_rotation((scale))) % 360; } diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index 146866f5bd..8c0b10e1bc 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -112,7 +112,7 @@ inline void lv_animimg_set_src(lv_obj_t *img, std::vector images #endif // USE_LVGL_ANIMIMG #ifdef USE_LVGL_METER -int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int value); +int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int32_t value); #endif // Parent class for things that wrap an LVGL object From 0418f2138af5ba4057708e91d06dd015a0d9d4d8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 1 May 2026 14:31:56 -0500 Subject: [PATCH 08/60] [esp32] Drop printf wrap on IDF 6.0+ (picolibc no longer needs it) (#16189) --- esphome/components/esp32/__init__.py | 12 +++- esphome/components/esp32/printf_stubs.cpp | 85 ++++++----------------- 2 files changed, 31 insertions(+), 66 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 77b405a449..30acbc3e41 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1740,7 +1740,17 @@ async def to_code(config): # Wrap FILE*-based printf functions to eliminate newlib's _vfprintf_r # (~11 KB). See printf_stubs.cpp for implementation. - if conf[CONF_ADVANCED][CONF_ENABLE_FULL_PRINTF]: + # + # The wrap is only beneficial against newlib. Picolibc's tinystdio + # implements vsnprintf by building a string-output FILE and calling + # vfprintf, so vfprintf is unconditionally linked in by any caller + # of snprintf/vsnprintf — effectively every build — and the wrap + # saves nothing while costing ~170 B of shim. IDF 5.x defaults to + # newlib on every variant; IDF 6.0+ switches to picolibc on every + # variant. + if conf[CONF_ADVANCED][CONF_ENABLE_FULL_PRINTF] or idf_version() >= cv.Version( + 6, 0, 0 + ): cg.add_define("USE_FULL_PRINTF") else: for symbol in ("vprintf", "printf", "fprintf", "vfprintf"): diff --git a/esphome/components/esp32/printf_stubs.cpp b/esphome/components/esp32/printf_stubs.cpp index 908b4023ea..489c503942 100644 --- a/esphome/components/esp32/printf_stubs.cpp +++ b/esphome/components/esp32/printf_stubs.cpp @@ -1,91 +1,48 @@ /* - * Linker wrap stubs for FILE*-based printf functions. + * Linker wrap stubs for FILE*-based printf functions (newlib only). * * ESP-IDF SDK components (gpio driver, ringbuf, log_write) reference - * fprintf(), printf(), vprintf(), and vfprintf() which pull in the full - * printf implementation (~11 KB on newlib's _vfprintf_r, ~2.8 KB on - * picolibc's vfprintf). This is a separate implementation from the one - * used by snprintf/vsnprintf that handles FILE* stream I/O with buffering - * and locking. + * fprintf(), printf(), vprintf(), and vfprintf(), which on newlib pull + * in _vfprintf_r (~11 KB) — a separate implementation from the one used + * by snprintf/vsnprintf that handles FILE* stream I/O with buffering. * * ESPHome replaces the ESP-IDF log handler via esp_log_set_vprintf_(), * so the SDK's vprintf() path is dead code at runtime. The fprintf() * and printf() calls in SDK components are only in debug/assert paths * (gpio_dump_io_configuration, ringbuf diagnostics) that are either * GC'd or never called. Crash backtraces and panic output are - * unaffected; they use esp_rom_printf() which is a ROM function - * and does not go through libc. + * unaffected; they use esp_rom_printf() which is a ROM function and + * does not go through libc. * - * On picolibc (default for IDF >= 5 on RISC-V, IDF >= 6 everywhere) we - * route output through a stack-allocated cookie FILE that forwards each - * byte to the real target stream via fputc(). Picolibc's tinystdio - * vfprintf walks the FILE::put callback one character at a time, so this - * costs ~32 bytes of stack for the cookie struct vs. a 512-byte format - * buffer. The buffered path overflows the loopTask stack on IDF 6. + * This wrap is newlib-only. On picolibc, vsnprintf is implemented as + * vfprintf into a string-output FILE, so vfprintf is unconditionally + * linked in by any caller of snprintf/vsnprintf and the wrap can never + * elide it — it just adds shim cost. Codegen forces USE_FULL_PRINTF + * on picolibc builds (IDF 6.0+ on all variants) so this file compiles + * to nothing there; the #error below catches a desynchronised gate. * - * On newlib (IDF <= 5 on Xtensa) we keep the original snprintf-then-fwrite - * path because that loopTask stack budget has plenty of headroom for the - * 512-byte buffer; the picolibc-only crash above does not affect it. + * Saves ~11 KB of flash on newlib. * - * Saves ~11 KB of flash on newlib, ~2.8 KB on picolibc. - * - * To disable these wraps, set enable_full_printf: true in the esp32 - * advanced config section. + * To disable this wrap on newlib, set enable_full_printf: true in the + * esp32 advanced config section. */ #if defined(USE_ESP_IDF) && !defined(USE_FULL_PRINTF) + +#ifdef __PICOLIBC__ +#error "printf wrap is net-negative on picolibc; codegen should set USE_FULL_PRINTF" +#endif + #include #include -#ifndef __PICOLIBC__ #include "esp_system.h" -#endif namespace esphome::esp32 {} // NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) extern "C" { -#ifdef __PICOLIBC__ - -#include -#include - -extern int __real_vfprintf(FILE *stream, const char *fmt, va_list ap); - -namespace { - -struct CookieFile { - FILE base; - FILE *target; -}; - -// cookie_put() recovers CookieFile* from FILE* via reinterpret_cast, which is -// only well-defined when FILE is the first member at offset 0 and CookieFile -// is standard-layout. -static_assert(offsetof(CookieFile, base) == 0, "FILE must be the first member of CookieFile"); -static_assert(std::is_standard_layout::value, "CookieFile must be standard-layout"); - -int cookie_put(char c, FILE *stream) { - auto *cookie = reinterpret_cast(stream); - return fputc(static_cast(c), cookie->target); -} - -const FILE COOKIE_FILE_TEMPLATE = FDEV_SETUP_STREAM(cookie_put, nullptr, nullptr, _FDEV_SETUP_WRITE); - -} // namespace - -int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { - CookieFile cookie; - cookie.base = COOKIE_FILE_TEMPLATE; - cookie.target = stream; - return __real_vfprintf(&cookie.base, fmt, ap); -} - -int __wrap_vprintf(const char *fmt, va_list ap) { return __wrap_vfprintf(stdout, fmt, ap); } - -#else // !__PICOLIBC__ - static constexpr size_t PRINTF_BUFFER_SIZE = 512; // These stubs are essentially dead code at runtime — ESPHome replaces the @@ -117,8 +74,6 @@ int __wrap_vfprintf(FILE *stream, const char *fmt, va_list ap) { return write_printf_buffer(stream, buf, vsnprintf(buf, sizeof(buf), fmt, ap)); } -#endif // __PICOLIBC__ - int __wrap_printf(const char *fmt, ...) { va_list ap; va_start(ap, fmt); From be84e6c9f48294c5e231e01ea21e7fadf562c815 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 3 May 2026 17:55:40 -0500 Subject: [PATCH 09/60] [api] Fall back to owning types for service array args used after a delay (#16140) --- esphome/components/api/__init__.py | 45 ++++++++++++++++++++------- tests/components/api/common-base.yaml | 18 +++++++++++ 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 84589d540d..623da2247e 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -72,17 +72,35 @@ APIUnregisterServiceCallAction = api_ns.class_( UserServiceTrigger = api_ns.class_("UserServiceTrigger", automation.Trigger) ListEntitiesServicesArgument = api_ns.class_("ListEntitiesServicesArgument") -SERVICE_ARG_NATIVE_TYPES: dict[str, MockObj] = { +# Owning element type for each YAML service variable type. Used to derive both +# the zero-copy native types and the owning fallback types below. +_SERVICE_ARG_SCALAR_TYPES: dict[str, MockObj] = { "bool": cg.bool_, "int": cg.int32, "float": cg.float_, + "string": cg.std_string, +} +SERVICE_ARG_NATIVE_TYPES: dict[str, MockObj] = { + # Scalars are passed by value; string uses a non-owning view into rx_buf_. + **_SERVICE_ARG_SCALAR_TYPES, "string": cg.StringRef, - "bool[]": cg.FixedVector.template(cg.bool_).operator("const").operator("ref"), - "int[]": cg.FixedVector.template(cg.int32).operator("const").operator("ref"), - "float[]": cg.FixedVector.template(cg.float_).operator("const").operator("ref"), - "string[]": cg.FixedVector.template(cg.std_string) - .operator("const") - .operator("ref"), + # Arrays are passed as non-owning const references into rx_buf_. + **{ + f"{name}[]": cg.FixedVector.template(t).operator("const").operator("ref") + for name, t in _SERVICE_ARG_SCALAR_TYPES.items() + }, +} +# Owning fallback types used when the action chain contains non-synchronous actions +# (delay, wait_until, script.wait, etc.). The default non-owning types reference +# storage in the receive buffer, which is reused once the synchronous portion of +# the chain returns. FixedVector is also non-copyable, so the deferred lambda +# capture in DelayAction::play_complex would fail to compile. +SERVICE_ARG_FALLBACK_TYPES: dict[str, MockObj] = { + "string": cg.std_string, + **{ + f"{name}[]": cg.std_vector.template(t) + for name, t in _SERVICE_ARG_SCALAR_TYPES.items() + }, } CONF_ENCRYPTION = "encryption" CONF_BATCH_DELAY = "batch_delay" @@ -382,17 +400,20 @@ async def to_code(config: ConfigType) -> None: func_args.append((cg.bool_, "return_response")) # Check if action chain has non-synchronous actions that would make - # non-owning StringRef dangle (rx_buf_ reused after delay) + # non-owning args (StringRef, const FixedVector&) dangle once the + # rx_buf_ is reused after a delay/wait_until/script.wait/etc. The + # FixedVector references would also fail to compile because they + # are non-copyable and DelayAction captures args by value. has_non_synchronous = automation.has_non_synchronous_actions( conf.get(CONF_THEN, []) ) service_arg_names: list[str] = [] for name, var_ in conf[CONF_VARIABLES].items(): - native = SERVICE_ARG_NATIVE_TYPES[var_] - # Fall back to std::string for string args if non-synchronous actions exist - if has_non_synchronous and native is cg.StringRef: - native = cg.std_string + if has_non_synchronous and var_ in SERVICE_ARG_FALLBACK_TYPES: + native = SERVICE_ARG_FALLBACK_TYPES[var_] + else: + native = SERVICE_ARG_NATIVE_TYPES[var_] service_template_args.append(native) func_args.append((native, name)) service_arg_names.append(name) diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index c766b61b13..504c52a57b 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -91,6 +91,24 @@ api: - float_arr.size() - string_arr[0].c_str() - string_arr.size() + # Test array + string args used after a non-synchronous action (delay). + # The default non-owning types (StringRef, const FixedVector&) would + # dangle once rx_buf_ is reused, and FixedVector is non-copyable so + # DelayAction's lambda capture would fail to compile. The api codegen + # must fall back to owning std::string / std::vector here. + - action: array_with_delay + variables: + name: string + int_arr: int[] + string_arr: string[] + then: + - delay: 20ms + - logger.log: + format: "Delayed: %s (%u ints, %u strings)" + args: + - name.c_str() + - int_arr.size() + - string_arr.size() # Test ContinuationAction (IfAction with then/else branches) - action: test_if_action variables: From 2d7f9dc48d9d072d0f3c291f31fa83d9d4e6604c Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 5 May 2026 08:12:20 +1200 Subject: [PATCH 10/60] [api] Use safe_print for log output and fix safe_print bytes-repr fallback (#16160) --- esphome/components/api/client.py | 8 ++- esphome/util.py | 26 +++++-- tests/unit_tests/test_util.py | 116 +++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 7 deletions(-) diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index 0c6c569c7d..0982ca905b 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -20,6 +20,7 @@ import contextlib from esphome.const import CONF_KEY, CONF_PORT, __version__ from esphome.core import CORE from esphome.platformio_api import process_stacktrace +from esphome.util import safe_print from . import CONF_ENCRYPTION @@ -60,7 +61,6 @@ async def async_run_logs( noise_psk=noise_psk, addresses=addresses, # Pass all addresses for automatic retry ) - dashboard = CORE.dashboard backtrace_state = False # Try platform-specific stacktrace handler first, fall back to generic @@ -82,7 +82,11 @@ async def async_run_logs( f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}.{nanoseconds:03}]" ) for parsed_msg in parse_log_message(text, timestamp): - print(parsed_msg.replace("\033", "\\033") if dashboard else parsed_msg) + # safe_print handles the dashboard \033 escaping and falls back + # to backslashreplace encoding on stdouts that can't represent + # the wifi signal-bar block characters (Windows redirected + # cp1252 pipe). + safe_print(parsed_msg) for raw_line in text.splitlines(): if platform_process_stacktrace: backtrace_state = platform_process_stacktrace( diff --git a/esphome/util.py b/esphome/util.py index 73cc3aa5ab..e96a52de86 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -94,13 +94,29 @@ def safe_print(message="", end="\n"): except UnicodeEncodeError: pass + # Fall back to the stream's actual encoding (e.g. cp1252 on Windows + # redirected pipes). Use "backslashreplace" so unencodable code points + # like the wifi signal-bar block characters (U+2582..U+2588) become + # readable ``\uXXXX`` escapes, and decode back to ``str`` so ``print`` + # never receives a ``bytes`` object (which would render as a ``b'...'`` + # repr). + encoding = sys.stdout.encoding or "ascii" try: - print(message.encode("utf-8", "backslashreplace"), end=end) + print( + message.encode(encoding, "backslashreplace").decode(encoding), + end=end, + ) + return except UnicodeEncodeError: - try: - print(message.encode("ascii", "backslashreplace"), end=end) - except UnicodeEncodeError: - print("Cannot print line because of invalid locale!") + pass + + try: + print( + message.encode("ascii", "backslashreplace").decode("ascii"), + end=end, + ) + except UnicodeEncodeError: + print("Cannot print line because of invalid locale!") def safe_input(prompt=""): diff --git a/tests/unit_tests/test_util.py b/tests/unit_tests/test_util.py index ff58fb1394..581b1aca99 100644 --- a/tests/unit_tests/test_util.py +++ b/tests/unit_tests/test_util.py @@ -709,3 +709,119 @@ def test_detect_rp2040_bootsel_timeout() -> None: result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 0 assert result.permission_error is False + + +class TestSafePrint: + """Tests for ``safe_print`` and its UnicodeEncodeError fallback chain.""" + + @pytest.fixture(autouse=True) + def _no_dashboard(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Default ``CORE.dashboard`` to False so each test starts hermetic.""" + from esphome.core import CORE + + monkeypatch.setattr(CORE, "dashboard", False) + + def test_prints_plain_message(self, capsys: pytest.CaptureFixture[str]) -> None: + """ASCII-only messages take the fast path through native ``print``.""" + util.safe_print("hello world") + assert capsys.readouterr().out == "hello world\n" + + def test_prints_unicode_on_utf8_stdout( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + """Non-ASCII goes straight through when stdout can encode it.""" + util.safe_print("bars: \u2582\u2584\u2586\u2588") + assert capsys.readouterr().out == "bars: \u2582\u2584\u2586\u2588\n" + + def test_dashboard_escapes_esc_byte( + self, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + r"""Dashboard mode escapes raw ``\033`` ESC bytes to literal ``\\033``.""" + from esphome.core import CORE + + monkeypatch.setattr(CORE, "dashboard", True) + util.safe_print("\033[0;32mhi\033[0m") + assert capsys.readouterr().out == "\\033[0;32mhi\\033[0m\n" + + def test_fallback_writes_string_not_bytes_repr( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Regression: cp1252 fallback must produce a printable str, not ``b'...'``. + + On Windows, when stdout is a redirected pipe (e.g. the dashboard), + Python uses cp1252, which cannot encode the wifi signal-bar block + characters (U+2582..U+2588). The previous fallback path called + ``print(message.encode(...))`` with a ``bytes`` object, which + Python's ``print`` rendered as a literal ``b'...'`` repr — visible + in the user's dashboard output. The fix re-encodes through the + stream's encoding with ``backslashreplace`` and decodes back to + ``str``. + """ + buf = io.BytesIO() + cp1252_stream = io.TextIOWrapper(buf, encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", cp1252_stream) + + util.safe_print("bars: \u2582\u2584\u2586\u2588 done") + cp1252_stream.flush() + output = buf.getvalue().decode("cp1252") + + # Output is a clean line, not the bytes repr. + assert not output.startswith("b'") + assert "b'bars" not in output + # Unencodable codepoints become readable backslash escapes. + assert "\\u2582\\u2584\\u2586\\u2588" in output + # Encodable parts survive unchanged. + assert "bars: " in output + assert " done" in output + assert output.endswith("\n") + + def test_fallback_with_dashboard_escaped_message( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Dashboard ESC escaping + cp1252 fallback compose correctly.""" + from esphome.core import CORE + + monkeypatch.setattr(CORE, "dashboard", True) + buf = io.BytesIO() + cp1252_stream = io.TextIOWrapper(buf, encoding="cp1252", errors="strict") + monkeypatch.setattr(sys, "stdout", cp1252_stream) + + util.safe_print("\033[0;32m\u2582\u2584\u2586\u2588\033[0m") + cp1252_stream.flush() + output = buf.getvalue().decode("cp1252") + + # Dashboard escaping turned ESC into literal "\033" (5 chars), which + # cp1252 can encode, so it survives the round-trip verbatim. + assert "\\033[0;32m" in output + assert "\\033[0m" in output + # Block characters became backslash escapes via backslashreplace. + assert "\\u2582\\u2584\\u2586\\u2588" in output + + def test_final_message_when_locale_is_invalid( + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + """If every encoding path fails, surface the locale-error sentinel.""" + original_print = print + call_count = 0 + + def fake_print(*args: Any, **kwargs: Any) -> None: + nonlocal call_count + call_count += 1 + # The first three calls are: native print, stream-encoding + # fallback, ASCII fallback. Make all three raise so we reach + # the final sentinel "Cannot print line..." which is expected + # to succeed (no encoding required). + if call_count <= 3: + raise UnicodeEncodeError("ascii", "x", 0, 1, "boom") + original_print(*args, **kwargs) + + monkeypatch.setattr("builtins.print", fake_print) + util.safe_print("x") + assert call_count == 4 + assert ( + capsys.readouterr().out == "Cannot print line because of invalid locale!\n" + ) From 197d4dac8e5e9da086f3596f90edbdce78886706 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 5 May 2026 08:27:10 +1200 Subject: [PATCH 11/60] Bump version to 2026.4.4 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 0e6a845ed8..2e86f54e96 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.4.3 +PROJECT_NUMBER = 2026.4.4 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 89b6ff15ee..bc31ab36b5 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.4.3" +__version__ = "2026.4.4" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 857e529803d10971ffd36089068b6590fac3dc6f Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 4 May 2026 18:41:50 -0400 Subject: [PATCH 12/60] [audio] Use the microMP3 library instead of esp-audio-libs (#16236) --- esphome/components/audio/__init__.py | 1 + esphome/components/audio/audio_decoder.cpp | 108 ++++++++++----------- esphome/components/audio/audio_decoder.h | 13 +-- esphome/idf_component.yml | 2 + 4 files changed, 61 insertions(+), 63 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 8528e77ae7..60ff40ea4b 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -395,6 +395,7 @@ async def to_code(config): ) if data.mp3_support: cg.add_define("USE_AUDIO_MP3_SUPPORT") + add_idf_component(name="esphome/micro-mp3", ref="0.2.0") _emit_memory_pair( data.mp3.buffer_memory, "CONFIG_MP3_DECODER_PREFER_PSRAM", diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index baa4c41c06..65a4db4e10 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -20,14 +20,6 @@ AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) this->output_transfer_buffer_ = AudioSinkTransferBuffer::create(output_buffer_size); } -AudioDecoder::~AudioDecoder() { -#ifdef USE_AUDIO_MP3_SUPPORT - if (this->audio_file_type_ == AudioFileType::MP3) { - esp_audio_libs::helix_decoder::MP3FreeDecoder(this->mp3_decoder_); - } -#endif -} - esp_err_t AudioDecoder::add_source(std::weak_ptr &input_ring_buffer) { auto source = AudioSourceTransferBuffer::create(this->input_buffer_size_); if (source == nullptr) { @@ -92,13 +84,10 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { #endif #ifdef USE_AUDIO_MP3_SUPPORT case AudioFileType::MP3: - this->mp3_decoder_ = esp_audio_libs::helix_decoder::MP3InitDecoder(); - - // MP3 always has 1152 samples per chunk - this->free_buffer_required_ = 1152 * sizeof(int16_t) * 2; // samples * size per sample * channels - - // Always reallocate the output transfer buffer to the smallest necessary size - this->output_transfer_buffer_->reallocate(this->free_buffer_required_); + this->mp3_decoder_ = make_unique(); + this->free_buffer_required_ = + this->output_transfer_buffer_->capacity(); // Adjusted and reallocated after reading the header + this->decoder_buffers_internally_ = true; break; #endif #ifdef USE_AUDIO_OPUS_SUPPORT @@ -312,51 +301,56 @@ FileDecoderState AudioDecoder::decode_flac_() { #ifdef USE_AUDIO_MP3_SUPPORT FileDecoderState AudioDecoder::decode_mp3_() { - // Look for the next sync word - int buffer_length = (int) this->input_buffer_->available(); - int32_t offset = esp_audio_libs::helix_decoder::MP3FindSyncWord(this->input_buffer_->data(), buffer_length); + // microMP3's samples_decoded value is samples per channel; e.g., what ESPHome typically calls an audio frame. + // microMP3 uses the term frame to refer to an MP3 frame: an encoded packet that contains multiple audio frames. + size_t bytes_consumed = 0; + size_t samples_decoded = 0; - if (offset < 0) { - // New data may have the sync word - this->input_buffer_->consume(buffer_length); + // microMP3 buffers internally: it consumes from our input buffer at its own pace, emits MP3_STREAM_INFO_READY once + // the first frame header is parsed, and only then produces PCM. It handles sync-word search and ID3v2 tag skipping. + micro_mp3::Mp3Result result = this->mp3_decoder_->decode( + this->input_buffer_->data(), this->input_buffer_->available(), this->output_transfer_buffer_->get_buffer_end(), + this->output_transfer_buffer_->free(), bytes_consumed, samples_decoded); + + this->input_buffer_->consume(bytes_consumed); + + if (result == micro_mp3::MP3_OK) { + if (samples_decoded > 0 && this->audio_stream_info_.has_value()) { + this->output_transfer_buffer_->increase_buffer_length( + this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); + } + } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { + // First successful header parse: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. + this->audio_stream_info_ = + audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); + this->free_buffer_required_ = + this->mp3_decoder_->get_samples_per_frame() * this->mp3_decoder_->get_channels() * sizeof(int16_t); + if (!this->output_transfer_buffer_->reallocate(this->free_buffer_required_)) { + return FileDecoderState::FAILED; + } + } else if (result == micro_mp3::MP3_NEED_MORE_DATA) { + return FileDecoderState::MORE_TO_PROCESS; + } else if (result == micro_mp3::MP3_OUTPUT_BUFFER_TOO_SMALL) { + // Reallocate to decode the frame on the next call + if (this->mp3_decoder_->get_channels() > 0) { + this->free_buffer_required_ = + this->mp3_decoder_->get_samples_per_frame() * this->mp3_decoder_->get_channels() * sizeof(int16_t); + } else { + // Fallback to worst-case size if channel info isn't available + this->free_buffer_required_ = this->mp3_decoder_->get_min_output_buffer_bytes(); + } + if (!this->output_transfer_buffer_->reallocate(this->free_buffer_required_)) { + return FileDecoderState::FAILED; + } + } else if (result == micro_mp3::MP3_DECODE_ERROR) { + // Corrupt frame skipped; recoverable, retry on next call + ESP_LOGW(TAG, "MP3 decoder skipped a corrupt frame"); return FileDecoderState::POTENTIALLY_FAILED; - } - - // Advance read pointer to match the offset for the syncword - this->input_buffer_->consume(offset); - const uint8_t *buffer_start = this->input_buffer_->data(); - - buffer_length = (int) this->input_buffer_->available(); - int err = esp_audio_libs::helix_decoder::MP3Decode(this->mp3_decoder_, &buffer_start, &buffer_length, - (int16_t *) this->output_transfer_buffer_->get_buffer_end(), 0); - - size_t consumed = this->input_buffer_->available() - buffer_length; - this->input_buffer_->consume(consumed); - - if (err) { - switch (err) { - case esp_audio_libs::helix_decoder::ERR_MP3_OUT_OF_MEMORY: - [[fallthrough]]; - case esp_audio_libs::helix_decoder::ERR_MP3_NULL_POINTER: - return FileDecoderState::FAILED; - break; - default: - // Most errors are recoverable by moving on to the next frame, so mark as potentailly failed - return FileDecoderState::POTENTIALLY_FAILED; - break; - } } else { - esp_audio_libs::helix_decoder::MP3FrameInfo mp3_frame_info; - esp_audio_libs::helix_decoder::MP3GetLastFrameInfo(this->mp3_decoder_, &mp3_frame_info); - if (mp3_frame_info.outputSamps > 0) { - int bytes_per_sample = (mp3_frame_info.bitsPerSample / 8); - this->output_transfer_buffer_->increase_buffer_length(mp3_frame_info.outputSamps * bytes_per_sample); - - if (!this->audio_stream_info_.has_value()) { - this->audio_stream_info_ = - audio::AudioStreamInfo(mp3_frame_info.bitsPerSample, mp3_frame_info.nChans, mp3_frame_info.samprate); - } - } + // MP3_ALLOCATION_FAILED, MP3_INPUT_INVALID, or any future error -- not recoverable + ESP_LOGE(TAG, "MP3 decoder failed: %d", static_cast(result)); + return FileDecoderState::FAILED; } return FileDecoderState::MORE_TO_PROCESS; diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index 6e3a228a68..4cbe8b6720 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -16,9 +16,6 @@ #include "esp_err.h" // esp-audio-libs -#ifdef USE_AUDIO_MP3_SUPPORT -#include -#endif #include // micro-flac @@ -26,6 +23,11 @@ #include #endif +// micro-mp3 +#ifdef USE_AUDIO_MP3_SUPPORT +#include +#endif + // micro-opus #ifdef USE_AUDIO_OPUS_SUPPORT #include @@ -62,8 +64,7 @@ class AudioDecoder { /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioDecoder(size_t input_buffer_size, size_t output_buffer_size); - /// @brief Deallocates the MP3 decoder (the flac, opus, and wav decoders are deallocated automatically) - ~AudioDecoder(); + ~AudioDecoder() = default; /// @brief Adds a source ring buffer for raw file data. Takes ownership of the ring buffer in a shared_ptr. /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership @@ -125,7 +126,7 @@ class AudioDecoder { #endif #ifdef USE_AUDIO_MP3_SUPPORT FileDecoderState decode_mp3_(); - esp_audio_libs::helix_decoder::HMP3Decoder mp3_decoder_; + std::unique_ptr mp3_decoder_; #endif #ifdef USE_AUDIO_OPUS_SUPPORT FileDecoderState decode_opus_(); diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index f5a8dd8c60..5ad9090215 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -9,6 +9,8 @@ dependencies: version: 0.2.0 esphome/micro-flac: version: 0.1.1 + esphome/micro-mp3: + version: 0.2.0 esphome/micro-opus: version: 0.4.0 espressif/esp-dsp: From 556783b95ba30c4a40eda780f058c0fee0b18bf9 Mon Sep 17 00:00:00 2001 From: Olivier ARCHER Date: Tue, 5 May 2026 01:19:52 +0200 Subject: [PATCH 13/60] [http_request] remove slow http_request warning on 8266 (#16239) --- esphome/components/http_request/http_request_arduino.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/esphome/components/http_request/http_request_arduino.cpp b/esphome/components/http_request/http_request_arduino.cpp index 217ad0064d..bb5e9427dd 100644 --- a/esphome/components/http_request/http_request_arduino.cpp +++ b/esphome/components/http_request/http_request_arduino.cpp @@ -70,12 +70,6 @@ std::shared_ptr HttpRequestArduino::perform(const std::string &ur stream_ptr = std::make_unique(); #endif // USE_HTTP_REQUEST_ESP8266_HTTPS -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 1, 0) // && USE_ARDUINO_VERSION_CODE < VERSION_CODE(?, ?, ?) - if (!secure) { - ESP_LOGW(TAG, "Using HTTP on Arduino version >= 3.1 is **very** slow. Consider setting framework version to 3.0.2 " - "in your YAML, or use HTTPS"); - } -#endif // USE_ARDUINO_VERSION_CODE bool status = container->client_.begin(*stream_ptr, url.c_str()); #elif defined(USE_RP2040) From d28498ac2c6892a775988bc6712bbd18e5cf12b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 23:39:44 +0000 Subject: [PATCH 14/60] Bump cryptography from 47.0.0 to 48.0.0 (#16245) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 789a3f7995..818453dc8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -cryptography==47.0.0 +cryptography==48.0.0 voluptuous==0.16.0 PyYAML==6.0.3 paho-mqtt==1.6.1 From f33d137669672fe909c3e4c625805f8190b4fc05 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 4 May 2026 19:45:11 -0400 Subject: [PATCH 15/60] [audio][media_player][speaker] WAV decoding is no longer always built (#16244) --- esphome/components/audio/__init__.py | 5 +++-- esphome/components/audio/audio.cpp | 6 ++++++ esphome/components/audio/audio.h | 2 ++ esphome/components/audio/audio_decoder.cpp | 6 ++++++ esphome/components/audio/audio_decoder.h | 18 +++++++++++++----- esphome/components/audio_file/__init__.py | 2 ++ esphome/components/media_player/__init__.py | 3 +++ .../speaker/media_player/__init__.py | 2 ++ .../media_player/speaker_media_player.cpp | 5 ++++- esphome/core/defines.h | 1 + 10 files changed, 42 insertions(+), 8 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 60ff40ea4b..7bd7ba9768 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -64,8 +64,7 @@ class AudioData: flac_support: bool = False mp3_support: bool = False opus_support: bool = False - # WAV defaults to True for backward compatibility; will become opt-in in a future release - wav_support: bool = True + wav_support: bool = False micro_decoder_support: bool = False flac: FlacOptions = field(default_factory=FlacOptions) mp3: Mp3Options = field(default_factory=Mp3Options) @@ -428,3 +427,5 @@ async def to_code(config): add_idf_sdkconfig_option( "CONFIG_OPUS_PSEUDOSTACK_SIZE", data.opus.pseudostack.size ) + if data.wav_support: + cg.add_define("USE_AUDIO_WAV_SUPPORT") diff --git a/esphome/components/audio/audio.cpp b/esphome/components/audio/audio.cpp index 3d675109e4..b977c4e918 100644 --- a/esphome/components/audio/audio.cpp +++ b/esphome/components/audio/audio.cpp @@ -55,8 +55,10 @@ const char *audio_file_type_to_string(AudioFileType file_type) { case AudioFileType::OPUS: return "OPUS"; #endif +#ifdef USE_AUDIO_WAV_SUPPORT case AudioFileType::WAV: return "WAV"; +#endif default: return "unknown"; } @@ -71,9 +73,11 @@ AudioFileType detect_audio_file_type(const char *content_type, const char *url) return AudioFileType::MP3; } #endif +#ifdef USE_AUDIO_WAV_SUPPORT if (strcasecmp(content_type, "audio/wav") == 0) { return AudioFileType::WAV; } +#endif #ifdef USE_AUDIO_FLAC_SUPPORT if (strcasecmp(content_type, "audio/flac") == 0 || strcasecmp(content_type, "audio/x-flac") == 0) { return AudioFileType::FLAC; @@ -91,9 +95,11 @@ AudioFileType detect_audio_file_type(const char *content_type, const char *url) // Fallback to URL extension if (url != nullptr && url[0] != '\0') { +#ifdef USE_AUDIO_WAV_SUPPORT if (str_endswith_ignore_case(url, ".wav")) { return AudioFileType::WAV; } +#endif #ifdef USE_AUDIO_MP3_SUPPORT if (str_endswith_ignore_case(url, ".mp3")) { return AudioFileType::MP3; diff --git a/esphome/components/audio/audio.h b/esphome/components/audio/audio.h index d3b41a362f..9259f0a3c6 100644 --- a/esphome/components/audio/audio.h +++ b/esphome/components/audio/audio.h @@ -116,7 +116,9 @@ enum class AudioFileType : uint8_t { #ifdef USE_AUDIO_OPUS_SUPPORT OPUS, #endif +#ifdef USE_AUDIO_WAV_SUPPORT WAV, +#endif }; struct AudioFile { diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index 65a4db4e10..156704fb86 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -98,6 +98,7 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { this->decoder_buffers_internally_ = true; break; #endif +#ifdef USE_AUDIO_WAV_SUPPORT case AudioFileType::WAV: this->wav_decoder_ = make_unique(); this->wav_decoder_->reset(); @@ -109,6 +110,7 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { this->output_transfer_buffer_->reallocate(this->free_buffer_required_); } break; +#endif case AudioFileType::NONE: default: return ESP_ERR_NOT_SUPPORTED; @@ -226,9 +228,11 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { state = this->decode_opus_(); break; #endif +#ifdef USE_AUDIO_WAV_SUPPORT case AudioFileType::WAV: state = this->decode_wav_(); break; +#endif case AudioFileType::NONE: default: state = FileDecoderState::IDLE; @@ -395,6 +399,7 @@ FileDecoderState AudioDecoder::decode_opus_() { } #endif +#ifdef USE_AUDIO_WAV_SUPPORT FileDecoderState AudioDecoder::decode_wav_() { if (!this->audio_stream_info_.has_value()) { // Header hasn't been processed @@ -441,6 +446,7 @@ FileDecoderState AudioDecoder::decode_wav_() { return FileDecoderState::END_OF_FILE; } +#endif } // namespace audio } // namespace esphome diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index 4cbe8b6720..8769e0b38b 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -15,9 +15,6 @@ #include "esp_err.h" -// esp-audio-libs -#include - // micro-flac #ifdef USE_AUDIO_FLAC_SUPPORT #include @@ -33,6 +30,11 @@ #include #endif +// esp-audio-libs +#ifdef USE_AUDIO_WAV_SUPPORT +#include +#endif + namespace esphome { namespace audio { @@ -56,7 +58,7 @@ class AudioDecoder { * @brief Class that facilitates decoding an audio file. * The audio file is read from a source (ring buffer or const data pointer), decoded, and sent to an audio sink * (ring buffer, speaker component, or callback). - * Supports wav, flac, mp3, and ogg opus formats. + * Supports flac, mp3, ogg opus, and wav formats (each enabled independently at compile time). */ public: /// @brief Allocates the output transfer buffer and stores the input buffer size for later use by add_source() @@ -119,7 +121,6 @@ class AudioDecoder { void set_pause_output_state(bool pause_state) { this->pause_output_ = pause_state; } protected: - std::unique_ptr wav_decoder_; #ifdef USE_AUDIO_FLAC_SUPPORT FileDecoderState decode_flac_(); std::unique_ptr flac_decoder_; @@ -132,7 +133,10 @@ class AudioDecoder { FileDecoderState decode_opus_(); std::unique_ptr opus_decoder_; #endif +#ifdef USE_AUDIO_WAV_SUPPORT FileDecoderState decode_wav_(); + std::unique_ptr wav_decoder_; +#endif std::unique_ptr input_buffer_; std::unique_ptr output_transfer_buffer_; @@ -142,14 +146,18 @@ class AudioDecoder { size_t input_buffer_size_{0}; size_t free_buffer_required_{0}; +#ifdef USE_AUDIO_WAV_SUPPORT size_t wav_bytes_left_{0}; +#endif uint32_t potentially_failed_count_{0}; uint32_t accumulated_frames_written_{0}; uint32_t playback_ms_{0}; bool end_of_file_{false}; +#ifdef USE_AUDIO_WAV_SUPPORT bool wav_has_known_end_{false}; +#endif bool decoder_buffers_internally_{false}; diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index 88be6db168..b246633c31 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -193,6 +193,8 @@ def _validate_supported_local_file(config: list[ConfigType]) -> list[ConfigType] audio.request_mp3_support() elif media_file_type_str == str(audio.AUDIO_FILE_TYPE_ENUM["OPUS"]): audio.request_opus_support() + elif media_file_type_str == str(audio.AUDIO_FILE_TYPE_ENUM["WAV"]): + audio.request_wav_support() return config diff --git a/esphome/components/media_player/__init__.py b/esphome/components/media_player/__init__.py index 0024e3b965..aa1e88dca9 100644 --- a/esphome/components/media_player/__init__.py +++ b/esphome/components/media_player/__init__.py @@ -133,6 +133,7 @@ def request_codecs_for_format_configs( audio.request_flac_support() audio.request_mp3_support() audio.request_opus_support() + audio.request_wav_support() else: if "FLAC" in needed_formats: audio.request_flac_support() @@ -140,6 +141,8 @@ def request_codecs_for_format_configs( audio.request_mp3_support() if "OPUS" in needed_formats: audio.request_opus_support() + if "WAV" in needed_formats: + audio.request_wav_support() # Local config key constants diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index fbc83ef12f..90d9309f46 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -210,6 +210,8 @@ def _final_validate(config): audio.request_mp3_support() elif fmt_name == "OPUS": audio.request_opus_support() + elif fmt_name == "WAV": + audio.request_wav_support() break return config diff --git a/esphome/components/speaker/media_player/speaker_media_player.cpp b/esphome/components/speaker/media_player/speaker_media_player.cpp index ab11a89c3f..afd93b3f45 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.cpp +++ b/esphome/components/speaker/media_player/speaker_media_player.cpp @@ -17,9 +17,12 @@ namespace speaker { // - Each stream has an individual speaker component for output // - Each stream is handled by an ``AudioPipeline`` object with two parts/tasks // - ``AudioReader`` handles reading from an HTTP source or from a PROGMEM flash set at compile time -// - ``AudioDecoder`` handles decoding the audio file. All formats are limited to two channels and 16 bits per sample +// - ``AudioDecoder`` handles decoding the audio file. All formats are limited to two channels and 16 bits per +// sample. +// Each format is enabled independently at compile time: // - FLAC // - MP3 (based on the libhelix decoder) +// - Ogg Opus // - WAV // - Each task runs until it is done processing the file or it receives a stop command // - Inter-task communication uses a FreeRTOS Event Group diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 93f4307e12..85454d3cc0 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -180,6 +180,7 @@ #define USE_AUDIO_FLAC_SUPPORT #define USE_AUDIO_MP3_SUPPORT #define USE_AUDIO_OPUS_SUPPORT +#define USE_AUDIO_WAV_SUPPORT #define USE_API #define USE_API_CLIENT_CONNECTED_TRIGGER #define USE_API_CLIENT_DISCONNECTED_TRIGGER From ea2b2b39201a80c13d2ae1620179b2edc7bb40e9 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Mon, 4 May 2026 21:12:26 -0400 Subject: [PATCH 16/60] [audio_file] Use microDecoder library instead of manual task management/decoding (#16237) --- .../audio_file/media_source/__init__.py | 28 +- .../media_source/audio_file_media_source.cpp | 332 +++++++----------- .../media_source/audio_file_media_source.h | 43 ++- 3 files changed, 165 insertions(+), 238 deletions(-) diff --git a/esphome/components/audio_file/media_source/__init__.py b/esphome/components/audio_file/media_source/__init__.py index e9e292a2b2..635a51b610 100644 --- a/esphome/components/audio_file/media_source/__init__.py +++ b/esphome/components/audio_file/media_source/__init__.py @@ -1,5 +1,7 @@ +from typing import Any + import esphome.codegen as cg -from esphome.components import media_source, psram +from esphome.components import audio, esp32, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -13,19 +15,30 @@ AudioFileMediaSource = audio_file_ns.class_( "AudioFileMediaSource", cg.Component, media_source.MediaSource ) + +def _request_micro_decoder(config: ConfigType) -> ConfigType: + audio.request_micro_decoder_support() + return config + + +def _validate_task_stack_in_psram(value: Any) -> bool: + if value := cv.boolean(value): + return cv.requires_component(psram.DOMAIN)(value) + return value + + CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioFileMediaSource, ) .extend( { - cv.Optional(CONF_TASK_STACK_IN_PSRAM): cv.All( - cv.boolean, cv.requires_component(psram.DOMAIN) - ), + cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), cv.only_on_esp32, + _request_micro_decoder, ) @@ -34,5 +47,8 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await media_source.register_media_source(var, config) - if CONF_TASK_STACK_IN_PSRAM in config: - cg.add(var.set_task_stack_in_psram(config[CONF_TASK_STACK_IN_PSRAM])) + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + esp32.add_idf_sdkconfig_option( + "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True + ) diff --git a/esphome/components/audio_file/media_source/audio_file_media_source.cpp b/esphome/components/audio_file/media_source/audio_file_media_source.cpp index fbb5ecd88d..0cda1eca9e 100644 --- a/esphome/components/audio_file/media_source/audio_file_media_source.cpp +++ b/esphome/components/audio_file/media_source/audio_file_media_source.cpp @@ -2,281 +2,185 @@ #ifdef USE_ESP32 -#include "esphome/components/audio/audio_decoder.h" +#include "esphome/core/log.h" + +#include +#include -#include #include namespace esphome::audio_file { -namespace { // anonymous namespace for internal linkage -struct AudioSinkAdapter : public audio::AudioSinkCallback { - media_source::MediaSource *source; - audio::AudioStreamInfo stream_info; - - size_t audio_sink_write(uint8_t *data, size_t length, TickType_t ticks_to_wait) override { - return this->source->write_output(data, length, pdTICKS_TO_MS(ticks_to_wait), this->stream_info); - } -}; -} // namespace - -#if defined(USE_AUDIO_OPUS_SUPPORT) -static constexpr uint32_t DECODE_TASK_STACK_SIZE = 5 * 1024; -#else -static constexpr uint32_t DECODE_TASK_STACK_SIZE = 3 * 1024; -#endif - static const char *const TAG = "audio_file_media_source"; -enum EventGroupBits : uint32_t { - // Requests to start playback (set by play_uri, handled by loop) - REQUEST_START = (1 << 0), - // Commands from main loop to decode task - COMMAND_STOP = (1 << 1), - COMMAND_PAUSE = (1 << 2), - // Decode task lifecycle signals (one-shot, cleared by loop) - TASK_STARTING = (1 << 7), - TASK_RUNNING = (1 << 8), - TASK_STOPPING = (1 << 9), - TASK_STOPPED = (1 << 10), - TASK_ERROR = (1 << 11), - // Decode task state (level-triggered, set/cleared by decode task) - TASK_PAUSED = (1 << 12), - ALL_BITS = 0x00FFFFFF, // All valid FreeRTOS event group bits -}; +static constexpr uint32_t AUDIO_WRITE_TIMEOUT_MS = 50; +static constexpr size_t DECODER_TASK_STACK_SIZE = 5120; +static constexpr uint8_t DECODER_TASK_PRIORITY = 2; +static constexpr uint32_t PAUSE_POLL_DELAY_MS = 20; +static constexpr char URI_PREFIX[] = "audio-file://"; + +namespace { // anonymous namespace for internal linkage + +// audio::AudioFileType and micro_decoder::AudioFileType use different numeric layouts (audio's +// values shift with USE_AUDIO_*_SUPPORT defines; micro_decoder's are fixed and guarded by +// MICRO_DECODER_CODEC_*). The codec request flow in audio/__init__.py keeps the two sets of +// guards aligned, so a switch with matching #ifdefs covers all reachable cases. +micro_decoder::AudioFileType to_micro_decoder_type(audio::AudioFileType type) { + switch (type) { +#ifdef USE_AUDIO_FLAC_SUPPORT + case audio::AudioFileType::FLAC: + return micro_decoder::AudioFileType::FLAC; +#endif +#ifdef USE_AUDIO_MP3_SUPPORT + case audio::AudioFileType::MP3: + return micro_decoder::AudioFileType::MP3; +#endif +#ifdef USE_AUDIO_OPUS_SUPPORT + case audio::AudioFileType::OPUS: + return micro_decoder::AudioFileType::OPUS; +#endif +#ifdef USE_AUDIO_WAV_SUPPORT + case audio::AudioFileType::WAV: + return micro_decoder::AudioFileType::WAV; +#endif + default: + return micro_decoder::AudioFileType::NONE; + } +} + +} // namespace void AudioFileMediaSource::dump_config() { - ESP_LOGCONFIG(TAG, "Audio File Media Source:"); - ESP_LOGCONFIG(TAG, " Task Stack in PSRAM: %s", this->task_stack_in_psram_ ? "Yes" : "No"); + ESP_LOGCONFIG(TAG, + "Audio File Media Source:\n" + " Decoder Task Stack in PSRAM: %s", + YESNO(this->decoder_task_stack_in_psram_)); } void AudioFileMediaSource::setup() { this->disable_loop(); - this->event_group_ = xEventGroupCreate(); - if (this->event_group_ == nullptr) { - ESP_LOGE(TAG, "Failed to create event group"); + micro_decoder::DecoderConfig config; + config.audio_write_timeout_ms = AUDIO_WRITE_TIMEOUT_MS; + config.decoder_priority = DECODER_TASK_PRIORITY; + config.decoder_stack_size = DECODER_TASK_STACK_SIZE; + config.decoder_stack_in_psram = this->decoder_task_stack_in_psram_; + + this->decoder_ = std::make_unique(config); + if (this->decoder_ == nullptr) { + ESP_LOGE(TAG, "Failed to allocate decoder"); this->mark_failed(); return; } + this->decoder_->set_listener(this); } -void AudioFileMediaSource::loop() { - EventBits_t event_bits = xEventGroupGetBits(this->event_group_); +void AudioFileMediaSource::loop() { this->decoder_->loop(); } - if (event_bits & REQUEST_START) { - xEventGroupClearBits(this->event_group_, REQUEST_START); - this->decoding_state_ = AudioFileDecodingState::START_TASK; - } - - switch (this->decoding_state_) { - case AudioFileDecodingState::START_TASK: { - if (!this->decode_task_.is_created()) { - xEventGroupClearBits(this->event_group_, ALL_BITS); - if (!this->decode_task_.create(decode_task, "AudioFileDec", DECODE_TASK_STACK_SIZE, this, 1, - this->task_stack_in_psram_)) { - ESP_LOGE(TAG, "Failed to create task"); - this->status_momentary_error("task_create", 1000); - this->set_state_(media_source::MediaSourceState::ERROR); - this->decoding_state_ = AudioFileDecodingState::IDLE; - return; - } - } - this->decoding_state_ = AudioFileDecodingState::DECODING; - break; - } - case AudioFileDecodingState::DECODING: { - if (event_bits & TASK_STARTING) { - ESP_LOGD(TAG, "Starting"); - xEventGroupClearBits(this->event_group_, TASK_STARTING); - } - - if (event_bits & TASK_RUNNING) { - ESP_LOGV(TAG, "Started"); - xEventGroupClearBits(this->event_group_, TASK_RUNNING); - this->set_state_(media_source::MediaSourceState::PLAYING); - } - - if ((event_bits & TASK_PAUSED) && this->get_state() != media_source::MediaSourceState::PAUSED) { - this->set_state_(media_source::MediaSourceState::PAUSED); - } else if (!(event_bits & TASK_PAUSED) && this->get_state() == media_source::MediaSourceState::PAUSED) { - this->set_state_(media_source::MediaSourceState::PLAYING); - } - - if (event_bits & TASK_STOPPING) { - ESP_LOGV(TAG, "Stopping"); - xEventGroupClearBits(this->event_group_, TASK_STOPPING); - } - - if (event_bits & TASK_ERROR) { - // Report error so the orchestrator knows playback failed; task will have already logged the specific error - this->set_state_(media_source::MediaSourceState::ERROR); - } - - if (event_bits & TASK_STOPPED) { - ESP_LOGD(TAG, "Stopped"); - xEventGroupClearBits(this->event_group_, ALL_BITS); - - this->decode_task_.deallocate(); - this->set_state_(media_source::MediaSourceState::IDLE); - this->decoding_state_ = AudioFileDecodingState::IDLE; - } - break; - } - case AudioFileDecodingState::IDLE: { - if (this->get_state() == media_source::MediaSourceState::ERROR && !this->status_has_error()) { - this->set_state_(media_source::MediaSourceState::IDLE); - } - break; - } - } - - if ((this->decoding_state_ == AudioFileDecodingState::IDLE) && - (this->get_state() == media_source::MediaSourceState::IDLE)) { - this->disable_loop(); - } -} +bool AudioFileMediaSource::can_handle(const std::string &uri) const { return uri.starts_with(URI_PREFIX); } // Called from the orchestrator's main loop, so no synchronization needed with loop() bool AudioFileMediaSource::play_uri(const std::string &uri) { - if (!this->is_ready() || this->is_failed() || this->status_has_error() || !this->has_listener() || - xEventGroupGetBits(this->event_group_) & REQUEST_START) { + if (!this->is_ready() || this->is_failed() || this->status_has_error() || !this->has_listener()) { return false; } - // Check if source is already playing if (this->get_state() != media_source::MediaSourceState::IDLE) { ESP_LOGE(TAG, "Cannot play '%s': source is busy", uri.c_str()); return false; } - // Validate URI starts with "audio-file://" - if (!uri.starts_with("audio-file://")) { + if (!uri.starts_with(URI_PREFIX)) { ESP_LOGE(TAG, "Invalid URI: '%s'", uri.c_str()); return false; } - // Strip "audio-file://" prefix and find the file - const char *file_id = uri.c_str() + 13; // "audio-file://" is 13 characters - + const char *file_id = uri.c_str() + sizeof(URI_PREFIX) - 1; + this->current_file_ = nullptr; for (const auto &named_file : get_named_audio_files()) { if (strcmp(named_file.file_id, file_id) == 0) { this->current_file_ = named_file.file; - xEventGroupSetBits(this->event_group_, EventGroupBits::REQUEST_START); - this->enable_loop(); - return true; + break; } } - ESP_LOGE(TAG, "Unknown file: '%s'", file_id); + if (this->current_file_ == nullptr) { + ESP_LOGE(TAG, "Unknown file: '%s'", file_id); + return false; + } + + micro_decoder::AudioFileType type = to_micro_decoder_type(this->current_file_->file_type); + if (this->decoder_->play_buffer(this->current_file_->data, this->current_file_->length, type)) { + this->pause_.store(false, std::memory_order_relaxed); + this->enable_loop(); + return true; + } + + ESP_LOGE(TAG, "Failed to start playback of '%s'", file_id); return false; } // Called from the orchestrator's main loop, so no synchronization needed with loop() void AudioFileMediaSource::handle_command(media_source::MediaSourceCommand command) { - if (this->decoding_state_ != AudioFileDecodingState::DECODING) { - return; - } - switch (command) { case media_source::MediaSourceCommand::STOP: - xEventGroupSetBits(this->event_group_, EventGroupBits::COMMAND_STOP); + this->decoder_->stop(); break; case media_source::MediaSourceCommand::PAUSE: - xEventGroupSetBits(this->event_group_, EventGroupBits::COMMAND_PAUSE); + // Only valid while actively playing; ignoring from IDLE/ERROR/PAUSED prevents the state + // machine from getting stuck in PAUSED when no playback is active (which would block the + // next play_uri() call via its IDLE-state precondition). + if (this->get_state() != media_source::MediaSourceState::PLAYING) + break; + // PAUSE does not stop the decoder task. Instead, on_audio_write() returns 0 and temporarily + // yields, which fills any internal buffering and applies back pressure that effectively + // pauses the decoder task. + this->set_state_(media_source::MediaSourceState::PAUSED); + this->pause_.store(true, std::memory_order_relaxed); break; case media_source::MediaSourceCommand::PLAY: - xEventGroupClearBits(this->event_group_, EventGroupBits::COMMAND_PAUSE); + if (this->get_state() != media_source::MediaSourceState::PAUSED) + break; + this->set_state_(media_source::MediaSourceState::PLAYING); + this->pause_.store(false, std::memory_order_relaxed); break; default: break; } } -void AudioFileMediaSource::decode_task(void *params) { - AudioFileMediaSource *this_source = static_cast(params); +// Called from the decoder task. Forwards to the orchestrator's listener, which is responsible for +// being thread-safe with respect to its own audio writer. +size_t AudioFileMediaSource::on_audio_write(const uint8_t *data, size_t length, uint32_t timeout_ms) { + if (this->pause_.load(std::memory_order_relaxed)) { + vTaskDelay(pdMS_TO_TICKS(PAUSE_POLL_DELAY_MS)); + return 0; + } + return this->write_output(data, length, timeout_ms, this->stream_info_); +} - do { // do-while(false) ensures RAII objects are destroyed on all exit paths via break +// Called from the decoder task before the first on_audio_write(). +void AudioFileMediaSource::on_stream_info(const micro_decoder::AudioStreamInfo &info) { + this->stream_info_ = audio::AudioStreamInfo(info.get_bits_per_sample(), info.get_channels(), info.get_sample_rate()); +} - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_STARTING); - - // 0 bytes for input transfer buffer makes it an inplace buffer - std::unique_ptr decoder = make_unique(0, 4096); - - esp_err_t err = decoder->start(this_source->current_file_->file_type); - if (err != ESP_OK) { - ESP_LOGE(TAG, "Failed to start decoder: %s", esp_err_to_name(err)); - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_ERROR | EventGroupBits::TASK_STOPPING); +// microDecoder invokes on_state_change() from inside decoder_->loop(), so this runs on the main +// loop thread and it's safe to call set_state_() directly. +void AudioFileMediaSource::on_state_change(micro_decoder::DecoderState state) { + switch (state) { + case micro_decoder::DecoderState::IDLE: + this->set_state_(media_source::MediaSourceState::IDLE); + this->disable_loop(); break; - } - - // Add the file as a const data source - decoder->add_source(this_source->current_file_->data, this_source->current_file_->length); - - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_RUNNING); - - AudioSinkAdapter audio_sink; - bool has_stream_info = false; - - while (true) { - EventBits_t event_bits = xEventGroupGetBits(this_source->event_group_); - - if (event_bits & EventGroupBits::COMMAND_STOP) { - break; - } - - bool paused = event_bits & EventGroupBits::COMMAND_PAUSE; - decoder->set_pause_output_state(paused); - if (paused) { - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_PAUSED); - vTaskDelay(pdMS_TO_TICKS(20)); - } else { - xEventGroupClearBits(this_source->event_group_, EventGroupBits::TASK_PAUSED); - } - - // Will stop gracefully once finished with the current file - audio::AudioDecoderState decoder_state = decoder->decode(true); - - if (decoder_state == audio::AudioDecoderState::FINISHED) { - break; - } else if (decoder_state == audio::AudioDecoderState::FAILED) { - ESP_LOGE(TAG, "Decoder failed"); - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_ERROR); - break; - } - - if (!has_stream_info && decoder->get_audio_stream_info().has_value()) { - has_stream_info = true; - - audio::AudioStreamInfo stream_info = decoder->get_audio_stream_info().value(); - - ESP_LOGD(TAG, "Bits per sample: %d, Channels: %d, Sample rate: %" PRIu32, stream_info.get_bits_per_sample(), - stream_info.get_channels(), stream_info.get_sample_rate()); - - if (stream_info.get_bits_per_sample() != 16 || stream_info.get_channels() > 2) { - ESP_LOGE(TAG, "Incompatible audio stream. Only 16 bits per sample and 1 or 2 channels are supported"); - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_ERROR); - break; - } - - audio_sink.source = this_source; - audio_sink.stream_info = stream_info; - esp_err_t err = decoder->add_sink(&audio_sink); - if (err != ESP_OK) { - ESP_LOGE(TAG, "Failed to add sink: %s", esp_err_to_name(err)); - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_ERROR); - break; - } - } - } - - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_STOPPING); - } while (false); - - // All RAII objects from the do-while block (decoder, audio_sink, etc.) are now destroyed. - - xEventGroupSetBits(this_source->event_group_, EventGroupBits::TASK_STOPPED); - vTaskSuspend(nullptr); // Suspend this task indefinitely until the loop method deletes it + case micro_decoder::DecoderState::PLAYING: + this->set_state_(media_source::MediaSourceState::PLAYING); + break; + case micro_decoder::DecoderState::FAILED: + this->set_state_(media_source::MediaSourceState::ERROR); + break; + default: + break; + } } } // namespace esphome::audio_file diff --git a/esphome/components/audio_file/media_source/audio_file_media_source.h b/esphome/components/audio_file/media_source/audio_file_media_source.h index 75e18c13b8..2c6189f272 100644 --- a/esphome/components/audio_file/media_source/audio_file_media_source.h +++ b/esphome/components/audio_file/media_source/audio_file_media_source.h @@ -8,41 +8,48 @@ #include "esphome/components/audio_file/audio_file.h" #include "esphome/components/media_source/media_source.h" #include "esphome/core/component.h" -#include "esphome/core/static_task.h" -#include -#include +#include +#include + +#include +#include +#include namespace esphome::audio_file { -enum class AudioFileDecodingState : uint8_t { - START_TASK, - DECODING, - IDLE, -}; - -class AudioFileMediaSource : public Component, public media_source::MediaSource { +// Inherits from two unrelated listener-style interfaces: +// - media_source::MediaSource: this source reports state and writes audio *to* an orchestrator +// (the orchestrator calls set_listener() on us with a MediaSourceListener*). +// - micro_decoder::DecoderListener: the underlying decoder calls back *into* us with decoded +// audio and state changes (we call decoder_->set_listener(this) in setup()). +class AudioFileMediaSource : public Component, public media_source::MediaSource, public micro_decoder::DecoderListener { public: void setup() override; void loop() override; void dump_config() override; + void set_task_stack_in_psram(bool task_stack_in_psram) { this->decoder_task_stack_in_psram_ = task_stack_in_psram; } + // MediaSource interface implementation bool play_uri(const std::string &uri) override; void handle_command(media_source::MediaSourceCommand command) override; - bool can_handle(const std::string &uri) const override { return uri.starts_with("audio-file://"); } + bool can_handle(const std::string &uri) const override; - void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } + // DecoderListener interface implementation + size_t on_audio_write(const uint8_t *data, size_t length, uint32_t timeout_ms) override; + void on_stream_info(const micro_decoder::AudioStreamInfo &info) override; + void on_state_change(micro_decoder::DecoderState state) override; protected: - static void decode_task(void *params); - + std::unique_ptr decoder_; + audio::AudioStreamInfo stream_info_; audio::AudioFile *current_file_{nullptr}; - AudioFileDecodingState decoding_state_{AudioFileDecodingState::IDLE}; - EventGroupHandle_t event_group_{nullptr}; - StaticTask decode_task_; - bool task_stack_in_psram_{false}; + // Written from the main loop in handle_command(), read from the decoder task in + // on_audio_write(). Must be atomic to avoid a data race. + std::atomic pause_{false}; + bool decoder_task_stack_in_psram_{false}; }; } // namespace esphome::audio_file From efff8fe8be8a8b14076e92ecef57a78132fcba56 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 5 May 2026 14:29:23 +1200 Subject: [PATCH 17/60] [platformio_api] Remove duplicated _strip_win_long_path_prefix (#16249) --- esphome/platformio_api.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index fa88776acf..81ff01306a 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -12,37 +12,6 @@ from esphome.util import FlashImage, run_external_process _LOGGER = logging.getLogger(__name__) -def _strip_win_long_path_prefix(path: str) -> str: - r"""Strip the Windows extended-length path prefix from ``path``. - - Handles both forms documented at - https://learn.microsoft.com/windows/win32/fileio/naming-a-file: - - * ``\\?\C:\path\to\file`` -> ``C:\path\to\file`` - * ``\\?\UNC\server\share\path`` -> ``\\server\share\path`` - - The NSIS-installed ``esphome.exe`` launcher on Windows starts Python with - ``sys.executable`` already prefixed with ``\\?\``. That prefix propagates - into PlatformIO's ``$PYTHONEXE`` (PlatformIO reads ``PYTHONEXEPATH`` from - the environment, falling back to ``os.path.normpath(sys.executable)``) - and ends up baked into SCons-emitted command lines for build steps such - as the esp8266 ``elf2bin`` invocation. ``cmd.exe`` does not understand - the ``\\?\`` prefix, so the build fails with - "The system cannot find the path specified." Stripping the prefix early - keeps the path shell-quotable. - - No-op on non-Windows platforms. - """ - if sys.platform != "win32": - return path - if path.startswith("\\\\?\\UNC\\"): - # \\?\UNC\server\share\... -> \\server\share\... - return "\\\\" + path[len("\\\\?\\UNC\\") :] - if path.startswith("\\\\?\\"): - return path[len("\\\\?\\") :] - return path - - def _strip_win_long_path_prefix(path: str) -> str: r"""Strip the Windows extended-length path prefix from ``path``. From edbb9f7b287da034eb529d325aa0f4245d2e3068 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 5 May 2026 07:15:32 -0500 Subject: [PATCH 18/60] [i2s_audio] Fix stereo playback when slot bit width exceeds data bit width (#16248) --- .../i2s_audio/speaker/i2s_audio_speaker_standard.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp index 0203464034..edb316e3a2 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp @@ -280,6 +280,9 @@ esp_err_t I2SAudioSpeaker::start_i2s_driver(audio::AudioStreamInfo &audio_stream } #else slot_cfg.slot_bit_width = this->slot_bit_width_; + if (this->slot_bit_width_ != I2S_SLOT_BIT_WIDTH_AUTO) { + slot_cfg.ws_width = static_cast(this->slot_bit_width_); + } #endif // USE_ESP32_VARIANT_ESP32 slot_cfg.slot_mask = slot_mask; From 87a705b1cc69ccfb07d2234c49356590db670621 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 5 May 2026 08:47:07 -0400 Subject: [PATCH 19/60] [audio] Bump microOpus to v0.4.1 (#16255) --- esphome/components/audio/__init__.py | 2 +- esphome/idf_component.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 7bd7ba9768..7ecc45db5a 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -402,7 +402,7 @@ async def to_code(config): ) if data.opus_support: cg.add_define("USE_AUDIO_OPUS_SUPPORT") - add_idf_component(name="esphome/micro-opus", ref="0.4.0") + add_idf_component(name="esphome/micro-opus", ref="0.4.1") if data.opus.floating_point is not None: add_idf_sdkconfig_option( "CONFIG_OPUS_FLOATING_POINT", data.opus.floating_point diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 5ad9090215..e8cad29439 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -12,7 +12,7 @@ dependencies: esphome/micro-mp3: version: 0.2.0 esphome/micro-opus: - version: 0.4.0 + version: 0.4.1 espressif/esp-dsp: version: "1.7.1" espressif/esp-tflite-micro: From 57397a318a2e08ea4c44bb7836358a7da20985f2 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 5 May 2026 12:21:02 -0400 Subject: [PATCH 20/60] [audio] Use the microWAV library for decoding (#16251) --- esphome/components/audio/__init__.py | 1 + esphome/components/audio/audio_decoder.cpp | 78 ++++++++-------------- esphome/components/audio/audio_decoder.h | 14 +--- esphome/idf_component.yml | 2 + 4 files changed, 35 insertions(+), 60 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 7ecc45db5a..80fd328e48 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -429,3 +429,4 @@ async def to_code(config): ) if data.wav_support: cg.add_define("USE_AUDIO_WAV_SUPPORT") + add_idf_component(name="esphome/micro-wav", ref="0.2.0") diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index 156704fb86..7abd03a36e 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -79,7 +79,6 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { this->flac_decoder_ = make_unique(); this->free_buffer_required_ = this->output_transfer_buffer_->capacity(); // Adjusted and reallocated after reading the header - this->decoder_buffers_internally_ = true; break; #endif #ifdef USE_AUDIO_MP3_SUPPORT @@ -87,7 +86,6 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { this->mp3_decoder_ = make_unique(); this->free_buffer_required_ = this->output_transfer_buffer_->capacity(); // Adjusted and reallocated after reading the header - this->decoder_buffers_internally_ = true; break; #endif #ifdef USE_AUDIO_OPUS_SUPPORT @@ -95,16 +93,12 @@ esp_err_t AudioDecoder::start(AudioFileType audio_file_type) { this->opus_decoder_ = make_unique(); this->free_buffer_required_ = this->output_transfer_buffer_->capacity(); // Adjusted and reallocated after reading the header - this->decoder_buffers_internally_ = true; break; #endif #ifdef USE_AUDIO_WAV_SUPPORT case AudioFileType::WAV: - this->wav_decoder_ = make_unique(); - this->wav_decoder_->reset(); - - // Processing WAVs doesn't actually require a specific amount of buffer size, as it is already in PCM format. - // Thus, we don't reallocate to a minimum size. + this->wav_decoder_ = make_unique(); + // 1 KiB suffices to always make progress while avoiding excessive CPU spinning for decoding this->free_buffer_required_ = 1024; if (this->output_transfer_buffer_->capacity() < this->free_buffer_required_) { this->output_transfer_buffer_->reallocate(this->free_buffer_required_); @@ -181,10 +175,8 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { // Decode more audio - // Only shift data on the first loop iteration to avoid unnecessary, slow moves - // If the decoder buffers internally, then never shift - size_t bytes_read = this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), - first_loop_iteration && !this->decoder_buffers_internally_); + // Never shift the input buffer; every decoder buffers internally and consumes only what it processed. + size_t bytes_read = this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); if (!first_loop_iteration && (this->input_buffer_->available() < bytes_processed)) { // Less data is available than what was processed in last iteration, so don't attempt to decode. @@ -401,50 +393,38 @@ FileDecoderState AudioDecoder::decode_opus_() { #ifdef USE_AUDIO_WAV_SUPPORT FileDecoderState AudioDecoder::decode_wav_() { - if (!this->audio_stream_info_.has_value()) { - // Header hasn't been processed + // microWAV's samples_decoded counts individual channel samples; e.g., for + // 16-bit stereo, 4 input bytes results in 2 samples_decoded. + size_t bytes_consumed = 0; + size_t samples_decoded = 0; - esp_audio_libs::wav_decoder::WAVDecoderResult result = - this->wav_decoder_->decode_header(this->input_buffer_->data(), this->input_buffer_->available()); + micro_wav::WAVDecoderResult result = this->wav_decoder_->decode( + this->input_buffer_->data(), this->input_buffer_->available(), this->output_transfer_buffer_->get_buffer_end(), + this->output_transfer_buffer_->free(), bytes_consumed, samples_decoded); - if (result == esp_audio_libs::wav_decoder::WAV_DECODER_SUCCESS_IN_DATA) { - this->input_buffer_->consume(this->wav_decoder_->bytes_processed()); + this->input_buffer_->consume(bytes_consumed); - this->audio_stream_info_ = audio::AudioStreamInfo( - this->wav_decoder_->bits_per_sample(), this->wav_decoder_->num_channels(), this->wav_decoder_->sample_rate()); - - this->wav_bytes_left_ = this->wav_decoder_->chunk_bytes_left(); - this->wav_has_known_end_ = (this->wav_bytes_left_ > 0); - return FileDecoderState::MORE_TO_PROCESS; - } else if (result == esp_audio_libs::wav_decoder::WAV_DECODER_WARNING_INCOMPLETE_DATA) { - // Available data didn't have the full header - return FileDecoderState::POTENTIALLY_FAILED; - } else { - return FileDecoderState::FAILED; + if (result == micro_wav::WAV_DECODER_SUCCESS) { + if (samples_decoded > 0 && this->audio_stream_info_.has_value()) { + this->output_transfer_buffer_->increase_buffer_length( + this->audio_stream_info_.value().samples_to_bytes(samples_decoded)); } + } else if (result == micro_wav::WAV_DECODER_HEADER_READY) { + // After HEADER_READY, get_bits_per_sample() returns the output bit depth + // (16 for A-law/mu-law, 32 for IEEE float, original value for PCM). + this->audio_stream_info_ = + audio::AudioStreamInfo(this->wav_decoder_->get_bits_per_sample(), this->wav_decoder_->get_channels(), + this->wav_decoder_->get_sample_rate()); + } else if (result == micro_wav::WAV_DECODER_NEED_MORE_DATA) { + return FileDecoderState::MORE_TO_PROCESS; + } else if (result == micro_wav::WAV_DECODER_END_OF_STREAM) { + return FileDecoderState::END_OF_FILE; } else { - if (!this->wav_has_known_end_ || (this->wav_bytes_left_ > 0)) { - size_t bytes_to_copy = this->input_buffer_->available(); - - if (this->wav_has_known_end_) { - bytes_to_copy = std::min(bytes_to_copy, this->wav_bytes_left_); - } - - bytes_to_copy = std::min(bytes_to_copy, this->output_transfer_buffer_->free()); - - if (bytes_to_copy > 0) { - std::memcpy(this->output_transfer_buffer_->get_buffer_end(), this->input_buffer_->data(), bytes_to_copy); - this->input_buffer_->consume(bytes_to_copy); - this->output_transfer_buffer_->increase_buffer_length(bytes_to_copy); - if (this->wav_has_known_end_) { - this->wav_bytes_left_ -= bytes_to_copy; - } - } - return FileDecoderState::IDLE; - } + ESP_LOGE(TAG, "WAV decoder failed: %d", static_cast(result)); + return FileDecoderState::FAILED; } - return FileDecoderState::END_OF_FILE; + return FileDecoderState::MORE_TO_PROCESS; } #endif diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index 8769e0b38b..58e982317c 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -30,9 +30,9 @@ #include #endif -// esp-audio-libs +// micro-wav #ifdef USE_AUDIO_WAV_SUPPORT -#include +#include #endif namespace esphome { @@ -135,7 +135,7 @@ class AudioDecoder { #endif #ifdef USE_AUDIO_WAV_SUPPORT FileDecoderState decode_wav_(); - std::unique_ptr wav_decoder_; + std::unique_ptr wav_decoder_; #endif std::unique_ptr input_buffer_; @@ -146,20 +146,12 @@ class AudioDecoder { size_t input_buffer_size_{0}; size_t free_buffer_required_{0}; -#ifdef USE_AUDIO_WAV_SUPPORT - size_t wav_bytes_left_{0}; -#endif uint32_t potentially_failed_count_{0}; uint32_t accumulated_frames_written_{0}; uint32_t playback_ms_{0}; bool end_of_file_{false}; -#ifdef USE_AUDIO_WAV_SUPPORT - bool wav_has_known_end_{false}; -#endif - - bool decoder_buffers_internally_{false}; bool pause_output_{false}; }; diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index e8cad29439..14fb11ace5 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -13,6 +13,8 @@ dependencies: version: 0.2.0 esphome/micro-opus: version: 0.4.1 + esphome/micro-wav: + version: 0.2.0 espressif/esp-dsp: version: "1.7.1" espressif/esp-tflite-micro: From be82e8faeb35fde79b4d5e8ad51e57d8cc8de8e5 Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Wed, 6 May 2026 01:02:26 +0200 Subject: [PATCH 21/60] [debug] Remove unused buffer in uicr lambda function (#16208) --- esphome/components/debug/debug_zephyr.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/esphome/components/debug/debug_zephyr.cpp b/esphome/components/debug/debug_zephyr.cpp index 81c8612784..e23a0f668a 100644 --- a/esphome/components/debug/debug_zephyr.cpp +++ b/esphome/components/debug/debug_zephyr.cpp @@ -401,7 +401,6 @@ size_t DebugComponent::get_device_info_(std::span #endif auto uicr = [](volatile uint32_t *data, uint8_t size) { std::string res; - char buf[sizeof(uint32_t) * 2 + 1]; for (size_t i = 0; i < size; i++) { if (i > 0) { res += ' '; From f30ad588ea421db25f0784b91365bb44f5991751 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:25:53 -0500 Subject: [PATCH 22/60] [cli] Add --ota-platform flag to pick web_server or native API OTA (#16207) --- esphome/__main__.py | 166 +++++- esphome/web_server_ota.py | 202 +++++++ tests/unit_tests/test_main.py | 283 ++++++++++ tests/unit_tests/test_web_server_ota.py | 670 ++++++++++++++++++++++++ 4 files changed, 1307 insertions(+), 14 deletions(-) create mode 100644 esphome/web_server_ota.py create mode 100644 tests/unit_tests/test_web_server_ota.py diff --git a/esphome/__main__.py b/esphome/__main__.py index 222e299f6d..f4a276b74c 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -28,6 +28,7 @@ from esphome.const import ( ALLOWED_NAME_CHARS, ARGUMENT_HELP_DEVICE, CONF_API, + CONF_AUTH, CONF_BAUD_RATE, CONF_BROKER, CONF_DEASSERT_RTS_DTR, @@ -47,6 +48,8 @@ from esphome.const import ( CONF_PORT, CONF_SUBSTITUTIONS, CONF_TOPIC, + CONF_USERNAME, + CONF_WEB_SERVER, ENV_NOGITIGNORE, KEY_CORE, KEY_NATIVE_IDF, @@ -349,6 +352,17 @@ def choose_upload_log_host( elif bootsel.permission_error: bootsel_permission_error = True + # Annotate the OTA chooser entry only in the non-default case: when the + # config has web_server OTA but no native API OTA, the upload will fall + # through to the HTTP path and the user benefits from seeing that + # explicitly. The native-API path is the default and gets a plain label + # to avoid noise on the most common scenario. For LOGGING the OTA + # transport doesn't apply, so always leave the label plain. + if purpose == Purpose.UPLOADING and not has_native_ota() and has_web_server_ota(): + ota_suffix = " via web_server" + else: + ota_suffix = "" + def add_ota_options() -> None: """Add OTA options, using mDNS discovery if name_add_mac_suffix is enabled.""" if (discovered := _discover_mac_suffix_devices()) is not None: @@ -356,11 +370,11 @@ def choose_upload_log_host( # intentionally skip the base-name fallback since with # name_add_mac_suffix on, the base name doesn't exist on the net. for host in discovered: - options.append((f"Over The Air ({host})", host)) + options.append((f"Over The Air{ota_suffix} ({host})", host)) elif has_resolvable_address(): - options.append((f"Over The Air ({CORE.address})", CORE.address)) + options.append((f"Over The Air{ota_suffix} ({CORE.address})", CORE.address)) if has_mqtt_ip_lookup(): - options.append(("Over The Air (MQTT IP lookup)", "MQTTIP")) + options.append((f"Over The Air{ota_suffix} (MQTT IP lookup)", "MQTTIP")) if purpose == Purpose.LOGGING: if has_mqtt_logging(): @@ -429,7 +443,19 @@ def has_api() -> bool: def has_ota() -> bool: - """Check if OTA upload is available (requires platform: esphome).""" + """Check if any network OTA upload is available. + + True if the config exposes either ``platform: esphome`` (native API + OTA) or ``platform: web_server`` (HTTP OTA). Both reach the device + over the same network stack, so the OTA discovery path treats them + interchangeably; ``upload_program`` picks the actual transport based + on ``--ota-platform`` and what's configured. + """ + return has_native_ota() or has_web_server_ota() + + +def has_native_ota() -> bool: + """Check if native API OTA upload is available (``platform: esphome``).""" if CONF_OTA not in CORE.config: return False return any( @@ -438,6 +464,16 @@ def has_ota() -> bool: ) +def has_web_server_ota() -> bool: + """Check if web_server OTA upload is available (``platform: web_server``).""" + if CONF_OTA not in CORE.config: + return False + return any( + ota_item.get(CONF_PLATFORM) == CONF_WEB_SERVER + for ota_item in CORE.config[CONF_OTA] + ) + + def has_mqtt_ip_lookup() -> bool: """Check if MQTT is available and IP lookup is supported.""" from esphome.components.mqtt import CONF_DISCOVER_IP @@ -1115,25 +1151,83 @@ def upload_program( return exit_code, host if exit_code == 0 else None - ota_conf = {} + requested_platform = getattr(args, "ota_platform", None) + chosen_platform = _choose_ota_platform(config, requested_platform) + + # Resolve MQTT magic strings to actual IP addresses + network_devices = _resolve_network_devices(devices, config, args) + + if chosen_platform == CONF_WEB_SERVER: + if getattr(args, "partition_table", False): + raise EsphomeError( + "--partition-table is only supported with the esphome OTA platform; " + "the web_server OTA path can only update the firmware image." + ) + binary = CORE.firmware_bin + if getattr(args, "file", None) is not None: + binary = Path(args.file) + return _upload_via_web_server(config, network_devices, binary) + + return _upload_via_native_api(config, network_devices, args) + + +def _choose_ota_platform(config: ConfigType, requested: str | None) -> str: + """Pick the OTA platform to use, optionally honoring ``--ota-platform``. + + Default behavior prefers ``esphome`` (native API) when it is configured. + The native API uses challenge-response auth with MD5/SHA256 hashing of a + server-issued nonce, so the password is never sent over the wire; the + ``web_server`` path uses HTTP Basic auth which transmits credentials in + cleartext over the LAN. (The native path also supports gzip compression + on ESP8266, where flash space is tight; on ESP32/RP2040/LibreTiny the + backend reports ``supports_compression() == false`` and the firmware is + sent uncompressed regardless of which platform is used.) Falls back to + ``web_server`` only when that is the only available platform. + """ + # Use a dict (insertion-ordered) instead of a list so error messages and + # membership checks see one entry per platform even if the user has + # multiple ``ota:`` items of the same platform; the web_server OTA + # platform's final-validate hook merges duplicates anyway. + available: dict[str, None] = {} for ota_item in config.get(CONF_OTA, []): - if ota_item[CONF_PLATFORM] == CONF_ESPHOME: + platform = ota_item.get(CONF_PLATFORM) + if platform in (CONF_ESPHOME, CONF_WEB_SERVER): + available[platform] = None + + if not available: + raise EsphomeError( + f"Cannot upload Over the Air as the {CONF_OTA} configuration is not " + f"present or does not include {CONF_PLATFORM}: {CONF_ESPHOME} or " + f"{CONF_PLATFORM}: {CONF_WEB_SERVER}" + ) + + if requested is not None: + if requested not in available: + raise EsphomeError( + f"--ota-platform {requested} was requested but the configuration " + f"only provides: {', '.join(available)}" + ) + return requested + + if CONF_ESPHOME in available: + return CONF_ESPHOME + return CONF_WEB_SERVER + + +def _upload_via_native_api( + config: ConfigType, network_devices: list[str], args: ArgsProtocol +) -> tuple[int, str | None]: + ota_conf: ConfigType = {} + for ota_item in config.get(CONF_OTA, []): + if ota_item.get(CONF_PLATFORM) == CONF_ESPHOME: ota_conf = ota_item break - if not ota_conf: - raise EsphomeError( - f"Cannot upload Over the Air as the {CONF_OTA} configuration is not present or does not include {CONF_PLATFORM}: {CONF_ESPHOME}" - ) - from esphome import espota2 remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) - # Resolve MQTT magic strings to actual IP addresses - network_devices = _resolve_network_devices(devices, config, args) - binary = CORE.firmware_bin ota_type = espota2.OTA_TYPE_UPDATE_APP if getattr(args, "partition_table", False): @@ -1157,6 +1251,28 @@ def upload_program( return espota2.run_ota(network_devices, remote_port, password, binary, ota_type) +def _upload_via_web_server( + config: ConfigType, network_devices: list[str], binary: Path +) -> tuple[int, str | None]: + web_conf = config.get(CONF_WEB_SERVER) + if not web_conf: + raise EsphomeError( + f"Cannot upload via web_server OTA: the {CONF_WEB_SERVER} component " + f"is not configured." + ) + + remote_port = int(web_conf[CONF_PORT]) + auth = web_conf.get(CONF_AUTH) or {} + username = auth.get(CONF_USERNAME) + password = auth.get(CONF_PASSWORD) + + from esphome import web_server_ota + + return web_server_ota.run_ota( + network_devices, remote_port, username, password, binary + ) + + # Layout of esp_partition_info_t on flash. Each entry is 32 bytes, leading with a # 16-bit little-endian magic. ESP-IDF defines ESP_PARTITION_MAGIC = 0x50AA (stored as # bytes 0xAA, 0x50) for partition entries and ESP_PARTITION_MAGIC_MD5 = 0xEBEB for the @@ -1877,6 +1993,17 @@ def parse_args(argv): "--file", help="Manually specify the binary file to upload.", ) + parser_upload.add_argument( + "--ota-platform", + choices=[CONF_ESPHOME, CONF_WEB_SERVER], + help=( + "OTA platform to use for network uploads. Defaults to " + f"'{CONF_ESPHOME}' (native API) when configured because it uses " + "challenge-response auth so the password is never sent in " + f"cleartext on the wire. Falls back to '{CONF_WEB_SERVER}' " + "(HTTP Basic auth) when that is the only configured platform." + ), + ) parser_upload.add_argument( "--partition-table", help="Upload as partition table (OTA).", @@ -1951,6 +2078,17 @@ def parse_args(argv): help="Build with native ESP-IDF instead of PlatformIO (ESP32 esp-idf framework only).", action="store_true", ) + parser_run.add_argument( + "--ota-platform", + choices=[CONF_ESPHOME, CONF_WEB_SERVER], + help=( + "OTA platform to use for network uploads. Defaults to " + f"'{CONF_ESPHOME}' (native API) when configured because it uses " + "challenge-response auth so the password is never sent in " + f"cleartext on the wire. Falls back to '{CONF_WEB_SERVER}' " + "(HTTP Basic auth) when that is the only configured platform." + ), + ) parser_clean = subparsers.add_parser( "clean-mqtt", diff --git a/esphome/web_server_ota.py b/esphome/web_server_ota.py new file mode 100644 index 0000000000..a49f46b270 --- /dev/null +++ b/esphome/web_server_ota.py @@ -0,0 +1,202 @@ +"""HTTP-based OTA upload via the ``web_server`` component's ``/update`` endpoint. + +This is the alternative to ``espota2`` (the native API OTA path). Useful when +a device only has ``platform: web_server`` configured under ``ota:``, or when +the user has lost the native OTA password but still has ``web_server`` basic +auth credentials. +""" + +from __future__ import annotations + +import io +import logging +from pathlib import Path +import secrets +import socket +from typing import BinaryIO + +import requests +from requests.auth import HTTPBasicAuth + +from esphome.core import EsphomeError +from esphome.helpers import ProgressBar, resolve_ip_address + +_LOGGER = logging.getLogger(__name__) + +OTA_PATH = "/update" +FORM_FIELD = "update" +# (connect_timeout, read_timeout). The device reboots after a successful +# upload so the read side must allow for a slow flash + response. +TIMEOUT = (20.0, 120.0) + + +class WebServerOTAError(EsphomeError): + pass + + +class _MultipartStreamer: + """Stream a single-file multipart/form-data body during transmission. + + ``requests.post(files=...)`` materializes the entire body in memory before + sending, so a progress callback wired into the file-like fires during + encoding instead of during the network send. Pass this via ``data=`` + (with ``__len__`` so urllib3 sets ``Content-Length`` instead of using + chunked transfer encoding); urllib3 then calls ``read(blocksize)`` + repeatedly during the POST and the progress bar tracks bytes leaving the + host. + """ + + def __init__(self, file: BinaryIO, file_size: int, filename: str) -> None: + self.boundary = f"esphomeOTA{secrets.token_hex(16)}" + prefix = ( + f"--{self.boundary}\r\n" + f'Content-Disposition: form-data; name="{FORM_FIELD}"; ' + f'filename="{filename}"\r\n' + f"Content-Type: application/octet-stream\r\n\r\n" + ).encode() + suffix = f"\r\n--{self.boundary}--\r\n".encode() + # Walked in order; ``read()`` advances to the next source on EOF. + self._sources: list[BinaryIO] = [io.BytesIO(prefix), file, io.BytesIO(suffix)] + self._idx = 0 + self._total = len(prefix) + file_size + len(suffix) + self._sent = 0 + self.progress = ProgressBar() + + def __len__(self) -> int: + return self._total + + @property + def content_type(self) -> str: + return f"multipart/form-data; boundary={self.boundary}" + + def read(self, size: int = -1) -> bytes: + remaining = self._total if size is None or size < 0 else size + out = bytearray() + while remaining > 0 and self._idx < len(self._sources): + chunk = self._sources[self._idx].read(remaining) + if not chunk: + self._idx += 1 + continue + out += chunk + remaining -= len(chunk) + if out: + self._sent += len(out) + self.progress.update(self._sent / self._total) + return bytes(out) + + +def _try_upload( + host: str, + port: int, + username: str | None, + password: str | None, + filename: Path, +) -> tuple[int, str | None]: + from esphome.core import CORE + + try: + addr_infos = resolve_ip_address(host, port, address_cache=CORE.address_cache) + except EsphomeError as err: + _LOGGER.error( + "Error resolving IP address of %s. Is it connected to WiFi?", host + ) + if not CORE.dashboard: + _LOGGER.error("(If you know the IP, try --device )") + raise WebServerOTAError(err) from err + + if not addr_infos: + _LOGGER.error("Could not resolve %s", host) + return 1, None + + file_size = filename.stat().st_size + _LOGGER.info("Uploading %s (%s bytes) via web_server OTA", filename, file_size) + auth = HTTPBasicAuth(username, password) if username and password else None + + # Iterate resolved IPs (IPv4 + IPv6 candidates) just like espota2 does. + for af, _socktype, _, _, sa in addr_infos: + ip = sa[0] + # IPv6 literals must be wrapped in brackets in URLs; link-local + # addresses need a percent-encoded zone index per RFC 6874. + if af == socket.AF_INET6: + scope = sa[3] if len(sa) >= 4 else 0 + host_part = f"[{ip}%25{scope}]" if scope else f"[{ip}]" + else: + host_part = ip + url = f"http://{host_part}:{port}{OTA_PATH}" + _LOGGER.info("Connecting to %s port %s...", ip, port) + + try: + with open(filename, "rb") as fh: + streamer = _MultipartStreamer(fh, file_size, filename.name) + try: + response = requests.post( + url, + data=streamer, + auth=auth, + timeout=TIMEOUT, + headers={ + "Content-Type": streamer.content_type, + "Connection": "close", + }, + ) + finally: + streamer.progress.done() + except requests.RequestException as err: + _LOGGER.error("OTA upload to %s port %s failed: %s", ip, port, err) + continue + + if response.status_code == 401: + raise WebServerOTAError( + "Authentication failed (HTTP 401). Check the 'web_server' " + "'auth' username and password." + ) + if response.status_code != 200: + detail = response.text.strip() or response.reason or "no response body" + raise WebServerOTAError( + f"Unexpected HTTP {response.status_code} response from device: {detail}" + ) + + # The endpoint returns HTTP 200 for both success and failure; the + # body is what tells us which (see ota_web_server.cpp handleRequest). + body = response.text.strip() + if "Successful" in body: + _LOGGER.info("Device response: %s", body) + _LOGGER.info("OTA successful") + return 0, ip + + raise WebServerOTAError( + f"Device reported OTA failure: {body or 'no response body'}" + ) + + return 1, None + + +def run_ota( + remote_hosts: str | list[str], + remote_port: int, + username: str | None, + password: str | None, + filename: Path, +) -> tuple[int, str | None]: + """Upload ``filename`` to the first reachable host via ``web_server`` OTA. + + Mirrors :func:`esphome.espota2.run_ota` so callers can swap between the + two paths with the same return contract: ``(0, host)`` on success or + ``(1, None)`` on failure. + """ + hosts = [remote_hosts] if isinstance(remote_hosts, str) else list(remote_hosts) + for host in hosts: + try: + exit_code, used_host = _try_upload( + host, remote_port, username, password, filename + ) + except WebServerOTAError as err: + _LOGGER.error("%s", err) + continue + if exit_code == 0: + return 0, used_host + # Reached only when every attempt failed; per-attempt errors were + # already logged. This summary line gives the user an unambiguous + # "stop reading, nothing worked" marker. + _LOGGER.error("OTA upload failed.") + return 1, None diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index f8a3ea888e..0b96000a57 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -43,6 +43,7 @@ from esphome.__main__ import ( has_non_ip_address, has_ota, has_resolvable_address, + has_web_server_ota, mqtt_get_ip, run_esphome, run_miniterm, @@ -58,6 +59,7 @@ from esphome.components import esp32 from esphome.components.esp32 import KEY_ESP32, KEY_VARIANT, VARIANT_ESP32 from esphome.const import ( CONF_API, + CONF_AUTH, CONF_BAUD_RATE, CONF_BROKER, CONF_DISABLED, @@ -76,6 +78,8 @@ from esphome.const import ( CONF_SUBSTITUTIONS, CONF_TOPIC, CONF_USE_ADDRESS, + CONF_USERNAME, + CONF_WEB_SERVER, CONF_WIFI, KEY_CORE, KEY_TARGET_PLATFORM, @@ -213,6 +217,13 @@ def mock_run_ota() -> Generator[Mock]: yield mock +@pytest.fixture +def mock_run_web_server_ota() -> Generator[Mock]: + """Mock web_server_ota.run_ota for testing.""" + with patch("esphome.web_server_ota.run_ota") as mock: + yield mock + + @pytest.fixture def mock_is_ip_address() -> Generator[Mock]: """Mock is_ip_address for testing.""" @@ -1114,6 +1125,7 @@ class MockArgs: reset: bool = False list_only: bool = False output: str | None = None + ota_platform: str | None = None partition_table: bool = False @@ -1878,6 +1890,277 @@ def test_upload_program_ota_no_config( upload_program(config, args, devices) +def test_has_web_server_ota_detects_platform() -> None: + """has_web_server_ota returns True when web_server OTA platform is configured.""" + setup_core( + config={ + CONF_OTA: [{CONF_PLATFORM: CONF_WEB_SERVER}], + } + ) + assert has_web_server_ota() is True + assert has_ota() is True + + +def test_has_web_server_ota_returns_false_without_config() -> None: + """has_web_server_ota returns False when only native OTA is configured.""" + setup_core( + config={ + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME}], + } + ) + assert has_web_server_ota() is False + assert has_ota() is True + + +def test_upload_program_web_server_only_auto_dispatches( + mock_run_web_server_ota: Mock, + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """When only web_server OTA is configured, upload_program picks it automatically.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_web_server_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [{CONF_PLATFORM: CONF_WEB_SERVER}], + CONF_WEB_SERVER: { + CONF_PORT: 80, + CONF_AUTH: {CONF_USERNAME: "admin", CONF_PASSWORD: "pw"}, + }, + } + args = MockArgs() + devices = ["192.168.1.100"] + + exit_code, host = upload_program(config, args, devices) + + assert exit_code == 0 + assert host == "192.168.1.100" + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_web_server_ota.assert_called_once_with( + ["192.168.1.100"], 80, "admin", "pw", expected_firmware + ) + mock_run_ota.assert_not_called() + + +def test_upload_program_web_server_no_auth( + mock_run_web_server_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """web_server OTA works without an auth block (passes None for credentials).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_web_server_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [{CONF_PLATFORM: CONF_WEB_SERVER}], + CONF_WEB_SERVER: {CONF_PORT: 8080}, + } + args = MockArgs() + devices = ["192.168.1.100"] + + exit_code, host = upload_program(config, args, devices) + + assert exit_code == 0 + assert host == "192.168.1.100" + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_web_server_ota.assert_called_once_with( + ["192.168.1.100"], 8080, None, None, expected_firmware + ) + + +def test_upload_program_both_platforms_default_prefers_native( + mock_run_ota: Mock, + mock_run_web_server_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """When both OTA platforms are configured, default selection is native API.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_PASSWORD: "secret", + }, + {CONF_PLATFORM: CONF_WEB_SERVER}, + ], + CONF_WEB_SERVER: {CONF_PORT: 80}, + } + args = MockArgs() + devices = ["192.168.1.100"] + + exit_code, host = upload_program(config, args, devices) + + assert exit_code == 0 + assert host == "192.168.1.100" + mock_run_ota.assert_called_once() + mock_run_web_server_ota.assert_not_called() + + +def test_upload_program_ota_platform_override_to_web_server( + mock_run_ota: Mock, + mock_run_web_server_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """--ota-platform web_server forces web_server OTA even when native is configured.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_web_server_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_PASSWORD: "secret", + }, + {CONF_PLATFORM: CONF_WEB_SERVER}, + ], + CONF_WEB_SERVER: {CONF_PORT: 80}, + } + args = MockArgs(ota_platform=CONF_WEB_SERVER) + devices = ["192.168.1.100"] + + exit_code, host = upload_program(config, args, devices) + + assert exit_code == 0 + assert host == "192.168.1.100" + mock_run_ota.assert_not_called() + mock_run_web_server_ota.assert_called_once() + + +def test_upload_program_ota_platform_unavailable( + mock_get_port_type: Mock, +) -> None: + """--ota-platform must reference a platform that is actually configured.""" + setup_core(platform=PLATFORM_ESP32) + mock_get_port_type.return_value = "NETWORK" + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_PASSWORD: "secret", + } + ], + } + args = MockArgs(ota_platform=CONF_WEB_SERVER) + devices = ["192.168.1.100"] + + with pytest.raises(EsphomeError, match="--ota-platform web_server"): + upload_program(config, args, devices) + + +def test_upload_program_web_server_missing_component( + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """web_server OTA without a web_server component fails with a clear error.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + + config = { + CONF_OTA: [{CONF_PLATFORM: CONF_WEB_SERVER}], + # No CONF_WEB_SERVER + } + args = MockArgs() + devices = ["192.168.1.100"] + + with pytest.raises(EsphomeError, match="web_server.*not configured"): + upload_program(config, args, devices) + + +def test_upload_program_unrelated_ota_platform_ignored( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """OTA list entries that are neither esphome nor web_server are ignored. + + Covers the false branch in _choose_ota_platform's filter loop and the + no-match branch in _upload_via_native_api's lookup loop. + """ + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [ + {CONF_PLATFORM: "http_request"}, # unrelated platform; ignored + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_PASSWORD: "secret", + }, + ], + } + args = MockArgs() + devices = ["192.168.1.100"] + + exit_code, host = upload_program(config, args, devices) + + assert exit_code == 0 + assert host == "192.168.1.100" + mock_run_ota.assert_called_once() + + +def test_upload_program_duplicate_platform_dedup_in_error( + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Duplicate same-platform OTA entries don't repeat in --ota-platform errors.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + + config = { + CONF_OTA: [ + {CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}, + {CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3233}, + ], + } + args = MockArgs(ota_platform=CONF_WEB_SERVER) + devices = ["192.168.1.100"] + + with pytest.raises(EsphomeError) as excinfo: + upload_program(config, args, devices) + + # Error mentions esphome once in the platform list, not "esphome, esphome". + msg = str(excinfo.value) + assert "esphome, esphome" not in msg + assert msg.endswith(": esphome") + + +def test_upload_program_only_unrelated_ota_platforms( + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Only unrelated OTA platforms configured -> raises like missing OTA.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + + config = { + CONF_OTA: [{CONF_PLATFORM: "http_request"}], + } + args = MockArgs() + devices = ["192.168.1.100"] + + with pytest.raises(EsphomeError, match="Cannot upload Over the Air"): + upload_program(config, args, devices) + + def test_upload_program_ota_with_mqtt_resolution( mock_mqtt_get_ip: Mock, mock_is_ip_address: Mock, diff --git a/tests/unit_tests/test_web_server_ota.py b/tests/unit_tests/test_web_server_ota.py new file mode 100644 index 0000000000..606905e36e --- /dev/null +++ b/tests/unit_tests/test_web_server_ota.py @@ -0,0 +1,670 @@ +"""Unit tests for esphome.web_server_ota module.""" + +from __future__ import annotations + +import io +import logging +from pathlib import Path +import socket +from unittest.mock import MagicMock, patch + +import pytest +import requests +from requests.auth import HTTPBasicAuth + +from esphome.core import CORE, EsphomeError +from esphome.helpers import ProgressBar +from esphome.web_server_ota import ( + OTA_PATH, + WebServerOTAError, + _MultipartStreamer, + run_ota, +) + + +@pytest.fixture +def firmware(tmp_path: Path) -> Path: + binary = tmp_path / "firmware.bin" + binary.write_bytes(b"\x00\x01\x02FIRMWARE\xff" * 64) + return binary + + +def _make_response(status: int, body: str) -> MagicMock: + response = MagicMock(spec=requests.Response) + response.status_code = status + response.text = body + response.reason = "" + return response + + +def _patch_resolve( + monkeypatch: pytest.MonkeyPatch, hosts: list[tuple[str, int]] +) -> None: + """Replace resolve_ip_address so tests don't actually do DNS.""" + addr_infos = [ + (socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port)) + for host, port in hosts + ] + monkeypatch.setattr( + "esphome.web_server_ota.resolve_ip_address", lambda *a, **kw: addr_infos + ) + + +# --------------------------------------------------------------------------- +# _MultipartStreamer +# --------------------------------------------------------------------------- + + +def test_multipart_streamer_emits_full_body() -> None: + """Streaming the whole body in one call yields prefix + file + suffix.""" + data = b"abcdef" * 100 + streamer = _MultipartStreamer(io.BytesIO(data), len(data), "fw.bin") + + body = streamer.read() + while True: + chunk = streamer.read() + if not chunk: + break + body += chunk + + assert body.startswith(f"--{streamer.boundary}\r\n".encode()) + assert b'name="update"' in body + assert b'filename="fw.bin"' in body + assert data in body + assert body.endswith(f"\r\n--{streamer.boundary}--\r\n".encode()) + + +def test_multipart_streamer_chunked_read_matches_full_read() -> None: + """Chunked reads (urllib3 calls read(8192) repeatedly) yield the same body.""" + data = b"abcdef" * 1000 # 6000 bytes + full = _MultipartStreamer(io.BytesIO(data), len(data), "fw.bin").read() + + streamed = bytearray() + s = _MultipartStreamer(io.BytesIO(data), len(data), "fw.bin") + # Same boundary lengths -> identical total length. + while True: + chunk = s.read(64) + if not chunk: + break + streamed += chunk + # Boundaries are random per instance, so compare lengths and structure. + assert len(streamed) == len(full) + assert streamed.startswith(f"--{s.boundary}\r\n".encode()) + assert streamed.endswith(f"\r\n--{s.boundary}--\r\n".encode()) + + +def test_multipart_streamer_len_matches_emitted_bytes() -> None: + """``__len__`` is what urllib3 uses to set Content-Length, so it must + equal the total bytes emitted by ``read``.""" + data = b"x" * 12345 + s = _MultipartStreamer(io.BytesIO(data), len(data), "fw.bin") + declared = len(s) + + emitted = 0 + while True: + chunk = s.read(1024) + if not chunk: + break + emitted += len(chunk) + + assert emitted == declared + + +def test_multipart_streamer_progress_ticks_during_read() -> None: + """Each read advances the progress bar (this is the whole point of + streaming via ``data=``: progress reflects bytes leaving the host).""" + data = b"x" * 1000 + s = _MultipartStreamer(io.BytesIO(data), len(data), "fw.bin") + + updates: list[float] = [] + s.progress.update = updates.append # type: ignore[method-assign] + + while True: + chunk = s.read(128) + if not chunk: + break + + assert updates, "progress.update was never called" + # Strictly non-decreasing. + assert updates == sorted(updates) + # Final update reaches (within FP) 1.0 because all bytes were read. + assert updates[-1] == pytest.approx(1.0, abs=1e-9) + + +def test_multipart_streamer_content_type_includes_boundary() -> None: + s = _MultipartStreamer(io.BytesIO(b""), 0, "fw.bin") + assert s.content_type == f"multipart/form-data; boundary={s.boundary}" + + +def test_multipart_streamer_zero_size_file() -> None: + """A zero-byte file still produces a well-formed body and progress is + skipped (avoiding a divide-by-zero on the empty file segment).""" + s = _MultipartStreamer(io.BytesIO(b""), 0, "empty.bin") + body = b"" + while True: + chunk = s.read(64) + if not chunk: + break + body += chunk + assert body.startswith(f"--{s.boundary}".encode()) + assert body.endswith(f"--{s.boundary}--\r\n".encode()) + + +def test_multipart_streamer_unique_boundary_per_instance() -> None: + a = _MultipartStreamer(io.BytesIO(b""), 0, "a") + b = _MultipartStreamer(io.BytesIO(b""), 0, "a") + assert a.boundary != b.boundary + + +def test_multipart_streamer_zero_size_read_returns_empty() -> None: + """``read(0)`` short-circuits without touching state.""" + s = _MultipartStreamer(io.BytesIO(b"x" * 10), 10, "fw.bin") + assert s.read(0) == b"" + # No bytes consumed. + assert s._sent == 0 + + +# --------------------------------------------------------------------------- +# run_ota +# --------------------------------------------------------------------------- + + +def test_run_ota_success(monkeypatch: pytest.MonkeyPatch, firmware: Path) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + exit_code, host = run_ota(["device.local"], 80, None, None, firmware) + + assert exit_code == 0 + assert host == "192.168.1.50" + post.assert_called_once() + args, kwargs = post.call_args + assert args == (f"http://192.168.1.50:80{OTA_PATH}",) + assert kwargs["auth"] is None + # Streaming body, not files=, so progress fires during transmission. + assert "files" not in kwargs + assert isinstance(kwargs["data"], _MultipartStreamer) + assert kwargs["headers"]["Content-Type"] == kwargs["data"].content_type + assert kwargs["headers"]["Connection"] == "close" + + +def test_run_ota_logs_device_response_body( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The device's HTTP response body is surfaced on success.""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + caplog.set_level(logging.INFO, logger="esphome.web_server_ota") + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ): + run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert "Device response: Update Successful!" in caplog.text + assert "OTA successful" in caplog.text + + +def test_run_ota_log_says_via_web_server( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The upload-start log line names the transport explicitly.""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + caplog.set_level(logging.INFO, logger="esphome.web_server_ota") + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ): + run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert "via web_server OTA" in caplog.text + + +def test_run_ota_sends_basic_auth( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + exit_code, _ = run_ota(["192.168.1.50"], 80, "admin", "secret", firmware) + + assert exit_code == 0 + auth = post.call_args.kwargs["auth"] + assert isinstance(auth, HTTPBasicAuth) + assert auth.username == "admin" + assert auth.password == "secret" + + +def test_run_ota_skips_auth_when_no_credentials( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert post.call_args.kwargs["auth"] is None + + +def test_run_ota_skips_auth_when_only_username( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """Both username and password are required to send Basic auth.""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + run_ota(["192.168.1.50"], 80, "admin", None, firmware) + + assert post.call_args.kwargs["auth"] is None + + +def test_run_ota_uses_update_url( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 8080)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + run_ota(["192.168.1.50"], 8080, None, None, firmware) + + url = post.call_args.args[0] + assert url == f"http://192.168.1.50:8080{OTA_PATH}" + assert OTA_PATH == "/update" + + +def test_run_ota_failure_response( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Failed!"), + ): + exit_code, host = run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + assert "OTA failure" in caplog.text + + +def test_run_ota_failure_response_empty_body( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, ""), + ): + exit_code, host = run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + assert "no response body" in caplog.text + + +def test_run_ota_auth_failed( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(401, "Unauthorized"), + ): + exit_code, host = run_ota(["192.168.1.50"], 80, "user", "wrong", firmware) + + assert exit_code == 1 + assert host is None + assert "Authentication failed" in caplog.text + + +def test_run_ota_unexpected_status_code( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(500, "Internal Error"), + ): + exit_code, host = run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + assert "Unexpected HTTP 500" in caplog.text + + +def test_run_ota_unexpected_status_empty_body_falls_back( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Empty response body uses response.reason / a fallback in the error.""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + response = _make_response(503, "") + response.reason = "Service Unavailable" + + with patch( + "esphome.web_server_ota.requests.post", + return_value=response, + ): + exit_code, host = run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + assert "Service Unavailable" in caplog.text + + +def test_run_ota_unexpected_status_no_body_no_reason( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Empty body and empty reason still produce a usable error message.""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + response = _make_response(599, "") + response.reason = "" + + with patch( + "esphome.web_server_ota.requests.post", + return_value=response, + ): + run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert "no response body" in caplog.text + + +def test_run_ota_connection_error_then_success( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """First resolved address fails to connect, second succeeds.""" + _patch_resolve( + monkeypatch, + [("192.168.1.10", 80), ("192.168.1.50", 80)], + ) + + with patch( + "esphome.web_server_ota.requests.post", + side_effect=[ + requests.ConnectionError("refused"), + _make_response(200, "Update Successful!"), + ], + ) as post: + exit_code, host = run_ota(["device.local"], 80, None, None, firmware) + + assert exit_code == 0 + assert host == "192.168.1.50" + assert post.call_count == 2 + + +def test_run_ota_request_exception_falls_through( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """A non-ConnectionError RequestException (e.g. timeout) falls through too.""" + _patch_resolve( + monkeypatch, + [("192.168.1.10", 80), ("192.168.1.50", 80)], + ) + + with patch( + "esphome.web_server_ota.requests.post", + side_effect=[ + requests.Timeout("read timeout"), + _make_response(200, "Update Successful!"), + ], + ): + exit_code, host = run_ota(["device.local"], 80, None, None, firmware) + + assert exit_code == 0 + assert host == "192.168.1.50" + + +def test_run_ota_all_addresses_unreachable( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """When every resolved address fails to connect, run_ota returns failure.""" + _patch_resolve( + monkeypatch, + [("192.168.1.10", 80), ("192.168.1.20", 80)], + ) + + with patch( + "esphome.web_server_ota.requests.post", + side_effect=requests.ConnectionError("refused"), + ): + exit_code, host = run_ota(["device.local"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + # Per-address failure is logged for each attempt; final summary follows. + assert caplog.text.count("OTA upload to ") >= 2 + assert "OTA upload failed." in caplog.text + + +def test_run_ota_no_resolved_addresses( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """If resolve_ip_address returns no candidates, log and return failure.""" + _patch_resolve(monkeypatch, []) + + exit_code, host = run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + assert "Could not resolve 192.168.1.50" in caplog.text + + +def test_run_ota_resolution_failure( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + def _raise(*_args, **_kwargs): + raise EsphomeError("dns failed") + + monkeypatch.setattr("esphome.web_server_ota.resolve_ip_address", _raise) + + exit_code, host = run_ota(["does.not.exist"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + + +def test_run_ota_resolution_failure_dashboard_mode( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Dashboard mode skips the '--device ' tip on resolution failure.""" + + def _raise(*_args, **_kwargs): + raise EsphomeError("dns failed") + + monkeypatch.setattr("esphome.web_server_ota.resolve_ip_address", _raise) + monkeypatch.setattr(CORE, "dashboard", True) + try: + exit_code, host = run_ota(["does.not.exist"], 80, None, None, firmware) + finally: + monkeypatch.setattr(CORE, "dashboard", False) + + assert exit_code == 1 + assert host is None + assert "--device " not in caplog.text + + +def test_run_ota_empty_hosts(firmware: Path) -> None: + exit_code, host = run_ota([], 80, None, None, firmware) + assert exit_code == 1 + assert host is None + + +def test_run_ota_string_host_accepted( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """A bare string is accepted in addition to a list of hosts.""" + _patch_resolve(monkeypatch, [("10.0.0.5", 80)]) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ): + exit_code, host = run_ota("10.0.0.5", 80, None, None, firmware) + + assert exit_code == 0 + assert host == "10.0.0.5" + + +def test_run_ota_multiple_hosts_first_fails( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """Multi-host fallthrough: first host's addresses all fail, second host wins.""" + addr_lookup = { + "primary.local": [ + (socket.AF_INET, socket.SOCK_STREAM, 0, "", ("192.168.1.10", 80)), + ], + "secondary.local": [ + (socket.AF_INET, socket.SOCK_STREAM, 0, "", ("192.168.1.50", 80)), + ], + } + + def _resolve(host, port, address_cache=None): # noqa: ARG001 + return addr_lookup[host] + + monkeypatch.setattr("esphome.web_server_ota.resolve_ip_address", _resolve) + + with patch( + "esphome.web_server_ota.requests.post", + side_effect=[ + requests.ConnectionError("refused"), + _make_response(200, "Update Successful!"), + ], + ): + exit_code, host = run_ota( + ["primary.local", "secondary.local"], 80, None, None, firmware + ) + + assert exit_code == 0 + assert host == "192.168.1.50" + + +def test_run_ota_all_hosts_return_failure_no_exception( + monkeypatch: pytest.MonkeyPatch, firmware: Path, caplog: pytest.LogCaptureFixture +) -> None: + """All hosts resolve to no addresses; run_ota cleanly returns failure.""" + addr_lookup = { + "a.local": [], + "b.local": [], + } + + def _resolve(host, port, address_cache=None): # noqa: ARG001 + return addr_lookup[host] + + monkeypatch.setattr("esphome.web_server_ota.resolve_ip_address", _resolve) + + exit_code, host = run_ota(["a.local", "b.local"], 80, None, None, firmware) + + assert exit_code == 1 + assert host is None + # Each host gets its own "Could not resolve" log line + final summary. + assert caplog.text.count("Could not resolve") == 2 + assert "OTA upload failed." in caplog.text + + +def test_web_server_ota_error_is_esphome_error() -> None: + assert issubclass(WebServerOTAError, EsphomeError) + + +def test_run_ota_finalizes_progress_bar_on_success( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """progress.done() fires on the success path (finally block).""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + done_called: list[bool] = [] + + with ( + patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ), + patch.object(ProgressBar, "done", lambda self: done_called.append(True)), + ): + run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert done_called + + +def test_run_ota_finalizes_progress_bar_on_failure( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """progress.done() fires when the request itself raises (finally block).""" + _patch_resolve(monkeypatch, [("192.168.1.50", 80)]) + + done_called: list[bool] = [] + + with ( + patch( + "esphome.web_server_ota.requests.post", + side_effect=requests.ConnectionError("boom"), + ), + patch.object(ProgressBar, "done", lambda self: done_called.append(True)), + ): + run_ota(["192.168.1.50"], 80, None, None, firmware) + + assert done_called + + +def test_run_ota_ipv6_url_brackets_host( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """IPv6 candidates are bracketed in the URL so the port parses correctly.""" + addr_infos = [ + (socket.AF_INET6, socket.SOCK_STREAM, 0, "", ("2001:db8::1", 80, 0, 0)), + ] + monkeypatch.setattr( + "esphome.web_server_ota.resolve_ip_address", lambda *a, **kw: addr_infos + ) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + exit_code, host = run_ota(["device.local"], 80, None, None, firmware) + + assert exit_code == 0 + assert host == "2001:db8::1" + url = post.call_args.args[0] + assert url == f"http://[2001:db8::1]:80{OTA_PATH}" + + +def test_run_ota_ipv6_link_local_includes_scope_id( + monkeypatch: pytest.MonkeyPatch, firmware: Path +) -> None: + """Link-local IPv6 candidates include the percent-encoded zone index.""" + addr_infos = [ + (socket.AF_INET6, socket.SOCK_STREAM, 0, "", ("fe80::1", 80, 0, 3)), + ] + monkeypatch.setattr( + "esphome.web_server_ota.resolve_ip_address", lambda *a, **kw: addr_infos + ) + + with patch( + "esphome.web_server_ota.requests.post", + return_value=_make_response(200, "Update Successful!"), + ) as post: + exit_code, _ = run_ota(["device.local"], 80, None, None, firmware) + + assert exit_code == 0 + url = post.call_args.args[0] + assert url == f"http://[fe80::1%253]:80{OTA_PATH}" From 39b2b901f7ae3cd6000a46bafaa56358a1f739bd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:26:19 -0500 Subject: [PATCH 23/60] [core] Replace scheduler pool vector with unbounded intrusive freelist (#16172) --- esphome/core/application.cpp | 7 ++ esphome/core/scheduler.cpp | 97 ++++++++++++------- esphome/core/scheduler.h | 34 ++++--- tests/benchmarks/core/bench_scheduler.cpp | 10 +- .../integration/fixtures/scheduler_pool.yaml | 10 +- tests/integration/test_scheduler_pool.py | 24 +++-- 6 files changed, 115 insertions(+), 67 deletions(-) diff --git a/esphome/core/application.cpp b/esphome/core/application.cpp index d03696fbb6..38d3503c2c 100644 --- a/esphome/core/application.cpp +++ b/esphome/core/application.cpp @@ -25,6 +25,10 @@ namespace esphome { static const char *const TAG = "app"; +// Delay after setup() finishes before trimming the scheduler freelist of its post-boot peak. +// 10 s is well past the bulk of post-setup async work (Wi-Fi/MQTT connects, first-read latency). +static constexpr uint32_t SCHEDULER_FREELIST_TRIM_DELAY_MS = 10000; + // Helper function for insertion sort of components by priority // Using insertion sort instead of std::stable_sort saves ~1.3KB of flash // by avoiding template instantiations (std::rotate, std::stable_sort, lambdas) @@ -112,6 +116,9 @@ void Application::setup() { ESP_LOGI(TAG, "setup() finished successfully!"); + // Trim the scheduler freelist of its post-boot peak once startup churn settles. + this->scheduler.set_timeout(this, SCHEDULER_FREELIST_TRIM_DELAY_MS, [this]() { this->scheduler.trim_freelist(); }); + #ifdef USE_SETUP_PRIORITY_OVERRIDE // Clear setup priority overrides to free memory clear_setup_priority_overrides(); diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 57deeab0da..a7c624486d 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -14,18 +14,8 @@ namespace esphome { static const char *const TAG = "scheduler"; -// Memory pool configuration constants -// Pool size of 5 matches typical usage patterns (2-4 active timers) -// - Minimal memory overhead (~250 bytes on ESP32) -// - Sufficient for most configs with a couple sensors/components -// - Still prevents heap fragmentation and allocation stalls -// - Complex setups with many timers will just allocate beyond the pool -// See https://github.com/esphome/backlog/issues/52 -static constexpr size_t MAX_POOL_SIZE = 5; - // Maximum number of logically deleted (cancelled) items before forcing cleanup. -// Set to 5 to match the pool size - when we have as many cancelled items as our -// pool can hold, it's time to clean up and recycle them. +// Empirically chosen to balance cleanup overhead against tombstone accumulation in items_. static constexpr uint32_t MAX_LOGICALLY_DELETED_ITEMS = 5; // max delay to start an interval sequence static constexpr uint32_t MAX_INTERVAL_DELAY = 5000; @@ -165,7 +155,7 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type delay = 1; } - // Take lock early to protect scheduler_item_pool_ access and retry-cancelled check + // Take lock early to protect scheduler_item_pool_head_ access and retry-cancelled check LockGuard guard{this->lock_}; // For retries, check if there's a cancelled timeout first - before allocating an item. @@ -599,7 +589,7 @@ uint32_t HOT Scheduler::call(uint32_t now) { if (now_64 - last_print > 2000) { last_print = now_64; std::vector old_items; - ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64, this->items_.size(), this->scheduler_item_pool_.size(), + ESP_LOGD(TAG, "Items: count=%zu, pool=%zu, now=%" PRIu64, this->items_.size(), this->scheduler_item_pool_size_, now_64); // Cleanup before debug output this->cleanup_(); @@ -894,30 +884,68 @@ bool HOT Scheduler::SchedulerItem::cmp(SchedulerItem *a, SchedulerItem *b) { : (a->next_execution_high_ > b->next_execution_high_); } -// Recycle a SchedulerItem back to the pool for reuse. -// IMPORTANT: Caller must hold the scheduler lock before calling this function. -// This protects scheduler_item_pool_ from concurrent access by other threads -// that may be acquiring items from the pool in set_timer_common_(). +// Recycle a SchedulerItem back to the freelist for reuse. +// IMPORTANT: Caller must hold the scheduler lock. void Scheduler::recycle_item_main_loop_(SchedulerItem *item) { if (item == nullptr) return; - if (this->scheduler_item_pool_.size() < MAX_POOL_SIZE) { - // Clear callback to release captured resources - item->callback = nullptr; - this->scheduler_item_pool_.push_back(item); + item->callback = nullptr; // release captured resources + item->next_free = this->scheduler_item_pool_head_; + this->scheduler_item_pool_head_ = item; + this->scheduler_item_pool_size_++; #ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Recycled item to pool (pool size now: %zu)", this->scheduler_item_pool_.size()); -#endif - } else { -#ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Pool full (size: %zu), deleting item", this->scheduler_item_pool_.size()); + ESP_LOGD(TAG, "Recycled item to pool (pool size now: %zu)", this->scheduler_item_pool_size_); #endif +} + +// Shrink a SchedulerItem* vector's capacity to its current size. +// std::vector::shrink_to_fit() is non-binding and our toolchain ignores it; the classic +// swap-with-copy idiom (std::vector(other).swap(other)) instantiates the iterator-range +// constructor which pulls in std::__throw_bad_array_new_length and ~120 B of related +// stdlib RTTI/typeinfo. Build into a temp via reserve + push_back instead, then move-assign: +// reserve uses operator new (throws bad_alloc, already linked) and push_back without growth +// is the noexcept tail path. Move-assign just swaps pointers. +// Out-of-line + noinline so the callers in trim_freelist() share one body. +void __attribute__((noinline)) Scheduler::shrink_scheduler_vector_(std::vector *v) { + if (v->capacity() == v->size()) + return; // already exact, common after a quiet period + std::vector tmp; + tmp.reserve(v->size()); + for (SchedulerItem *p : *v) + tmp.push_back(p); + *v = std::move(tmp); +} + +void Scheduler::trim_freelist() { + LockGuard guard{this->lock_}; + SchedulerItem *item = this->scheduler_item_pool_head_; + size_t freed = 0; + while (item != nullptr) { + SchedulerItem *next = item->next_free; delete item; #ifdef ESPHOME_DEBUG_SCHEDULER this->debug_live_items_--; #endif + item = next; + freed++; } + this->scheduler_item_pool_head_ = nullptr; + this->scheduler_item_pool_size_ = 0; + + // items_/to_add_/defer_queue_ retain their boot-peak vector capacity (vector grows + // by doubling and otherwise keeps the peak). Reclaim that slack as well. + shrink_scheduler_vector_(&this->items_); + shrink_scheduler_vector_(&this->to_add_); +#ifndef ESPHOME_THREAD_SINGLE + shrink_scheduler_vector_(&this->defer_queue_); +#endif + +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Freelist trimmed (%zu items freed)", freed); +#else + (void) freed; +#endif } #ifdef ESPHOME_DEBUG_SCHEDULER @@ -942,14 +970,15 @@ void Scheduler::debug_log_timer_(const SchedulerItem *item, NameType name_type, } #endif /* ESPHOME_DEBUG_SCHEDULER */ -// Helper to get or create a scheduler item from the pool -// IMPORTANT: Caller must hold the scheduler lock before calling this function. +// Pop from freelist or allocate. IMPORTANT: caller must hold the lock and must overwrite +// `item->component` before releasing it -- the popped slot still holds the freelist link. Scheduler::SchedulerItem *Scheduler::get_item_from_pool_locked_() { - if (!this->scheduler_item_pool_.empty()) { - SchedulerItem *item = this->scheduler_item_pool_.back(); - this->scheduler_item_pool_.pop_back(); + if (this->scheduler_item_pool_head_ != nullptr) { + SchedulerItem *item = this->scheduler_item_pool_head_; + this->scheduler_item_pool_head_ = item->next_free; + this->scheduler_item_pool_size_--; #ifdef ESPHOME_DEBUG_SCHEDULER - ESP_LOGD(TAG, "Reused item from pool (pool size now: %zu)", this->scheduler_item_pool_.size()); + ESP_LOGD(TAG, "Reused item from pool (pool size now: %zu)", this->scheduler_item_pool_size_); #endif return item; } @@ -967,7 +996,7 @@ Scheduler::SchedulerItem *Scheduler::get_item_from_pool_locked_() { bool Scheduler::debug_verify_no_leak_() const { // Invariant: every live SchedulerItem must be in exactly one container. // debug_live_items_ tracks allocations minus deletions. - size_t accounted = this->items_.size() + this->to_add_.size() + this->scheduler_item_pool_.size(); + size_t accounted = this->items_.size() + this->to_add_.size() + this->scheduler_item_pool_size_; #ifndef ESPHOME_THREAD_SINGLE accounted += this->defer_queue_.size(); #endif @@ -981,7 +1010,7 @@ bool Scheduler::debug_verify_no_leak_() const { ")", static_cast(this->debug_live_items_), static_cast(accounted), static_cast(this->items_.size()), static_cast(this->to_add_.size()), - static_cast(this->scheduler_item_pool_.size()) + static_cast(this->scheduler_item_pool_size_) #ifndef ESPHOME_THREAD_SINGLE , static_cast(this->defer_queue_.size()) diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 7a6be6bea9..b640aa86fe 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -132,6 +132,12 @@ class Scheduler { // @return Timestamp of the last item that ran, or `now` unchanged if none ran. uint32_t call(uint32_t now); + // Reclaim memory held by the post-boot peak. Frees every SchedulerItem in the + // recycle freelist and shrinks items_/to_add_/defer_queue_ vector capacity to + // their current sizes (std::vector grows by doubling and otherwise retains the + // peak). Live items in those vectors are preserved. + void trim_freelist(); + // Move items from to_add_ into the main heap. // IMPORTANT: This method should only be called from the main thread (loop task). // Inlined: the fast path (nothing to add) is just an atomic load / empty check. @@ -177,8 +183,12 @@ class Scheduler { protected: struct SchedulerItem { - // Ordered by size to minimize padding - Component *component; + // Ordered by size to minimize padding. + // `component` while live; `next_free` while in scheduler_item_pool_head_ (mutually exclusive). + union { + Component *component; + SchedulerItem *next_free; + }; // Optimized name storage using tagged union - zero heap allocation union { const char *static_name; // For STATIC_STRING (string literals) and SELF_POINTER (caller's `this`) @@ -355,6 +365,10 @@ class Scheduler { SchedulerItem *get_item_from_pool_locked_(); private: + // Out-of-line helper that shrinks a SchedulerItem* vector's capacity to its current + // size. Centralised so trim_freelist() doesn't pay flash cost per call site. + void shrink_scheduler_vector_(std::vector *v); + // Helper to cancel matching items - must be called with lock held. // When find_first=true, stops after the first match (used by set_timer_common_ where // the cancel-before-add invariant guarantees at most one match). @@ -713,19 +727,15 @@ class Scheduler { #endif } - // Memory pool for recycling SchedulerItem objects to reduce heap churn. - // Design decisions: - // - std::vector is used instead of a fixed array because many systems only need 1-2 scheduler items - // - The vector grows dynamically up to MAX_POOL_SIZE (5) only when needed, saving memory on simple setups - // - Pool size of 5 matches typical usage (2-4 timers) while keeping memory overhead low (~250 bytes on ESP32) - // - The pool significantly reduces heap fragmentation which is critical because heap allocation/deallocation - // can stall the entire system, causing timing issues and dropped events for any components that need - // to synchronize between tasks (see https://github.com/esphome/backlog/issues/52) - std::vector scheduler_item_pool_; + // Intrusive freelist threaded through SchedulerItem::next_free. Unbounded so it quiesces at the + // app's concurrent-timer high-water mark; the previous fixed cap caused steady-state new/delete + // churn on devices with many timers (see https://github.com/esphome/backlog/issues/52). + SchedulerItem *scheduler_item_pool_head_{nullptr}; + size_t scheduler_item_pool_size_{0}; #ifdef ESPHOME_DEBUG_SCHEDULER // Leak detection: tracks total live SchedulerItem allocations. - // Invariant: debug_live_items_ == items_.size() + to_add_.size() + defer_queue_.size() + scheduler_item_pool_.size() + // Invariant: debug_live_items_ == items_.size() + to_add_.size() + defer_queue_.size() + scheduler_item_pool_size_ // Verified periodically in call() to catch leaks early. size_t debug_live_items_{0}; diff --git a/tests/benchmarks/core/bench_scheduler.cpp b/tests/benchmarks/core/bench_scheduler.cpp index 214fe0e4b8..32bbc2de88 100644 --- a/tests/benchmarks/core/bench_scheduler.cpp +++ b/tests/benchmarks/core/bench_scheduler.cpp @@ -101,8 +101,8 @@ static void Scheduler_SetTimeout(benchmark::State &state) { Component dummy_component; // Register 3 timeouts then call() — realistic worst case where multiple - // components schedule in the same loop iteration. Keeps item count within - // the recycling pool (MAX_POOL_SIZE=5) to avoid spurious malloc/free. + // components schedule in the same loop iteration. warm_pool fills the + // freelist so acquire/recycle never falls back to malloc. static constexpr int kBatchSize = 3; static_assert(kInnerIterations % kBatchSize == 0, "kInnerIterations must be divisible by kBatchSize"); warm_pool(scheduler, &dummy_component, kBatchSize, 1000); @@ -209,9 +209,9 @@ static void Scheduler_SetTimeout_ExceedPool(benchmark::State &state) { Scheduler scheduler; Component dummy_component; - // Register 10 timeouts then call() — exceeds MAX_POOL_SIZE=5 to measure - // the performance cliff when the recycling pool is exhausted and items - // must be malloc'd/freed. + // Register 10 timeouts then call() — larger working set than the 3-item + // batches above. With the unbounded freelist, warm_pool preallocates 10 + // items so this measures steady-state, not malloc cliff. static constexpr int kBatchSize = 10; static_assert(kInnerIterations % kBatchSize == 0, "kInnerIterations must be divisible by kBatchSize"); warm_pool(scheduler, &dummy_component, kBatchSize, 1000); diff --git a/tests/integration/fixtures/scheduler_pool.yaml b/tests/integration/fixtures/scheduler_pool.yaml index 5389125188..989c1535b0 100644 --- a/tests/integration/fixtures/scheduler_pool.yaml +++ b/tests/integration/fixtures/scheduler_pool.yaml @@ -221,14 +221,10 @@ script: - id: test_full_pool_reuse then: - lambda: |- - ESP_LOGI("test", "Phase 6: Testing pool size limits after Phase 5 items complete"); + ESP_LOGI("test", "Phase 6: Testing pool reuse after Phase 5 items complete"); - // At this point, all Phase 5 timeouts should have completed and been recycled. - // The pool should be at its maximum size (5). - // Creating 10 new items tests that: - // - First 5 items reuse from the pool - // - Remaining 5 items allocate new (pool empty) - // - Pool doesn't grow beyond MAX_POOL_SIZE of 5 + // Phase 5 timeouts have completed and been recycled. The freelist is unbounded; + // creating 10 new items reuses from it and only allocates fresh when empty. auto *component = id(test_sensor); int full_reuse_count = 10; diff --git a/tests/integration/test_scheduler_pool.py b/tests/integration/test_scheduler_pool.py index 021917cc25..cc25190e30 100644 --- a/tests/integration/test_scheduler_pool.py +++ b/tests/integration/test_scheduler_pool.py @@ -180,16 +180,22 @@ async def test_scheduler_pool( # Verify pool behavior assert pool_recycle_count > 0, "Should have recycled items to pool" - # Check pool metrics - if pool_recycle_count > 0: - max_pool_size = 0 - for line in log_lines: - if match := recycle_pattern.search(line): - size = int(match.group(1)) - max_pool_size = max(max_pool_size, size) + # Pool is unbounded; the cap was the source of the churn it was meant to prevent. + assert pool_full_count == 0, ( + f"Pool should never report full (got {pool_full_count})" + ) - # Pool can grow up to its maximum of 5 - assert max_pool_size <= 5, f"Pool grew beyond maximum ({max_pool_size})" + # Verify the pool actually grew past the old MAX_POOL_SIZE=5 cap. + # Phase 5 + Phase 6 schedule 8 + 10 same-component timeouts respectively, so the + # observed peak should comfortably exceed 5. Without this lower-bound check, a + # silent regression that re-introduced a small cap could pass the test above. + max_pool_size = 0 + for line in log_lines: + if match := recycle_pattern.search(line): + max_pool_size = max(max_pool_size, int(match.group(1))) + assert max_pool_size > 5, ( + f"Pool should grow past the old cap of 5; observed peak {max_pool_size}" + ) # Log summary for debugging print("\nScheduler Pool Test Summary (Python Orchestrated):") From 67491c3194067a38b9463d24f7bac9cd64ea9c76 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:26:52 -0500 Subject: [PATCH 24/60] [packages] Add resolve_packages single-call seam (#16235) --- esphome/components/packages/__init__.py | 59 +++++++++ .../component_tests/packages/test_packages.py | 120 ++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 47a1fd20a7..d63f17aa7e 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -611,3 +611,62 @@ def merge_packages(config: dict) -> dict: config = reduce(lambda new, old: merge_config(old, new), merge_list, config) del config[CONF_PACKAGES] return config + + +def resolve_packages( + config: dict[str, Any], + *, + command_line_substitutions: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Load and merge ``packages:`` in one call; return the flattened config. + + Convenience wrapper around :func:`do_packages_pass` followed by + :func:`merge_packages`. External tools that want the package- + merged dict (without going through full schema validation via + :func:`esphome.config.read_config`) get one stable seam to call + instead of having to chain the two functions and stay in sync + with the pipeline order. + + Note: the full :func:`esphome.config.validate_config` pipeline + runs two extra passes around the merge that this wrapper + deliberately skips: + + 1. :func:`esphome.components.substitutions.do_substitution_pass` + runs BETWEEN :func:`do_packages_pass` and + :func:`merge_packages`, so ``${var}`` placeholders inside + package content are NOT resolved here. Callers that need + substitution should invoke ``do_substitution_pass`` + themselves between calls, or go through the full + ``validate_config``. + 2. :func:`esphome.config.resolve_extend_remove` runs AFTER + :func:`merge_packages`, so top-level ``!remove`` / ``!extend`` + markers are NOT applied here. A package-contributed block + paired with a top-level ``key: !remove`` will still appear + in the returned dict (the marker just sits next to it). + + The wrapper exists for the "what blocks did packages + contribute?" question — metadata callers that just need to + see merged top-level keys. It is NOT a stand-in for + :func:`esphome.config.validate_config` and the two passes + above are the reasons why. + + Used by: + + - ``esphome/device-builder`` — the new WebSocket dashboard + backend reads device metadata (api / wifi / target-platform + flags) off the merged config so packages contribute the same + blocks the compiler sees, not just whatever sits at the top + of the user's YAML. See + https://github.com/esphome/device-builder/issues/288 for the + bug this fixes. + + Returns *config* unchanged when ``packages:`` isn't present, so + callers can apply this unconditionally without having to peek + at the config first. + """ + if CONF_PACKAGES not in config: + return config + config = do_packages_pass( + config, command_line_substitutions=command_line_substitutions + ) + return merge_packages(config) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 13a6da9f2c..8c809c5e91 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -14,6 +14,7 @@ from esphome.components.packages import ( do_packages_pass, is_package_definition, merge_packages, + resolve_packages, ) from esphome.components.substitutions import ContextVars, do_substitution_pass import esphome.config as config_module @@ -1621,3 +1622,122 @@ def test_remote_package_vars_resolved_against_sibling_package_substitutions( actual = packages_pass(config) assert actual[CONF_SENSOR][0]["pin"] == "GPIO5" + + +# --------------------------------------------------------------------------- +# resolve_packages — single-call wrapper around do_packages_pass + merge_packages +# --------------------------------------------------------------------------- + + +def test_resolve_packages_returns_config_unchanged_without_packages() -> None: + """No ``packages:`` key → no-op, same dict back.""" + config = {CONF_ESPHOME: {CONF_NAME: "test"}, CONF_WIFI: {CONF_SSID: "x"}} + result = resolve_packages(config) + assert result is config + assert CONF_PACKAGES not in result + + +def test_resolve_packages_loads_and_merges_in_one_call() -> None: + """End-to-end: a config with one local-dict package gets its blocks flattened.""" + config = { + CONF_ESPHOME: {CONF_NAME: "main"}, + CONF_PACKAGES: { + "shared": { + CONF_WIFI: {CONF_SSID: "from_package"}, + CONF_SENSOR: [ + {CONF_PLATFORM: "template", CONF_NAME: "from_package_sensor"}, + ], + } + }, + } + result = resolve_packages(config) + # ``packages:`` is gone — it was consumed by the merge. + assert CONF_PACKAGES not in result + # Blocks contributed by the package are now top-level. + assert result[CONF_WIFI][CONF_SSID] == "from_package" + assert result[CONF_SENSOR][0][CONF_NAME] == "from_package_sensor" + # The main config's own keys survive untouched. + assert result[CONF_ESPHOME][CONF_NAME] == "main" + + +def test_resolve_packages_preserves_main_config_overrides() -> None: + """Main-config values win over package values for the same key. + + Pinning the precedence ESPHome's compiler uses so any future + refactor of the wrapper doesn't accidentally flip the order. + """ + config = { + CONF_ESPHOME: {CONF_NAME: "main"}, + CONF_WIFI: {CONF_SSID: "main_wins"}, + CONF_PACKAGES: { + "shared": {CONF_WIFI: {CONF_SSID: "package_loses"}}, + }, + } + result = resolve_packages(config) + assert result[CONF_WIFI][CONF_SSID] == "main_wins" + + +def test_resolve_packages_forwards_command_line_substitutions() -> None: + """``command_line_substitutions`` reaches the underlying ``do_packages_pass``. + + The wrapper exists so external tools have one stable seam; if + that seam silently dropped a kwarg the underlying call accepts, + callers would see surprising behaviour. This pins the + pass-through. + """ + config = { + CONF_ESPHOME: {CONF_NAME: "main"}, + CONF_PACKAGES: {"shared": {CONF_WIFI: {CONF_SSID: "from_package"}}}, + } + with patch( + "esphome.components.packages.do_packages_pass", + wraps=do_packages_pass, + ) as spy: + resolve_packages(config, command_line_substitutions={"foo": "bar"}) + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("command_line_substitutions") == {"foo": "bar"} + + +def test_resolve_packages_does_not_run_substitutions() -> None: + """``${var}`` placeholders inside package content stay literal. + + The full ``validate_config`` pipeline runs ``do_substitution_pass`` + BETWEEN ``do_packages_pass`` and ``merge_packages``; this wrapper + skips it on purpose. Pin that contract so a future refactor can't + silently start resolving substitutions and break callers that + deliberately compose the passes themselves. + """ + config = { + CONF_ESPHOME: {CONF_NAME: "main"}, + CONF_SUBSTITUTIONS: {"ssid_value": "resolved_ssid"}, + CONF_PACKAGES: { + "shared": {CONF_WIFI: {CONF_SSID: "${ssid_value}"}}, + }, + } + result = resolve_packages(config) + # Without ``do_substitution_pass`` the placeholder is preserved. + assert result[CONF_WIFI][CONF_SSID] == "${ssid_value}" + + +def test_resolve_packages_does_not_apply_extend_remove() -> None: + """Top-level ``!remove`` / ``!extend`` markers stay in the merged dict. + + The full ``validate_config`` pipeline runs ``resolve_extend_remove`` + AFTER ``merge_packages``; this wrapper skips it on purpose. Pin + that contract: a package-contributed block paired with a top-level + ``!remove`` is left as-is for callers to handle (or for them to + call ``resolve_extend_remove`` themselves). + """ + config = { + CONF_ESPHOME: {CONF_NAME: "main"}, + CONF_WIFI: Remove(), + CONF_PACKAGES: { + "shared": {CONF_WIFI: {CONF_SSID: "from_package"}}, + }, + } + result = resolve_packages(config) + # ``merge_packages`` keeps the top-level ``!remove`` (it wins + # over the package value during merge), and the marker is not + # resolved by this wrapper. + assert isinstance(result[CONF_WIFI], Remove) From 4404dd68ba56d3c041b32cd8494bb386e3545134 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:27:18 -0500 Subject: [PATCH 25/60] [cover] Fix ControlAction / CoverPublishAction trigger args with reference types (#16227) --- esphome/components/cover/__init__.py | 19 +++++++++++++++---- esphome/components/cover/automation.h | 11 +++++++++-- tests/components/template/common-base.yaml | 13 +++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/esphome/components/cover/__init__.py b/esphome/components/cover/__init__.py index 954ad7a345..839ca532e6 100644 --- a/esphome/components/cover/__init__.py +++ b/esphome/components/cover/__init__.py @@ -328,17 +328,28 @@ async def build_apply_lambda_action( Used by both `cover.control` and `cover.template.publish` (and shared with the template/cover platform). Constants are emitted as flash immediates; user lambdas are invoked inline so trigger args still flow. - The trigger arg types are wrapped as `const T &` to match the - `void (*)(..., const Ts &...)` ApplyFn signature. + Trigger arg types are normalized to `const std::remove_cvref_t &` + to match the ApplyFn signature for any T (value, ref, or const-ref). """ paren = await cg.get_variable(config[CONF_ID]) + # Normalize trigger args to `const std::remove_cvref_t &` so the + # apply lambda and any inner field lambdas (generated below via + # `process_lambda`) share one parameter spelling that's well-formed for + # any T. + normalized_args = [ + (cg.RawExpression(f"const std::remove_cvref_t<{cg.safe_exp(t)}> &"), n) + for t, n in args + ] + fwd_args = ", ".join(name for _, name in args) body_lines: list[str] = [] for field in fields: if (value := config.get(field.conf_key)) is None: continue if isinstance(value, Lambda): - inner = await cg.process_lambda(value, args, return_type=field.type_) + inner = await cg.process_lambda( + value, normalized_args, return_type=field.type_ + ) value_expr = f"({inner})({fwd_args})" else: value_expr = str(cg.safe_exp(value)) @@ -346,7 +357,7 @@ async def build_apply_lambda_action( apply_args = [ *prefix_args, - *((t.operator("const").operator("ref"), n) for t, n in args), + *normalized_args, ] apply_lambda = LambdaExpression( ["\n".join(body_lines)], diff --git a/esphome/components/cover/automation.h b/esphome/components/cover/automation.h index e2384c2359..ee7a4f5f76 100644 --- a/esphome/components/cover/automation.h +++ b/esphome/components/cover/automation.h @@ -51,10 +51,17 @@ template class ToggleAction : public Action { // plus one parent pointer, regardless of how many fields the user set. // Trigger args are forwarded to the apply function so user lambdas // (e.g. `position: !lambda "return x;"`) keep working. +// +// Trigger args are normalized to `const std::remove_cvref_t &...` so +// the codegen can emit a matching parameter list for both the apply lambda +// and any inner field lambdas without producing invalid C++ source text +// (e.g. `const T & &` if Ts already carries a reference, or `const const +// T &` if Ts already carries a const). This keeps trigger args no-copy +// regardless of whether the trigger supplies `T`, `T &`, or `const T &`. template class ControlAction : public Action { public: - using ApplyFn = void (*)(CoverCall &, const Ts &...); + using ApplyFn = void (*)(CoverCall &, const std::remove_cvref_t &...); ControlAction(Cover *cover, ApplyFn apply) : cover_(cover), apply_(apply) {} void play(const Ts &...x) override { @@ -70,7 +77,7 @@ template class ControlAction : public Action { template class CoverPublishAction : public Action { public: - using ApplyFn = void (*)(Cover *, const Ts &...); + using ApplyFn = void (*)(Cover *, const std::remove_cvref_t &...); CoverPublishAction(Cover *cover, ApplyFn apply) : cover_(cover), apply_(apply) {} void play(const Ts &...x) override { diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 984ef129ad..b97cafd25c 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -369,6 +369,19 @@ number: - valve.control: id: template_valve position: !lambda "return x / 100.0f;" + # Same regression test for cover.control: forces the apply-lambda + # codegen to handle a non-empty trigger Ts (float). + - platform: template + id: template_cover_position_number + optimistic: true + min_value: 0 + max_value: 100 + step: 1 + on_value: + then: + - cover.control: + id: template_cover_with_triggers + position: !lambda "return x / 100.0f;" select: - platform: template From f5c1b8839da0fdea431d11542dfd6f6bebe5332a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:29:10 -0500 Subject: [PATCH 26/60] [web_server] Use entity_types.h X-macro for ListEntitiesIterator declarations (#16077) --- .../components/web_server/list_entities.cpp | 4 + esphome/components/web_server/list_entities.h | 83 +++---------------- 2 files changed, 15 insertions(+), 72 deletions(-) diff --git a/esphome/components/web_server/list_entities.cpp b/esphome/components/web_server/list_entities.cpp index c1e7599c7e..869ed3ea17 100644 --- a/esphome/components/web_server/list_entities.cpp +++ b/esphome/components/web_server/list_entities.cpp @@ -167,5 +167,9 @@ bool ListEntitiesIterator::on_update(update::UpdateEntity *obj) { } #endif +#ifdef USE_MEDIA_PLAYER +bool ListEntitiesIterator::on_media_player(media_player::MediaPlayer *obj) { return true; } +#endif + } // namespace esphome::web_server #endif diff --git a/esphome/components/web_server/list_entities.h b/esphome/components/web_server/list_entities.h index 9cfc6c7e33..3edb84f555 100644 --- a/esphome/components/web_server/list_entities.h +++ b/esphome/components/web_server/list_entities.h @@ -24,78 +24,17 @@ class ListEntitiesIterator final : public ComponentIterator { #elif defined(USE_ARDUINO) ListEntitiesIterator(const WebServer *ws, DeferredUpdateEventSource *es); #endif -#ifdef USE_BINARY_SENSOR - bool on_binary_sensor(binary_sensor::BinarySensor *obj) override; -#endif -#ifdef USE_COVER - bool on_cover(cover::Cover *obj) override; -#endif -#ifdef USE_FAN - bool on_fan(fan::Fan *obj) override; -#endif -#ifdef USE_LIGHT - bool on_light(light::LightState *obj) override; -#endif -#ifdef USE_SENSOR - bool on_sensor(sensor::Sensor *obj) override; -#endif -#ifdef USE_SWITCH - bool on_switch(switch_::Switch *obj) override; -#endif -#ifdef USE_BUTTON - bool on_button(button::Button *obj) override; -#endif -#ifdef USE_TEXT_SENSOR - bool on_text_sensor(text_sensor::TextSensor *obj) override; -#endif -#ifdef USE_CLIMATE - bool on_climate(climate::Climate *obj) override; -#endif -#ifdef USE_NUMBER - bool on_number(number::Number *obj) override; -#endif -#ifdef USE_DATETIME_DATE - bool on_date(datetime::DateEntity *obj) override; -#endif -#ifdef USE_DATETIME_TIME - bool on_time(datetime::TimeEntity *obj) override; -#endif -#ifdef USE_DATETIME_DATETIME - bool on_datetime(datetime::DateTimeEntity *obj) override; -#endif -#ifdef USE_TEXT - bool on_text(text::Text *obj) override; -#endif -#ifdef USE_SELECT - bool on_select(select::Select *obj) override; -#endif -#ifdef USE_LOCK - bool on_lock(lock::Lock *obj) override; -#endif -#ifdef USE_VALVE - bool on_valve(valve::Valve *obj) override; -#endif -#ifdef USE_MEDIA_PLAYER - bool on_media_player(media_player::MediaPlayer *obj) override { return true; } -#endif -#ifdef USE_ALARM_CONTROL_PANEL - bool on_alarm_control_panel(alarm_control_panel::AlarmControlPanel *obj) override; -#endif -#ifdef USE_WATER_HEATER - bool on_water_heater(water_heater::WaterHeater *obj) override; -#endif -#ifdef USE_INFRARED - bool on_infrared(infrared::Infrared *obj) override; -#endif -#ifdef USE_RADIO_FREQUENCY - bool on_radio_frequency(radio_frequency::RadioFrequency *obj) override; -#endif -#ifdef USE_EVENT - bool on_event(event::Event *obj) override; -#endif -#ifdef USE_UPDATE - bool on_update(update::UpdateEntity *obj) override; -#endif + +// Entity overrides (generated from entity_types.h). +// Implementations live in list_entities.cpp. +// NOLINTBEGIN(bugprone-macro-parentheses) +#define ENTITY_TYPE_(type, singular, plural, count, upper) bool on_##singular(type *obj) override; +#define ENTITY_CONTROLLER_TYPE_(type, singular, plural, count, upper, callback) \ + ENTITY_TYPE_(type, singular, plural, count, upper) +#include "esphome/core/entity_types.h" +#undef ENTITY_TYPE_ +#undef ENTITY_CONTROLLER_TYPE_ + // NOLINTEND(bugprone-macro-parentheses) bool completed() { return this->state_ == IteratorState::NONE; } protected: From bf1c339dc1ff83b34ece07e45b40de3239eb149e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:29:32 -0500 Subject: [PATCH 27/60] [api] Use entity_types.h X-macro for ListEntitiesIterator declarations (#16076) --- esphome/components/api/list_entities.h | 83 ++++---------------------- 1 file changed, 11 insertions(+), 72 deletions(-) diff --git a/esphome/components/api/list_entities.h b/esphome/components/api/list_entities.h index 95c626feb1..88fbdb77c8 100644 --- a/esphome/components/api/list_entities.h +++ b/esphome/components/api/list_entities.h @@ -18,83 +18,22 @@ class APIConnection; class ListEntitiesIterator final : public ComponentIterator { public: ListEntitiesIterator(APIConnection *client); -#ifdef USE_BINARY_SENSOR - bool on_binary_sensor(binary_sensor::BinarySensor *entity) override; -#endif -#ifdef USE_COVER - bool on_cover(cover::Cover *entity) override; -#endif -#ifdef USE_FAN - bool on_fan(fan::Fan *entity) override; -#endif -#ifdef USE_LIGHT - bool on_light(light::LightState *entity) override; -#endif -#ifdef USE_SENSOR - bool on_sensor(sensor::Sensor *entity) override; -#endif -#ifdef USE_SWITCH - bool on_switch(switch_::Switch *entity) override; -#endif -#ifdef USE_BUTTON - bool on_button(button::Button *entity) override; -#endif -#ifdef USE_TEXT_SENSOR - bool on_text_sensor(text_sensor::TextSensor *entity) override; -#endif + +// Entity overrides (generated from entity_types.h). +// All implementations live in list_entities.cpp via LIST_ENTITIES_HANDLER. +// NOLINTBEGIN(bugprone-macro-parentheses) +#define ENTITY_TYPE_(type, singular, plural, count, upper) bool on_##singular(type *entity) override; +#define ENTITY_CONTROLLER_TYPE_(type, singular, plural, count, upper, callback) \ + ENTITY_TYPE_(type, singular, plural, count, upper) +#include "esphome/core/entity_types.h" +#undef ENTITY_TYPE_ +#undef ENTITY_CONTROLLER_TYPE_ + // NOLINTEND(bugprone-macro-parentheses) #ifdef USE_API_USER_DEFINED_ACTIONS bool on_service(UserServiceDescriptor *service) override; #endif #ifdef USE_CAMERA bool on_camera(camera::Camera *entity) override; -#endif -#ifdef USE_CLIMATE - bool on_climate(climate::Climate *entity) override; -#endif -#ifdef USE_NUMBER - bool on_number(number::Number *entity) override; -#endif -#ifdef USE_DATETIME_DATE - bool on_date(datetime::DateEntity *entity) override; -#endif -#ifdef USE_DATETIME_TIME - bool on_time(datetime::TimeEntity *entity) override; -#endif -#ifdef USE_DATETIME_DATETIME - bool on_datetime(datetime::DateTimeEntity *entity) override; -#endif -#ifdef USE_TEXT - bool on_text(text::Text *entity) override; -#endif -#ifdef USE_SELECT - bool on_select(select::Select *entity) override; -#endif -#ifdef USE_LOCK - bool on_lock(lock::Lock *entity) override; -#endif -#ifdef USE_VALVE - bool on_valve(valve::Valve *entity) override; -#endif -#ifdef USE_MEDIA_PLAYER - bool on_media_player(media_player::MediaPlayer *entity) override; -#endif -#ifdef USE_ALARM_CONTROL_PANEL - bool on_alarm_control_panel(alarm_control_panel::AlarmControlPanel *entity) override; -#endif -#ifdef USE_WATER_HEATER - bool on_water_heater(water_heater::WaterHeater *entity) override; -#endif -#ifdef USE_INFRARED - bool on_infrared(infrared::Infrared *entity) override; -#endif -#ifdef USE_RADIO_FREQUENCY - bool on_radio_frequency(radio_frequency::RadioFrequency *entity) override; -#endif -#ifdef USE_EVENT - bool on_event(event::Event *entity) override; -#endif -#ifdef USE_UPDATE - bool on_update(update::UpdateEntity *entity) override; #endif bool on_end() override; From 700676b34001d58483b7df5dbd4e339ac3d882ca Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 5 May 2026 18:29:48 -0500 Subject: [PATCH 28/60] [api] Use entity_types.h X-macro for InitialStateIterator declarations (#16075) --- esphome/components/api/subscribe_state.cpp | 5 +- esphome/components/api/subscribe_state.h | 86 ++++------------------ 2 files changed, 18 insertions(+), 73 deletions(-) diff --git a/esphome/components/api/subscribe_state.cpp b/esphome/components/api/subscribe_state.cpp index 4bbc17018e..09b5640d8a 100644 --- a/esphome/components/api/subscribe_state.cpp +++ b/esphome/components/api/subscribe_state.cpp @@ -67,7 +67,10 @@ INITIAL_STATE_HANDLER(water_heater, water_heater::WaterHeater) INITIAL_STATE_HANDLER(update, update::UpdateEntity) #endif -// Special cases (button and event) are already defined inline in subscribe_state.h +// event is an ENTITY_CONTROLLER_TYPE_ but has no state to send. +#ifdef USE_EVENT +bool InitialStateIterator::on_event(event::Event *entity) { return true; } +#endif InitialStateIterator::InitialStateIterator(APIConnection *client) : client_(client) {} diff --git a/esphome/components/api/subscribe_state.h b/esphome/components/api/subscribe_state.h index f20611e06a..6b1ae9651d 100644 --- a/esphome/components/api/subscribe_state.h +++ b/esphome/components/api/subscribe_state.h @@ -19,78 +19,20 @@ class APIConnection; class InitialStateIterator final : public ComponentIterator { public: InitialStateIterator(APIConnection *client); -#ifdef USE_BINARY_SENSOR - bool on_binary_sensor(binary_sensor::BinarySensor *entity) override; -#endif -#ifdef USE_COVER - bool on_cover(cover::Cover *entity) override; -#endif -#ifdef USE_FAN - bool on_fan(fan::Fan *entity) override; -#endif -#ifdef USE_LIGHT - bool on_light(light::LightState *entity) override; -#endif -#ifdef USE_SENSOR - bool on_sensor(sensor::Sensor *entity) override; -#endif -#ifdef USE_SWITCH - bool on_switch(switch_::Switch *entity) override; -#endif -#ifdef USE_BUTTON - bool on_button(button::Button *button) override { return true; }; -#endif -#ifdef USE_TEXT_SENSOR - bool on_text_sensor(text_sensor::TextSensor *entity) override; -#endif -#ifdef USE_CLIMATE - bool on_climate(climate::Climate *entity) override; -#endif -#ifdef USE_NUMBER - bool on_number(number::Number *entity) override; -#endif -#ifdef USE_DATETIME_DATE - bool on_date(datetime::DateEntity *entity) override; -#endif -#ifdef USE_DATETIME_TIME - bool on_time(datetime::TimeEntity *entity) override; -#endif -#ifdef USE_DATETIME_DATETIME - bool on_datetime(datetime::DateTimeEntity *entity) override; -#endif -#ifdef USE_TEXT - bool on_text(text::Text *entity) override; -#endif -#ifdef USE_SELECT - bool on_select(select::Select *entity) override; -#endif -#ifdef USE_LOCK - bool on_lock(lock::Lock *entity) override; -#endif -#ifdef USE_VALVE - bool on_valve(valve::Valve *entity) override; -#endif -#ifdef USE_MEDIA_PLAYER - bool on_media_player(media_player::MediaPlayer *entity) override; -#endif -#ifdef USE_ALARM_CONTROL_PANEL - bool on_alarm_control_panel(alarm_control_panel::AlarmControlPanel *entity) override; -#endif -#ifdef USE_WATER_HEATER - bool on_water_heater(water_heater::WaterHeater *entity) override; -#endif -#ifdef USE_INFRARED - bool on_infrared(infrared::Infrared *infrared) override { return true; }; -#endif -#ifdef USE_RADIO_FREQUENCY - bool on_radio_frequency(radio_frequency::RadioFrequency *radio_frequency) override { return true; }; -#endif -#ifdef USE_EVENT - bool on_event(event::Event *event) override { return true; }; -#endif -#ifdef USE_UPDATE - bool on_update(update::UpdateEntity *entity) override; -#endif + +// Entity overrides (generated from entity_types.h). +// ENTITY_TYPE_ entities have no state to send and default to a no-op. +// ENTITY_CONTROLLER_TYPE_ entities are implemented in subscribe_state.cpp via INITIAL_STATE_HANDLER, +// except on_event which has no state (defined out-of-line in subscribe_state.cpp). +// NOLINTBEGIN(bugprone-macro-parentheses) +#define ENTITY_TYPE_(type, singular, plural, count, upper) \ + bool on_##singular(type *entity) override { return true; } +#define ENTITY_CONTROLLER_TYPE_(type, singular, plural, count, upper, callback) \ + bool on_##singular(type *entity) override; +#include "esphome/core/entity_types.h" +#undef ENTITY_TYPE_ +#undef ENTITY_CONTROLLER_TYPE_ + // NOLINTEND(bugprone-macro-parentheses) protected: APIConnection *client_; From 2d6af1f7e5967e04aa7a2c5e337e77c8005a9a08 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Tue, 5 May 2026 20:22:53 -0400 Subject: [PATCH 29/60] [audio] Bump esp-audio-libs to v3.0.0 (#16263) --- esphome/components/audio/__init__.py | 2 +- esphome/idf_component.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 80fd328e48..cfb2ad4e75 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -334,7 +334,7 @@ async def to_code(config): add_idf_component( name="esphome/esp-audio-libs", - ref="2.0.4", + ref="3.0.0", ) data = _get_data() diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 14fb11ace5..37c0da11f5 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -2,7 +2,7 @@ dependencies: bblanchon/arduinojson: version: "7.4.2" esphome/esp-audio-libs: - version: 2.0.4 + version: 3.0.0 esphome/esp-micro-speech-features: version: 1.2.3 esphome/micro-decoder: From a99c1b3e08de399c1edc9b9364ef293fb0dc1b1a Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Wed, 6 May 2026 02:37:03 +0200 Subject: [PATCH 30/60] [nrf52] add reserve area for bootloader (#16204) --- esphome/components/nrf52/boards.py | 9 ++++++--- esphome/components/zigbee/__init__.py | 13 ------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/esphome/components/nrf52/boards.py b/esphome/components/nrf52/boards.py index 6064fe844a..4c33cd9939 100644 --- a/esphome/components/nrf52/boards.py +++ b/esphome/components/nrf52/boards.py @@ -31,12 +31,15 @@ BOARDS_ZEPHYR = { # https://learn.adafruit.com/introducing-the-adafruit-nrf52840-feather?view=all#hathach-memory-map BOOTLOADER_CONFIG = { BOOTLOADER_ADAFRUIT_NRF52_SD132: [ - Section("empty_app_offset", 0x0, 0x26000, "flash_primary"), + Section("SoftDevice", 0x0, 0x26000, "flash_primary"), + Section("Adafruit_nRF52_Bootloader", 0xF4000, 0xC000, "flash_primary"), ], BOOTLOADER_ADAFRUIT_NRF52_SD140_V6: [ - Section("empty_app_offset", 0x0, 0x26000, "flash_primary"), + Section("SoftDevice", 0x0, 0x26000, "flash_primary"), + Section("Adafruit_nRF52_Bootloader", 0xF4000, 0xC000, "flash_primary"), ], BOOTLOADER_ADAFRUIT_NRF52_SD140_V7: [ - Section("empty_app_offset", 0x0, 0x27000, "flash_primary"), + Section("SoftDevice", 0x0, 0x27000, "flash_primary"), + Section("Adafruit_nRF52_Bootloader", 0xF4000, 0xC000, "flash_primary"), ], } diff --git a/esphome/components/zigbee/__init__.py b/esphome/components/zigbee/__init__.py index 018dab7348..8605b4fa1a 100644 --- a/esphome/components/zigbee/__init__.py +++ b/esphome/components/zigbee/__init__.py @@ -9,9 +9,6 @@ from esphome.components.esp32.const import ( VARIANT_ESP32C6, VARIANT_ESP32H2, ) -from esphome.components.nrf52.boards import BOOTLOADER_CONFIG, Section -from esphome.components.zephyr import zephyr_add_pm_static, zephyr_data -from esphome.components.zephyr.const import KEY_BOOTLOADER import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_INTERNAL, CONF_MODEL, CONF_NAME from esphome.core import CORE, CoroPriority, coroutine_with_priority @@ -53,15 +50,6 @@ _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@luar123", "@tomaszduda23"] -def zigbee_set_core_data(config: ConfigType) -> ConfigType: - if CORE.is_nrf52 and zephyr_data()[KEY_BOOTLOADER] in BOOTLOADER_CONFIG: - zephyr_add_pm_static( - [Section("empty_after_zboss_offset", 0xF4000, 0xC000, "flash_primary")] - ) - - return config - - BINARY_SENSOR_SCHEMA = cv.Schema( { cv.Optional(CONF_REPORT): cv.All( @@ -119,7 +107,6 @@ CONFIG_SCHEMA = cv.All( ).extend(cv.COMPONENT_SCHEMA), _validate_router_sleepy, zigbee_require_vfs_select, - zigbee_set_core_data, cv.Any( cv.All( cv.only_on_esp32, From e9f7579910386c07065a6184bb0245d4fd6170d9 Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Wed, 6 May 2026 03:37:40 +0200 Subject: [PATCH 31/60] [logger] give a chance to print crash (#16203) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/logger/logger_zephyr.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/esphome/components/logger/logger_zephyr.cpp b/esphome/components/logger/logger_zephyr.cpp index 7fa9e42c6a..e9caa8d9d9 100644 --- a/esphome/components/logger/logger_zephyr.cpp +++ b/esphome/components/logger/logger_zephyr.cpp @@ -80,8 +80,8 @@ void Logger::pre_setup() { this->uart_dev_ = uart_dev; #if defined(USE_LOGGER_WAIT_FOR_CDC) && defined(USE_LOGGER_UART_SELECTION_USB_CDC) uint32_t dtr = 0; - uint32_t count = (10 * 100); // wait 10 sec for USB CDC to have early logs - while (dtr == 0 && count-- != 0) { + int32_t count = (10 * 100); // wait 10 sec for USB CDC to have early logs + while (dtr == 0 && count-- > 0) { uart_line_ctrl_get(this->uart_dev_, UART_LINE_CTRL_DTR, &dtr); delay(10); arch_feed_wdt(); @@ -160,6 +160,11 @@ void Logger::dump_crash_() { #if defined(CONFIG_THREAD_NAME) ESP_LOGE(TAG, "Thread: %s", crash_buf.thread); #endif + int32_t count = (2 * 100); // wait 2 sec to give a chance to print crash + while (count-- > 0) { + delay(10); + arch_feed_wdt(); + } } } From 6f6d991dd21f003bc612010ba2cd785e8a9611dc Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 6 May 2026 21:42:11 +1200 Subject: [PATCH 32/60] [ha-addon] Add opt-in toggle for the new ESPHome Device Builder (#16247) --- .../etc/cont-init.d/40-device-builder.sh | 22 +++++++++++++++++++ .../etc/s6-overlay/s6-rc.d/esphome/run | 7 ++++++ .../etc/s6-overlay/s6-rc.d/init-nginx/run | 8 +++++++ .../etc/s6-overlay/s6-rc.d/nginx/run | 8 +++++++ 4 files changed, 45 insertions(+) create mode 100755 docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh diff --git a/docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh b/docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh new file mode 100755 index 0000000000..b990469762 --- /dev/null +++ b/docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh @@ -0,0 +1,22 @@ +#!/usr/bin/with-contenv bashio +# ============================================================================== +# Installs the latest prerelease of esphome-device-builder when the +# `use_new_device_builder` config option is enabled. +# This is a temporary install-on-boot step until esphome-device-builder +# becomes a direct dependency of esphome. +# ============================================================================== + +if ! bashio::config.true 'use_new_device_builder'; then + exit 0 +fi + +bashio::log.info "Installing latest prerelease of esphome-device-builder..." +if command -v uv > /dev/null; then + uv pip install --system --no-cache-dir --prerelease=allow --upgrade \ + esphome-device-builder || + bashio::exit.nok "Failed installing esphome-device-builder." +else + pip install --no-cache-dir --pre --upgrade esphome-device-builder || + bashio::exit.nok "Failed installing esphome-device-builder." +fi +bashio::log.info "Installed esphome-device-builder." diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run index cdbaff6c04..64ac0b18d2 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run @@ -49,5 +49,12 @@ if bashio::fs.directory_exists '/config/esphome/.esphome'; then rm -rf /config/esphome/.esphome fi +if bashio::config.true 'use_new_device_builder'; then + bashio::log.info "Starting ESPHome Device Builder..." + exec esphome-device-builder /config/esphome \ + --ha-addon \ + --ingress-port "$(bashio::addon.ingress_port)" +fi + bashio::log.info "Starting ESPHome dashboard..." exec esphome dashboard /config/esphome --socket /var/run/esphome.sock --ha-addon diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run index 2725f56670..18c75898ec 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run @@ -4,6 +4,14 @@ # Community Hass.io Add-ons: ESPHome # Configures NGINX for use with ESPHome # ============================================================================== + +# When the new device builder is enabled it serves HA ingress directly, +# so nginx is not used at all -- skip configuration. +if bashio::config.true 'use_new_device_builder'; then + bashio::log.info "Skipping NGINX setup: new device builder serves ingress directly." + bashio::exit.ok +fi + mkdir -p /var/log/nginx # Generate Ingress configuration diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run index e96991cdd1..bb5f52e10c 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run @@ -5,6 +5,14 @@ # Runs the NGINX proxy # ============================================================================== +# The new device builder handles HA ingress itself, so nginx is bypassed. +# Block the longrun forever so s6 keeps the dependency satisfied and does +# not respawn it. +if bashio::config.true 'use_new_device_builder'; then + bashio::log.info "NGINX bypassed: new device builder serves ingress directly." + exec sleep infinity +fi + bashio::log.info "Waiting for ESPHome dashboard to come up..." while [[ ! -S /var/run/esphome.sock ]]; do From febf8815c733778a25c8ab358b017a579217a94d Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Wed, 6 May 2026 05:59:51 -0400 Subject: [PATCH 33/60] [audio_file][speaker] Eliminate code duplication for files built into firmware (#16266) --- esphome/components/audio_file/__init__.py | 81 ++++---- .../speaker/media_player/__init__.py | 183 ++---------------- .../speaker/common-media_player.yaml | 13 ++ tests/components/speaker/test.wav | Bin 0 -> 46 bytes 4 files changed, 73 insertions(+), 204 deletions(-) create mode 100644 tests/components/speaker/test.wav diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index b246633c31..23c90e9b76 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -199,51 +199,60 @@ def _validate_supported_local_file(config: list[ConfigType]) -> list[ConfigType] return config +def audio_files_schema() -> cv.All: + """Schema for a list of audio file entries. + + Validates each entry, downloads any web files, and detects the audio file + type while requesting codec support. Reusable by other components (e.g. + speaker media_player) that embed audio files in firmware without going + through the audio_file component's C++ registry. + """ + return cv.All( + cv.ensure_list(MEDIA_FILE_TYPE_SCHEMA), + partial(download_web_files_in_config, path_for=_compute_local_file_path), + _validate_supported_local_file, + ) + + +def generate_audio_file_code(file_config: ConfigType) -> MockObj: + """Generate the progmem data, AudioFile struct, and Pvariable for one file. + + Returns the created Pvariable. Caller is responsible for any further + registration (the audio_file component additionally registers each file in + its named C++ registry; other consumers may skip that). + """ + cache = _get_data().file_cache + file_id = str(file_config[CONF_ID]) + if file_id in cache: + data, media_file_type = cache[file_id] + else: + data, media_file_type = read_audio_file_and_type(file_config) + + rhs = [HexInt(x) for x in data] + prog_arr = cg.progmem_array(file_config[CONF_RAW_DATA_ID], rhs) + + media_files_struct = cg.StructInitializer( + audio.AudioFile, + ("data", prog_arr), + ("length", len(rhs)), + ("file_type", media_file_type), + ) + + return cg.new_Pvariable(file_config[CONF_ID], media_files_struct) + + CONFIG_SCHEMA = cv.All( cv.only_on_esp32, - cv.ensure_list(MEDIA_FILE_TYPE_SCHEMA), - partial(download_web_files_in_config, path_for=_compute_local_file_path), - _validate_supported_local_file, + audio_files_schema(), ) async def to_code(config: list[ConfigType]) -> None: - cache = _get_data().file_cache - for file_config in config: file_id = str(file_config[CONF_ID]) - data, media_file_type = cache[file_id] - - rhs = [HexInt(x) for x in data] - prog_arr = cg.progmem_array(file_config[CONF_RAW_DATA_ID], rhs) - - media_files_struct = cg.StructInitializer( - audio.AudioFile, - ( - "data", - prog_arr, - ), - ( - "length", - len(rhs), - ), - ( - "file_type", - media_file_type, - ), - ) - - cg.new_Pvariable( - file_config[CONF_ID], - media_files_struct, - ) - - # Store file ID for cross-component access + file_var = generate_audio_file_code(file_config) _get_data().file_ids[file_id] = file_config[CONF_ID] + cg.add(audio_file_ns.add_named_audio_file(file_var, file_id)) # Register all files in the shared C++ registry cg.add_define("AUDIO_FILE_MAX_FILES", len(config)) - for file_config in config: - file_id = str(file_config[CONF_ID]) - file_var = await cg.get_variable(file_config[CONF_ID]) - cg.add(audio_file_ns.add_named_audio_file(file_var, file_id)) diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index 90d9309f46..094043c292 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -1,13 +1,19 @@ """Speaker Media Player Setup.""" -from functools import partial -import hashlib import logging -from pathlib import Path -from esphome import automation, external_files +from esphome import automation import esphome.codegen as cg -from esphome.components import audio, esp32, media_player, network, ota, psram, speaker +from esphome.components import ( + audio, + audio_file, + esp32, + media_player, + network, + ota, + psram, + speaker, +) from esphome.components.const import ( CONF_VOLUME_INCREMENT, CONF_VOLUME_INITIAL, @@ -17,23 +23,16 @@ from esphome.components.const import ( import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, - CONF_FILE, CONF_FILES, CONF_FORMAT, CONF_ID, CONF_NUM_CHANNELS, CONF_ON_TURN_OFF, CONF_ON_TURN_ON, - CONF_PATH, - CONF_RAW_DATA_ID, CONF_SAMPLE_RATE, CONF_SPEAKER, CONF_TASK_STACK_IN_PSRAM, - CONF_TYPE, - CONF_URL, ) -from esphome.core import CORE, HexInt -from esphome.external_files import download_web_files_in_config _LOGGER = logging.getLogger(__name__) @@ -44,9 +43,6 @@ DEPENDENCIES = ["network"] CODEOWNERS = ["@kahrendt", "@synesthesiam"] DOMAIN = "media_player" -TYPE_LOCAL = "local" -TYPE_WEB = "web" - CONF_ANNOUNCEMENT = "announcement" CONF_ANNOUNCEMENT_PIPELINE = "announcement_pipeline" CONF_CODEC_SUPPORT_ENABLED = "codec_support_enabled" # Remove before 2026.10.0 @@ -83,87 +79,12 @@ StopStreamAction = speaker_ns.class_( ) -def _compute_local_file_path(value: dict) -> Path: - url = value[CONF_URL] - h = hashlib.new("sha256") - h.update(url.encode()) - key = h.hexdigest()[:8] - base_dir = external_files.compute_local_file_dir(DOMAIN) - _LOGGER.debug("_compute_local_file_path: base_dir=%s", base_dir / key) - return base_dir / key - - _PURPOSE_MAP = { "MEDIA": media_player.MEDIA_PLAYER_FORMAT_PURPOSE_ENUM["default"], "ANNOUNCEMENT": media_player.MEDIA_PLAYER_FORMAT_PURPOSE_ENUM["announcement"], } -def _file_schema(value): - if isinstance(value, str): - return _validate_file_shorthand(value) - return TYPED_FILE_SCHEMA(value) - - -def _read_audio_file_and_type(file_config): - conf_file = file_config[CONF_FILE] - file_source = conf_file[CONF_TYPE] - if file_source == TYPE_LOCAL: - path = CORE.relative_config_path(conf_file[CONF_PATH]) - elif file_source == TYPE_WEB: - path = _compute_local_file_path(conf_file) - else: - raise cv.Invalid("Unsupported file source") - - with open(path, "rb") as f: - data = f.read() - - import puremagic - - try: - file_type: str = puremagic.from_string(data) - file_type = file_type.removeprefix(".") - except puremagic.PureError as e: - raise cv.Invalid( - f"Unable to determine audio file type of '{path}'. " - f"Try re-encoding the file into a supported format. Details: {e}" - ) from e - - media_file_type = audio.AUDIO_FILE_TYPE_ENUM["NONE"] - if file_type in ("wav"): - media_file_type = audio.AUDIO_FILE_TYPE_ENUM["WAV"] - elif file_type in ("mp3", "mpeg", "mpga"): - media_file_type = audio.AUDIO_FILE_TYPE_ENUM["MP3"] - elif file_type in ("flac"): - media_file_type = audio.AUDIO_FILE_TYPE_ENUM["FLAC"] - elif ( - file_type in ("ogg") - and len(data) >= 36 - and data.startswith(b"OggS") - and data[28:36] == b"OpusHead" - ): - media_file_type = audio.AUDIO_FILE_TYPE_ENUM["OPUS"] - - return data, media_file_type - - -def _validate_file_shorthand(value): - value = cv.string_strict(value) - if value.startswith("http://") or value.startswith("https://"): - return _file_schema( - { - CONF_TYPE: TYPE_WEB, - CONF_URL: value, - } - ) - return _file_schema( - { - CONF_TYPE: TYPE_LOCAL, - CONF_PATH: value, - } - ) - - _validate_pipeline = media_player.validate_preferred_format( "speaker media_player", CONF_SPEAKER ) @@ -192,60 +113,15 @@ def _final_validate(config): CONF_CODEC_SUPPORT_ENABLED, ) - # Request codecs based on pipeline formats + # Request codecs based on pipeline formats. Codecs needed by local files are + # already requested during CONFIG_SCHEMA validation (via audio_files_schema). media_player.request_codecs_for_format_configs( config, [CONF_ANNOUNCEMENT_PIPELINE, CONF_MEDIA_PIPELINE] ) - # Validate local files and request any additional codecs they need - for file_config in config.get(CONF_FILES, []): - _, media_file_type = _read_audio_file_and_type(file_config) - if str(media_file_type) == str(audio.AUDIO_FILE_TYPE_ENUM["NONE"]): - raise cv.Invalid("Unsupported local media file") - for fmt_name, fmt_enum in audio.AUDIO_FILE_TYPE_ENUM.items(): - if str(media_file_type) == str(fmt_enum): - if fmt_name == "FLAC": - audio.request_flac_support() - elif fmt_name == "MP3": - audio.request_mp3_support() - elif fmt_name == "OPUS": - audio.request_opus_support() - elif fmt_name == "WAV": - audio.request_wav_support() - break - return config -LOCAL_SCHEMA = cv.Schema( - { - cv.Required(CONF_PATH): cv.file_, - } -) - -WEB_SCHEMA = cv.Schema( - { - cv.Required(CONF_URL): cv.url, - } -) - - -TYPED_FILE_SCHEMA = cv.typed_schema( - { - TYPE_LOCAL: LOCAL_SCHEMA, - TYPE_WEB: WEB_SCHEMA, - }, -) - - -MEDIA_FILE_TYPE_SCHEMA = cv.Schema( - { - cv.Required(CONF_ID): cv.declare_id(audio.AudioFile), - cv.Required(CONF_FILE): _file_schema, - cv.GenerateID(CONF_RAW_DATA_ID): cv.declare_id(cg.uint8), - } -) - PIPELINE_SCHEMA = cv.Schema( { cv.GenerateID(): cv.declare_id(AudioPipeline), @@ -278,12 +154,7 @@ CONFIG_SCHEMA = cv.All( ), # Remove before 2026.10.0 cv.Optional(CONF_CODEC_SUPPORT_ENABLED): cv.Any(cv.boolean, cv.string), - cv.Optional(CONF_FILES): cv.All( - cv.ensure_list(MEDIA_FILE_TYPE_SCHEMA), - partial( - download_web_files_in_config, path_for=_compute_local_file_path - ), - ), + cv.Optional(CONF_FILES): audio_file.audio_files_schema(), cv.Optional(CONF_TASK_STACK_IN_PSRAM): cv.All( cv.boolean, cv.requires_component(psram.DOMAIN) ), @@ -380,31 +251,7 @@ async def to_code(config): ) for file_config in config.get(CONF_FILES, []): - data, media_file_type = _read_audio_file_and_type(file_config) - - rhs = [HexInt(x) for x in data] - prog_arr = cg.progmem_array(file_config[CONF_RAW_DATA_ID], rhs) - - media_files_struct = cg.StructInitializer( - audio.AudioFile, - ( - "data", - prog_arr, - ), - ( - "length", - len(rhs), - ), - ( - "file_type", - media_file_type, - ), - ) - - cg.new_Pvariable( - file_config[CONF_ID], - media_files_struct, - ) + audio_file.generate_audio_file_code(file_config) @automation.register_action( diff --git a/tests/components/speaker/common-media_player.yaml b/tests/components/speaker/common-media_player.yaml index a849e04b33..3b2212a0ca 100644 --- a/tests/components/speaker/common-media_player.yaml +++ b/tests/components/speaker/common-media_player.yaml @@ -17,3 +17,16 @@ media_player: volume_max: 0.95 volume_min: 0.0 task_stack_in_psram: true + files: + - id: speaker_test_audio + file: + type: local + path: $component_dir/test.wav + +script: + - id: play_built_in_file + then: + - media_player.speaker.play_on_device_media_file: + id: speaker_media_player_id + media_file: speaker_test_audio + announcement: true diff --git a/tests/components/speaker/test.wav b/tests/components/speaker/test.wav new file mode 100644 index 0000000000000000000000000000000000000000..f9d07ef2238eb2fcb355055466d3789ee1a1fe0b GIT binary patch literal 46 vcmWIYbaPW Date: Wed, 6 May 2026 21:22:43 +1000 Subject: [PATCH 34/60] [lvgl] Allow line points as percentages (#16209) --- esphome/components/lvgl/lv_validation.py | 4 +- esphome/components/lvgl/schemas.py | 12 +- esphome/components/lvgl/widgets/canvas.py | 24 ++- esphome/components/lvgl/widgets/line.py | 17 +- .../lvgl/config/line_points.yaml | 84 ++++++++++ tests/component_tests/lvgl/test_line.py | 147 ++++++++++++++++++ tests/components/lvgl/lvgl-package.yaml | 5 +- 7 files changed, 271 insertions(+), 22 deletions(-) create mode 100644 tests/component_tests/lvgl/config/line_points.yaml create mode 100644 tests/component_tests/lvgl/test_line.py diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index 503730098e..974eed9e81 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -41,7 +41,7 @@ from .helpers import ( lv_fonts_used, requires_component, ) -from .types import lv_gradient_t, lv_opa_t +from .types import lv_coord_t, lv_gradient_t, lv_opa_t LV_OPA = LvConstant("LV_OPA_", "TRANSP", "COVER") @@ -277,7 +277,7 @@ def pixels_or_percent_validator(value): pixels_or_percent = LValidator( pixels_or_percent_validator, - uint32, + lv_coord_t, retmapper=lambda x: x if isinstance(x, int) else literal(f"lv_pct({int(x * 100)})"), ) diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 2c57452a55..62117fbd32 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -123,8 +123,8 @@ ENCODER_SCHEMA = cv.Schema( POINT_SCHEMA = cv.Schema( { - cv.Required(CONF_X): cv.templatable(cv.int_), - cv.Required(CONF_Y): cv.templatable(cv.int_), + cv.Required(CONF_X): lvalid.pixels_or_percent, + cv.Required(CONF_Y): lvalid.pixels_or_percent, } ) @@ -137,9 +137,13 @@ def point_schema(value): """ if isinstance(value, dict): return POINT_SCHEMA(value) + if isinstance(value, list): + if len(value) != 2: + raise cv.Invalid("Invalid point format, should be , ") + return POINT_SCHEMA({CONF_X: value[0], CONF_Y: value[1]}) try: - x, y = map(int, value.split(",")) - return {CONF_X: x, CONF_Y: y} + x, y = str(value).split(",") + return POINT_SCHEMA({CONF_X: x, CONF_Y: y}) except ValueError: pass # not raising this in the catch block because pylint doesn't like it diff --git a/esphome/components/lvgl/widgets/canvas.py b/esphome/components/lvgl/widgets/canvas.py index f12766bae1..1308b82dcd 100644 --- a/esphome/components/lvgl/widgets/canvas.py +++ b/esphome/components/lvgl/widgets/canvas.py @@ -52,6 +52,7 @@ from ..lv_validation import ( lv_text, opacity, pixels, + pixels_or_percent, size, ) from ..lvcode import LocalVariable, lv, lv_assign, lv_expr @@ -59,7 +60,7 @@ from ..schemas import STYLE_PROPS, TEXT_SCHEMA, point_schema, remap_property from ..types import LvType, ObjUpdateAction from . import Widget, WidgetType, get_widgets from .img import CONF_IMAGE -from .line import lv_point_precise_t, process_coord +from .line import lv_point_precise_t CONF_CANVAS = "canvas" CONF_BUFFER_ID = "buffer_id" @@ -434,6 +435,13 @@ LINE_PROPS = { } +def _validate_points(config): + for index, point in enumerate(config[CONF_POINTS]): + if not all(isinstance(p, int) for p in point.values()): + raise cv.Invalid("Points must be integers", path=[CONF_POINTS, index]) + return config + + @automation.register_action( "lvgl.canvas.draw_line", ObjUpdateAction, @@ -444,12 +452,15 @@ LINE_PROPS = { cv.Required(CONF_POINTS): cv.ensure_list(point_schema), **{cv.Optional(prop): validator for prop, validator in LINE_PROPS.items()}, } - ), + ).add_extra(_validate_points), synchronous=True, ) async def canvas_draw_line(config, action_id, template_arg, args): points = [ - [await process_coord(p[CONF_X]), await process_coord(p[CONF_Y])] + [ + await pixels.process(p[CONF_X]), + await pixels.process(p[CONF_Y]), + ] for p in config[CONF_POINTS] ] @@ -470,12 +481,15 @@ async def canvas_draw_line(config, action_id, template_arg, args): cv.Required(CONF_POINTS): cv.ensure_list(point_schema), **{cv.Optional(prop): STYLE_PROPS[prop] for prop in RECT_PROPS}, }, - ), + ).add_extra(_validate_points), synchronous=True, ) async def canvas_draw_polygon(config, action_id, template_arg, args): points = [ - [await process_coord(p[CONF_X]), await process_coord(p[CONF_Y])] + [ + await pixels_or_percent.process(p[CONF_X]), + await pixels_or_percent.process(p[CONF_Y]), + ] for p in config[CONF_POINTS] ] # Close the polygon diff --git a/esphome/components/lvgl/widgets/line.py b/esphome/components/lvgl/widgets/line.py index 3112cc28d0..19f421cbbd 100644 --- a/esphome/components/lvgl/widgets/line.py +++ b/esphome/components/lvgl/widgets/line.py @@ -1,12 +1,12 @@ import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_X, CONF_Y -from esphome.core import Lambda -from ..defines import CONF_MAIN, call_lambda +from ..defines import CONF_MAIN +from ..lv_validation import pixels_or_percent from ..lvcode import lv_add from ..schemas import point_schema -from ..types import LvCompound, LvType, lv_coord_t +from ..types import LvCompound, LvType from . import Widget, WidgetType CONF_LINE = "line" @@ -17,12 +17,6 @@ lv_point_t = cg.global_ns.struct("lv_point_t") lv_point_precise_t = cg.global_ns.struct("lv_point_precise_t") -async def process_coord(coord): - if isinstance(coord, Lambda): - return call_lambda(await cg.process_lambda(coord, [], return_type=lv_coord_t)) - return cg.safe_exp(coord) - - class LineType(WidgetType): def __init__(self): super().__init__( @@ -36,7 +30,10 @@ class LineType(WidgetType): async def to_code(self, w: Widget, config): if CONF_POINTS in config: points = [ - [await process_coord(p[CONF_X]), await process_coord(p[CONF_Y])] + [ + await pixels_or_percent.process(p[CONF_X]), + await pixels_or_percent.process(p[CONF_Y]), + ] for p in config[CONF_POINTS] ] lv_add(w.var.set_points(points)) diff --git a/tests/component_tests/lvgl/config/line_points.yaml b/tests/component_tests/lvgl/config/line_points.yaml new file mode 100644 index 0000000000..5d7be3bc20 --- /dev/null +++ b/tests/component_tests/lvgl/config/line_points.yaml @@ -0,0 +1,84 @@ +esphome: + name: test-line + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: + number: GPIO3 + +lvgl: + widgets: + # Dict format + - line: + id: line_dict + points: + - x: 10 + y: 20 + - x: 100 + y: 200 + - x: 0 + y: 0 + + # List format + - line: + id: line_list + points: + - [10, 20] + - [100, 200] + - [0, 0] + + # String format + - line: + id: line_string + points: + - "10, 20" + - "100, 200" + - "0, 0" + + # Percentage - dict format + - line: + id: line_pct_dict + points: + - x: "50%" + y: "75%" + + # Percentage - list format + - line: + id: line_pct_list + points: + - ["50%", "75%"] + + # Percentage - string format + - line: + id: line_pct_string + points: + - "50%, 75%" + + # Mixed integer and percentage + - line: + id: line_mixed_dict + points: + - x: 10 + y: "50%" + - x: "25%" + y: 200 + + - line: + id: line_mixed_list + points: + - [10, "50%"] + - ["25%", 200] diff --git a/tests/component_tests/lvgl/test_line.py b/tests/component_tests/lvgl/test_line.py new file mode 100644 index 0000000000..fce0ef8fa8 --- /dev/null +++ b/tests/component_tests/lvgl/test_line.py @@ -0,0 +1,147 @@ +"""Tests for the LVGL line widget point schema and code generation.""" + +from __future__ import annotations + +import re + +import pytest + +from esphome.components.lvgl.schemas import point_schema +from esphome.config_validation import Invalid +from esphome.const import CONF_X, CONF_Y + +# --------------------------------------------------------------------------- +# Validation: point_schema normalises dict / list / string to same result +# --------------------------------------------------------------------------- + + +class TestPointSchemaValidation: + """Test that all point input formats normalise to the same dict.""" + + @pytest.mark.parametrize( + "dict_input,list_input,string_input", + [ + ({CONF_X: 10, CONF_Y: 20}, [10, 20], "10, 20"), + ({CONF_X: 0, CONF_Y: 0}, [0, 0], "0, 0"), + ({CONF_X: 100, CONF_Y: 200}, [100, 200], "100, 200"), + ({CONF_X: -5, CONF_Y: -10}, [-5, -10], "-5, -10"), + ], + ) + def test_integer_formats_produce_same_result( + self, dict_input, list_input, string_input + ): + result_dict = point_schema(dict_input) + result_list = point_schema(list_input) + result_string = point_schema(string_input) + + assert result_dict == result_list + assert result_dict == result_string + + def test_percentage_formats_produce_same_result(self): + result_dict = point_schema({CONF_X: "50%", CONF_Y: "75%"}) + result_list = point_schema(["50%", "75%"]) + result_string = point_schema("50%, 75%") + + assert result_dict == result_list + assert result_dict == result_string + + def test_pixel_suffix_matches_plain_integer(self): + result_px = point_schema({CONF_X: "10px", CONF_Y: "20px"}) + result_int = point_schema({CONF_X: 10, CONF_Y: 20}) + + assert result_px == result_int + + @pytest.mark.parametrize( + "value", + [ + {CONF_X: 50, CONF_Y: 75}, + [50, 75], + "50, 75", + ], + ) + def test_output_contains_x_and_y(self, value): + result = point_schema(value) + + assert CONF_X in result + assert CONF_Y in result + + def test_list_wrong_length_raises(self): + with pytest.raises(Invalid, match="Invalid point"): + point_schema([1]) + + with pytest.raises(Invalid, match="Invalid point"): + point_schema([1, 2, 3]) + + def test_string_without_comma_raises(self): + with pytest.raises(Invalid, match="Invalid point"): + point_schema("garbage") + + def test_string_extra_commas_raises(self): + with pytest.raises(Invalid, match="Invalid point"): + point_schema("1,2,3") + + +# --------------------------------------------------------------------------- +# Code generation: different point formats produce identical C++ output +# --------------------------------------------------------------------------- + +_SET_POINTS_RE = re.compile(r"(\w+)->set_points\((.+?)\);") + + +def _extract_set_points(main_cpp: str) -> dict[str, str]: + """Return {var_name: args_text} for every set_points() call found.""" + return {m.group(1): m.group(2) for m in _SET_POINTS_RE.finditer(main_cpp)} + + +class TestLineCodeGeneration: + """Verify that alternative point formats generate identical C++ code.""" + + @pytest.fixture() + def main_cpp(self, generate_main, component_config_path) -> str: + return generate_main(component_config_path("line_points.yaml")) + + @pytest.fixture() + def set_points_calls(self, main_cpp) -> dict[str, str]: + return _extract_set_points(main_cpp) + + def test_integer_points_all_formats_match(self, set_points_calls): + """Dict, list, and string formats with integer points produce same set_points call.""" + assert set_points_calls["line_dict"] == set_points_calls["line_list"] + assert set_points_calls["line_dict"] == set_points_calls["line_string"] + + def test_percentage_points_all_formats_match(self, set_points_calls): + """Dict, list, and string formats with percentage points produce same set_points call.""" + assert set_points_calls["line_pct_dict"] == set_points_calls["line_pct_list"] + assert set_points_calls["line_pct_dict"] == set_points_calls["line_pct_string"] + + def test_mixed_points_formats_match(self, set_points_calls): + """Dict and list formats with mixed int/percent points produce same set_points call.""" + assert ( + set_points_calls["line_mixed_dict"] == set_points_calls["line_mixed_list"] + ) + + def test_integer_points_contain_expected_values(self, set_points_calls): + """Integer points appear literally in the generated code.""" + args = set_points_calls["line_dict"] + for val in ("10", "20", "100", "200"): + assert val in args + + def test_percentage_points_use_lv_pct(self, set_points_calls): + """Percentage points are generated using the lv_pct() macro.""" + args = set_points_calls["line_pct_dict"] + assert "lv_pct(50)" in args + assert "lv_pct(75)" in args + + def test_all_lines_present(self, set_points_calls): + """All expected line IDs have a set_points call.""" + expected = { + "line_dict", + "line_list", + "line_string", + "line_pct_dict", + "line_pct_list", + "line_pct_string", + "line_mixed_dict", + "line_mixed_list", + } + assert expected.issubset(set_points_calls.keys()) diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 9c4ad4bbf8..39d7472054 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -1038,7 +1038,10 @@ lvgl: - 5, 5 - x: !lambda return random_uint32() % 100; y: !lambda return random_uint32() % 100; - - 70, 70 + - x: 10% + y: 50% + - 70%, 70% + - [75%, 75%] - 120, 10 - 180, 60 - 240, 10 From 85f33978e73a28df88955db52d8ccdb7557b5290 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 06:23:35 -0500 Subject: [PATCH 35/60] [core] Skip external component update on `esphome clean` (#16268) --- esphome/__main__.py | 5 +++-- tests/unit_tests/test_main.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index f4a276b74c..825a502dbf 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2301,8 +2301,9 @@ def run_esphome(argv): CORE.config_path = conf_path CORE.dashboard = args.dashboard - # For logs command, skip updating external components - skip_external = args.command == "logs" + # Commands that don't need fresh external components: logs just connects + # to the device, and clean is about to delete the build directory. + skip_external = args.command in ("logs", "clean") config = read_config( dict(args.substitution) if args.substitution else {}, skip_external_update=skip_external, diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 0b96000a57..4ab7bb3344 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -5038,6 +5038,32 @@ def test_run_esphome_non_bundle_skips_extraction(tmp_path: Path) -> None: assert result == 2 +@pytest.mark.parametrize( + ("command", "expected_skip"), + [ + ("logs", True), + ("clean", True), + ("compile", False), + ("config", False), + ("run", False), + ("clean-mqtt", False), + ], +) +def test_run_esphome_skip_external_update_per_command( + tmp_path: Path, command: str, expected_skip: bool +) -> None: + """read_config is invoked with skip_external_update=True only for commands + that don't need fresh external components (logs, clean).""" + yaml_file = tmp_path / "device.yaml" + yaml_file.write_text("esphome:\n name: test\n") + + with patch("esphome.__main__.read_config", return_value=None) as mock_read: + run_esphome(["esphome", command, str(yaml_file)]) + + mock_read.assert_called_once() + assert mock_read.call_args.kwargs["skip_external_update"] is expected_skip + + def test_get_configured_xtal_freq_reads_sdkconfig(tmp_path: Path) -> None: """Test reading XTAL_FREQ from sdkconfig.""" CORE.name = "test-device" From 29db5fa4bb79333310837c9287405200bdba12b9 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Wed, 6 May 2026 08:11:06 -0400 Subject: [PATCH 36/60] [script] Make pre-commit and helpers work on Windows (#16260) Co-authored-by: Jonathan Swoboda --- .pre-commit-config.yaml | 4 ++-- script/ci-custom.py | 2 +- script/run-in-env.py | 9 ++++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad82bd8e5d..da5fb94d5e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: hooks: - id: pylint name: pylint - entry: python3 script/run-in-env.py pylint + entry: python script/run-in-env.py pylint language: system types: [python] files: ^esphome/.+\.py$ @@ -68,5 +68,5 @@ repos: additional_dependencies: [] - id: ci-custom name: ci-custom - entry: python3 script/run-in-env.py script/ci-custom.py + entry: python script/run-in-env.py script/ci-custom.py language: system diff --git a/script/ci-custom.py b/script/ci-custom.py index b257a3818b..8cd8fd7544 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -250,7 +250,7 @@ def lint_ext_check(fname): ] ) def lint_executable_bit(fname: Path) -> str | None: - ex = EXECUTABLE_BIT[str(fname)] + ex = EXECUTABLE_BIT[fname.as_posix()] if ex != 100644: return ( f"File has invalid executable bit {ex}. If running from a windows machine please " diff --git a/script/run-in-env.py b/script/run-in-env.py index 9283ba9940..996db60554 100755 --- a/script/run-in-env.py +++ b/script/run-in-env.py @@ -44,7 +44,14 @@ def find_and_activate_virtualenv(): def run_command(): # Execute the remaining arguments in the new environment if len(sys.argv) > 1: - result = subprocess.run(sys.argv[1:], check=False, close_fds=False) + args = sys.argv[1:] + # Windows CreateProcess doesn't follow shebangs, so prepend the + # current interpreter when the entry is a .py script. Using + # sys.executable also pins the nested call to the same Python that + # ran us — no ambiguous PATH lookup for "python". + if args[0].endswith(".py"): + args = [sys.executable, *args] + result = subprocess.run(args, check=False, close_fds=False) sys.exit(result.returncode) else: print( From f06ad8c4366f4f51ae54370f9713ccf29a998573 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 07:32:19 -0500 Subject: [PATCH 37/60] [http_request] Add regression test for light action inside on_response (#16270) --- .../components/http_request/http_request.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/components/http_request/http_request.yaml b/tests/components/http_request/http_request.yaml index ef67671c91..46d4b88ec5 100644 --- a/tests/components/http_request/http_request.yaml +++ b/tests/components/http_request/http_request.yaml @@ -50,12 +50,33 @@ esphome: format: "After delay, body still: %s" args: - body.c_str() + # Regression test for esphome/esphome#16224: a LightControlAction + # nested inside on_response with capture_response: true puts + # `std::string &` into the trigger's Ts..., which exposed a codegen + # bug where the apply lambda's parameter list did not match the + # ApplyFn signature. + - light.turn_on: + id: test_regression_light + brightness: 100% + effect: "None" http_request: useragent: esphome/tagreader timeout: 10s verify_ssl: ${verify_ssl} +output: + - platform: template + id: test_regression_output + type: float + write_action: + - logger.log: "set" + +light: + - platform: monochromatic + id: test_regression_light + output: test_regression_output + script: - id: does_not_compile parameters: From ff0c5f575e84f8143fd42a40f0c8a360599aa543 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 07:32:35 -0500 Subject: [PATCH 38/60] [bundle] Include secrets.yaml when `!secret` keys are quoted (#16271) --- esphome/bundle.py | 12 +++++---- tests/unit_tests/test_bundle.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/esphome/bundle.py b/esphome/bundle.py index efa80acc8c..70c4fad0fd 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -98,11 +98,13 @@ _KNOWN_FILE_EXTENSIONS = frozenset( ) -# Matches !secret references in YAML text. This is intentionally a simple -# regex scan rather than a YAML parse — it may match inside comments or -# multi-line strings, which is the conservative direction (include more -# secrets rather than fewer). -_SECRET_RE = re.compile(r"!secret\s+(\S+)") +# Matches !secret references in YAML text. An optional surrounding +# quote pair around the key is allowed and ignored: YAML treats +# ``!secret 'foo'`` and ``!secret foo`` as the same key. This is +# intentionally a simple regex scan rather than a YAML parse — it may +# match inside comments or multi-line strings, which is the conservative +# direction (include more secrets rather than fewer). +_SECRET_RE = re.compile(r"""!secret\s+['"]?([^\s'"]+)""") def _find_used_secret_keys(yaml_files: list[Path]) -> set[str]: diff --git a/tests/unit_tests/test_bundle.py b/tests/unit_tests/test_bundle.py index 89bf1a33b3..5d046252da 100644 --- a/tests/unit_tests/test_bundle.py +++ b/tests/unit_tests/test_bundle.py @@ -170,6 +170,23 @@ def test_find_used_secret_keys_deduplicates(tmp_path: Path) -> None: assert keys == {"key1"} +def test_find_used_secret_keys_quoted(tmp_path: Path) -> None: + """Quoted !secret keys should resolve to the same key as unquoted form. + + YAML strips surrounding quotes during parsing, so the secrets.yaml + lookup uses the unquoted key. The bundle scan must do the same. + """ + yaml1 = tmp_path / "a.yaml" + yaml1.write_text( + "single: !secret 'wifi_ssid'\n" + 'double: !secret "wifi_pw"\n' + "bare: !secret api_key\n" + ) + + keys = _find_used_secret_keys([yaml1]) + assert keys == {"wifi_ssid", "wifi_pw", "api_key"} + + # --------------------------------------------------------------------------- # _add_bytes_to_tar # --------------------------------------------------------------------------- @@ -1217,6 +1234,35 @@ def test_create_bundle_filters_secrets(tmp_path: Path) -> None: assert "should_not_appear" not in secrets_data +def test_create_bundle_filters_secrets_quoted(tmp_path: Path) -> None: + """Bundling must include secrets.yaml when !secret keys are quoted. + + Regression test for issue 16259: quoted !secret references previously + captured the quotes as part of the key, so no key matched secrets.yaml + entries and the secrets file was dropped from the bundle entirely. + """ + config_dir = _setup_config_dir(tmp_path) + + secrets = config_dir / "secrets.yaml" + secrets.write_text("ota_password: hunter2\nunused: should_not_appear\n") + + config_yaml = "ota:\n password: !secret 'ota_password'\n" + (config_dir / "test.yaml").write_text(config_yaml) + + creator = ConfigBundleCreator({}) + result = creator.create_bundle() + + assert result.manifest[ManifestKey.HAS_SECRETS] is True + + buf = io.BytesIO(result.data) + with tarfile.open(fileobj=buf, mode="r:gz") as tar: + secrets_data = tar.extractfile("secrets.yaml").read().decode() + + assert "ota_password" in secrets_data + assert "hunter2" in secrets_data + assert "unused" not in secrets_data + + def test_create_bundle_no_secrets(tmp_path: Path) -> None: _setup_config_dir(tmp_path) From caaa1aefc7d4c467fb07289bcf83f13ecb464bf6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 08:41:17 -0500 Subject: [PATCH 39/60] [substitutions] Fix sibling references inside dict-valued substitutions (#16273) --- esphome/components/substitutions/__init__.py | 13 ++++++++++++- .../18-dict_self_reference.approved.yaml | 16 ++++++++++++++++ .../18-dict_self_reference.input.yaml | 18 ++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index fb7cd7c51b..ea79054c88 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -278,7 +278,18 @@ def _push_context( """Resolve a variable, recursively resolving any dependencies it references.""" value = unresolved_vars.pop(key, Missing) if value is Missing: - return Missing + # Either already resolved (in resolved_vars) or currently being + # resolved (self-reference from inside a dict-valued substitution). + # Returning what we have lets sibling references inside a dict + # value, e.g. ``${device.manufacturer}`` inside ``device.name``, + # see literal sibling values during their own resolution. + return resolved_vars.get(key, Missing) + if isinstance(value, dict): + # Dict-valued substitutions form a namespace; eagerly publish the + # original mapping so its members can reference each other while + # the dict's own substitution pass is still running. The entry is + # replaced with the fully-substituted dict once recursion returns. + resolved_vars[key] = value try: value = substitute(value, [], resolver_context, True) except UndefinedError as err: diff --git a/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml new file mode 100644 index 0000000000..e5e6d4568e --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml @@ -0,0 +1,16 @@ +substitutions: + device: + manufacturer: espressif + model: esp32 + mac_suffix: ffffff + name: espressif-esp32-ffffff + network: + host: example.com + port: 8080 + url: http://example.com:8080/api +esphome: + name: espressif-esp32-ffffff +test_list: + - espressif-esp32-ffffff + - http://example.com:8080/api + - espressif/esp32 diff --git a/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml new file mode 100644 index 0000000000..b27c4b8c29 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml @@ -0,0 +1,18 @@ +substitutions: + device: + manufacturer: "espressif" + model: "esp32" + mac_suffix: "ffffff" + name: ${device.manufacturer}-${device.model}-${device.mac_suffix} + network: + host: "example.com" + port: 8080 + url: "http://${network.host}:${network.port}/api" + +esphome: + name: ${device.name} + +test_list: + - ${device.name} + - ${network.url} + - "${device.manufacturer}/${device.model}" From 545ee03f42e830b8ea0899d06886ab4b8f22ba85 Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Wed, 6 May 2026 10:15:04 -0400 Subject: [PATCH 40/60] [atm90e32] Fix calibration instance not saving in flash properly (#14152) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 139 ++++++++++++++++------- esphome/components/atm90e32/atm90e32.h | 3 + esphome/components/atm90e32/sensor.py | 1 + 3 files changed, 105 insertions(+), 38 deletions(-) diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index db29702c54..693b1b4961 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -1,6 +1,7 @@ #include "atm90e32.h" #include #include +#include #include #include "esphome/core/log.h" @@ -8,6 +9,25 @@ namespace esphome { namespace atm90e32 { static const char *const TAG = "atm90e32"; + +static uint32_t pref_hash(const char *prefix, const char *name_space) { + auto hash = fnv1_hash(prefix); + return fnv1_hash_extend(hash, name_space); +} + +template +static int migrate_legacy_pref_if_needed(ESPPreferenceObject ¤t_pref, ESPPreferenceObject &legacy_pref, + T *scratch) { + T current{}; + if (current_pref.load(¤t)) { + return 0; + } + if (!legacy_pref.load(scratch)) { + return 0; + } + return current_pref.save(scratch) ? 1 : -1; +} + void ATM90E32Component::loop() { if (this->get_publish_interval_flag_()) { this->set_publish_interval_flag_(false); @@ -112,10 +132,14 @@ void ATM90E32Component::get_cs_summary_(std::span bu this->cs_->dump_summary(buffer.data(), buffer.size()); } +const char *ATM90E32Component::get_calibration_id_() { return this->instance_id_; } + void ATM90E32Component::setup() { this->spi_setup(); - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); + char legacy_cs[GPIO_SUMMARY_MAX_LEN]; + this->get_cs_summary_(legacy_cs); + const bool has_distinct_legacy_namespace = strcmp(cs, legacy_cs) != 0; uint16_t mmode0 = 0x87; // 3P4W 50Hz uint16_t high_thresh = 0; @@ -162,15 +186,46 @@ void ATM90E32Component::setup() { if (this->enable_offset_calibration_) { // Initialize flash storage for offset calibrations - uint32_t o_hash = fnv1_hash("_offset_calibration_"); - o_hash = fnv1_hash_extend(o_hash, cs); + uint32_t o_hash = pref_hash("_offset_calibration_", cs); this->offset_pref_ = global_preferences->make_preference(o_hash, true); - this->restore_offset_calibrations_(); + bool migrated_offset = false; + if (has_distinct_legacy_namespace) { + uint32_t legacy_o_hash = pref_hash("_offset_calibration_", legacy_cs); + auto legacy_offset_pref = global_preferences->make_preference(legacy_o_hash, true); + OffsetCalibration offset_data[3]{}; + int migration_status = migrate_legacy_pref_if_needed(this->offset_pref_, legacy_offset_pref, &offset_data); + migrated_offset = migration_status > 0; + if (migration_status > 0) { + ESP_LOGI(TAG, "[CALIBRATION][%s] Migrated offset calibrations from legacy storage.", cs); + } else if (migration_status < 0) { + ESP_LOGW(TAG, "[CALIBRATION][%s] Failed to migrate offset calibrations from legacy storage.", cs); + } + } // Initialize flash storage for power offset calibrations - uint32_t po_hash = fnv1_hash("_power_offset_calibration_"); - po_hash = fnv1_hash_extend(po_hash, cs); + uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + bool migrated_power_offset = false; + if (has_distinct_legacy_namespace) { + uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); + auto legacy_power_offset_pref = + global_preferences->make_preference(legacy_po_hash, true); + PowerOffsetCalibration power_offset_data[3]{}; + int migration_status = + migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); + migrated_power_offset = migration_status > 0; + if (migration_status > 0) { + ESP_LOGI(TAG, "[CALIBRATION][%s] Migrated power offset calibrations from legacy storage.", cs); + } else if (migration_status < 0) { + ESP_LOGW(TAG, "[CALIBRATION][%s] Failed to migrate power offset calibrations from legacy storage.", cs); + } + } + + if (migrated_offset || migrated_power_offset) { + global_preferences->sync(); + } + + this->restore_offset_calibrations_(); this->restore_power_offset_calibrations_(); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", @@ -189,9 +244,27 @@ void ATM90E32Component::setup() { if (this->enable_gain_calibration_) { // Initialize flash storage for gain calibration - uint32_t g_hash = fnv1_hash("_gain_calibration_"); - g_hash = fnv1_hash_extend(g_hash, cs); + uint32_t g_hash = pref_hash("_gain_calibration_", cs); this->gain_calibration_pref_ = global_preferences->make_preference(g_hash, true); + bool migrated_gain = false; + if (has_distinct_legacy_namespace) { + uint32_t legacy_g_hash = pref_hash("_gain_calibration_", legacy_cs); + auto legacy_gain_calibration_pref = global_preferences->make_preference(legacy_g_hash, true); + GainCalibration gain_data[3]{}; + int migration_status = + migrate_legacy_pref_if_needed(this->gain_calibration_pref_, legacy_gain_calibration_pref, &gain_data); + migrated_gain = migration_status > 0; + if (migration_status > 0) { + ESP_LOGI(TAG, "[CALIBRATION][%s] Migrated gain calibrations from legacy storage.", cs); + } else if (migration_status < 0) { + ESP_LOGW(TAG, "[CALIBRATION][%s] Failed to migrate gain calibrations from legacy storage.", cs); + } + } + + if (migrated_gain) { + global_preferences->sync(); + } + this->restore_gain_calibrations_(); if (!this->using_saved_calibrations_) { @@ -221,8 +294,7 @@ void ATM90E32Component::setup() { } void ATM90E32Component::log_calibration_status_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); bool offset_mismatch = false; bool power_mismatch = false; @@ -573,8 +645,7 @@ float ATM90E32Component::get_chip_temperature_() { } void ATM90E32Component::run_gain_calibrations() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); if (!this->enable_gain_calibration_) { ESP_LOGW(TAG, "[CALIBRATION][%s] Gain calibration is disabled! Enable it first with enable_gain_calibration: true", cs); @@ -674,8 +745,7 @@ void ATM90E32Component::run_gain_calibrations() { } void ATM90E32Component::save_gain_calibration_to_memory_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); bool success = this->gain_calibration_pref_.save(&this->gain_phase_); global_preferences->sync(); if (success) { @@ -688,8 +758,7 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } void ATM90E32Component::save_offset_calibration_to_memory_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); bool success = this->offset_pref_.save(&this->offset_phase_); global_preferences->sync(); if (success) { @@ -705,8 +774,7 @@ void ATM90E32Component::save_offset_calibration_to_memory_() { } void ATM90E32Component::save_power_offset_calibration_to_memory_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); bool success = this->power_offset_pref_.save(&this->power_offset_phase_); global_preferences->sync(); if (success) { @@ -722,8 +790,7 @@ void ATM90E32Component::save_power_offset_calibration_to_memory_() { } void ATM90E32Component::run_offset_calibrations() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); if (!this->enable_offset_calibration_) { ESP_LOGW(TAG, "[CALIBRATION][%s] Offset calibration is disabled! Enable it first with enable_offset_calibration: true", @@ -753,8 +820,7 @@ void ATM90E32Component::run_offset_calibrations() { } void ATM90E32Component::run_power_offset_calibrations() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); if (!this->enable_offset_calibration_) { ESP_LOGW( TAG, @@ -827,15 +893,16 @@ void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t } void ATM90E32Component::restore_gain_calibrations_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); for (uint8_t i = 0; i < 3; ++i) { this->config_gain_phase_[i].voltage_gain = this->phase_[i].voltage_gain_; this->config_gain_phase_[i].current_gain = this->phase_[i].ct_gain_; this->gain_phase_[i] = this->config_gain_phase_[i]; } - if (this->gain_calibration_pref_.load(&this->gain_phase_)) { + bool have_data = this->gain_calibration_pref_.load(&this->gain_phase_); + + if (have_data) { bool all_zero = true; bool same_as_config = true; for (uint8_t phase = 0; phase < 3; ++phase) { @@ -882,12 +949,12 @@ void ATM90E32Component::restore_gain_calibrations_() { } void ATM90E32Component::restore_offset_calibrations_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); for (uint8_t i = 0; i < 3; ++i) this->config_offset_phase_[i] = this->offset_phase_[i]; bool have_data = this->offset_pref_.load(&this->offset_phase_); + bool all_zero = true; if (have_data) { for (auto &phase : this->offset_phase_) { @@ -925,12 +992,12 @@ void ATM90E32Component::restore_offset_calibrations_() { } void ATM90E32Component::restore_power_offset_calibrations_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); for (uint8_t i = 0; i < 3; ++i) this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); + bool all_zero = true; if (have_data) { for (auto &phase : this->power_offset_phase_) { @@ -968,8 +1035,7 @@ void ATM90E32Component::restore_power_offset_calibrations_() { } void ATM90E32Component::clear_gain_calibrations() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); if (!this->using_saved_calibrations_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored gain calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ----------------------------------------------------------", cs); @@ -1018,8 +1084,7 @@ void ATM90E32Component::clear_gain_calibrations() { } void ATM90E32Component::clear_offset_calibrations() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); if (!this->restored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); @@ -1061,8 +1126,7 @@ void ATM90E32Component::clear_offset_calibrations() { } void ATM90E32Component::clear_power_offset_calibrations() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); if (!this->restored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); @@ -1137,8 +1201,7 @@ int16_t ATM90E32Component::calibrate_power_offset(uint8_t phase, bool reactive) } bool ATM90E32Component::verify_gain_writes_() { - char cs[GPIO_SUMMARY_MAX_LEN]; - this->get_cs_summary_(cs); + const char *cs = this->get_calibration_id_(); bool success = true; for (uint8_t phase = 0; phase < 3; phase++) { uint16_t read_voltage = this->read16_(voltage_gain_registers[phase]); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index 95154812cb..62c7bada86 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -102,6 +102,7 @@ class ATM90E32Component : public PollingComponent, void clear_gain_calibrations(); void set_enable_offset_calibration(bool flag) { enable_offset_calibration_ = flag; } void set_enable_gain_calibration(bool flag) { enable_gain_calibration_ = flag; } + void set_instance_id(const char *id) { instance_id_ = id; } int16_t calibrate_offset(uint8_t phase, bool voltage); int16_t calibrate_power_offset(uint8_t phase, bool reactive); void run_gain_calibrations(); @@ -183,6 +184,7 @@ class ATM90E32Component : public PollingComponent, bool verify_gain_writes_(); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); + const char *get_calibration_id_(); void get_cs_summary_(std::span buffer); struct ATM90E32Phase { @@ -263,6 +265,7 @@ class ATM90E32Component : public PollingComponent, bool peak_current_signed_{false}; bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; + const char *instance_id_{nullptr}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/esphome/components/atm90e32/sensor.py b/esphome/components/atm90e32/sensor.py index 7e5d85c57a..dc46138add 100644 --- a/esphome/components/atm90e32/sensor.py +++ b/esphome/components/atm90e32/sensor.py @@ -193,6 +193,7 @@ CONFIG_SCHEMA = ( async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) + cg.add(var.set_instance_id(str(config[CONF_ID]))) await cg.register_component(var, config) await spi.register_spi_device(var, config) From 6e1a59da3e583c5624a18da82b62d85256e7f0f4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 09:53:14 -0500 Subject: [PATCH 41/60] [packages] Make package !include vars visible to its substitutions block (#16274) --- esphome/components/packages/__init__.py | 59 +++++++++++++++---- .../18-package_vars_in_subs.approved.yaml | 9 +++ .../18-package_vars_in_subs.input.yaml | 9 +++ .../18-package_vars_in_subs_inc.yaml | 8 +++ 4 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs_inc.yaml diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index d63f17aa7e..06a64208b6 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -414,25 +414,39 @@ def _substitute_package_definition( def _update_substitutions_context( parent_context: UserDict, package_substitutions: dict[str, Any], + eval_context: ContextVars | None = None, ) -> None: """Resolve and add new substitutions to the parent context. Skips keys already present (higher-priority sources win). - String values are substituted against the current context so that - cross-references between substitutions are expanded when possible. + String values are substituted against *eval_context* (or *parent_context* + if not provided) so that cross-references between substitutions are + expanded when possible. Resolved values are written into *parent_context* + and back into *package_substitutions* so that subsequent merges into the + consolidated ``substitutions:`` block carry the resolved value (the + package's ``!include vars`` are no longer in scope after this function + returns). + + *eval_context* may layer additional vars (e.g. a package's own ``!include + vars``) on top of *parent_context* so that a package's substitutions can + reference vars passed in by the parent file. """ + if eval_context is None: + eval_context = ContextVars(parent_context) for key, value in package_substitutions.items(): if key in parent_context: continue if not isinstance(value, str): parent_context[key] = value continue - parent_context[key] = substitute( + resolved = substitute( item=value, path=[CONF_SUBSTITUTIONS, key], - parent_context=ContextVars(parent_context), + parent_context=eval_context, strict_undefined=False, ) + parent_context[key] = resolved + package_substitutions[key] = resolved class _PackageProcessor: @@ -508,11 +522,36 @@ class _PackageProcessor: package_config = _process_remote_package(package_config) return package_config - def collect_substitutions(self, package_config: dict) -> None: - """Extract substitutions from a package and merge into the shared context.""" + def collect_substitutions( + self, + package_config: dict, + context_vars: ContextVars | None, + ) -> ContextVars: + """Extract substitutions from a package and merge into the shared context. + + Returns the context updated with the package's ``!include vars`` (or + an equivalent of *context_vars* if the package has none) so the caller + can reuse it when recursing into nested packages. ``None`` inputs are + normalized to an empty :class:`ContextVars`, so the result is always + non-``None``. + """ + # Push the package's own !include vars before evaluating its + # substitutions so they can reference vars passed in by the parent + # (e.g. ``vars: {my_variable: ...}`` on the include entry). + package_context = push_context( + package_config, context_vars if context_vars is not None else ContextVars() + ) if subs := package_config.pop(CONF_SUBSTITUTIONS, {}): + # Resolve before merging so that values referencing the package's + # ``!include vars`` are baked into the consolidated substitutions + # block; once we return, the package vars are no longer in scope. + # ``package_context`` is a ChainMap whose chain already terminates + # in ``self.parent_context`` (set up by ``do_packages_pass``), so + # ``parent_context`` mutations from ``_update_substitutions_context`` + # remain visible to evaluation reads. + _update_substitutions_context(self.parent_context, subs, package_context) self.substitutions.data = merge_config(subs, self.substitutions.data) - _update_substitutions_context(self.parent_context, subs) + return package_context def process_package( self, @@ -525,13 +564,13 @@ class _PackageProcessor: package_config ) package_config = self.resolve_package(package_config, context_vars, path) - self.collect_substitutions(package_config) + context_vars = self.collect_substitutions(package_config, context_vars) if CONF_PACKAGES not in package_config: return package_config - # Push context from !include vars on the package root and on the packages key - context_vars = push_context(package_config, context_vars) + # Push context from !include vars on the packages key (the package root + # was already pushed in collect_substitutions above). context_vars = push_context(package_config[CONF_PACKAGES], context_vars) # Disable the deprecated single-package fallback for remote # packages. _process_remote_package returns dicts with diff --git a/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.approved.yaml b/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.approved.yaml new file mode 100644 index 0000000000..647a33a983 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.approved.yaml @@ -0,0 +1,9 @@ +binary_sensor: + - platform: template + id: front_door_enrolling + name: Front Door Enrolling +substitutions: + enrolling_id: front_door_enrolling + enrolling_name: Front Door Enrolling +esphome: + name: test diff --git a/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.input.yaml b/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.input.yaml new file mode 100644 index 0000000000..21a0f2d235 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs.input.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +packages: + fingerprint: !include + file: 18-package_vars_in_subs_inc.yaml + vars: + sensor_name: "Front Door" + sensor_id_prefix: "front_door" diff --git a/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs_inc.yaml b/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs_inc.yaml new file mode 100644 index 0000000000..8b420d73e7 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-package_vars_in_subs_inc.yaml @@ -0,0 +1,8 @@ +substitutions: + enrolling_id: ${sensor_id_prefix}_enrolling + enrolling_name: ${sensor_name} Enrolling + +binary_sensor: + - platform: template + id: ${enrolling_id} + name: ${enrolling_name} From 90693fb39aac35c2303e8b6165fa4638085dbd8d Mon Sep 17 00:00:00 2001 From: Mat931 <49403702+Mat931@users.noreply.github.com> Date: Wed, 6 May 2026 14:56:33 +0000 Subject: [PATCH 42/60] [core] Fix WiFi connection in safe mode (#16269) Co-authored-by: J. Nick Koston Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/core/config.py | 27 ++--- tests/integration/conftest.py | 111 +++++++++--------- .../fixtures/safe_mode_loop_runs.yaml | 25 ++++ tests/integration/host_prefs.py | 39 ++++++ tests/integration/test_safe_mode_loop_runs.py | 94 +++++++++++++++ tests/integration/test_template_text_save.py | 15 +-- tests/unit_tests/core/test_config.py | 70 +++++++++++ 7 files changed, 299 insertions(+), 82 deletions(-) create mode 100644 tests/integration/fixtures/safe_mode_loop_runs.yaml create mode 100644 tests/integration/host_prefs.py create mode 100644 tests/integration/test_safe_mode_loop_runs.py diff --git a/esphome/core/config.py b/esphome/core/config.py index b4e81ce49f..fe55c0fe25 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -568,14 +568,9 @@ async def _add_controller_registry_define() -> None: @coroutine_with_priority(CoroPriority.FINAL) async def _add_looping_components() -> None: - # Emit a constexpr that computes the looping component count at C++ compile time - # and pre-init the FixedVector with the exact capacity. Uses std::is_same_v to - # detect loop() overrides. The constexpr goes in main.cpp's global section where - # all component types are in scope. calculate_looping_components_() then skips - # the counting pass and only does the two population passes. + # Emit ESPHOME_LOOPING_COMPONENT_COUNT. Sizing of looping_components_ + # happens in core to_code() so it lands before safe_mode's early return. entries = CORE.data.get("looping_component_entries", []) - if not entries: - return # Build constexpr sum for the exact count, deduplicating by type # Uses HasLoopOverride which handles ambiguous &T::loop from multiple inheritance @@ -583,7 +578,7 @@ async def _add_looping_components() -> None: terms = [ f"({count} * HasLoopOverride<{cpp_type}>::value)" for cpp_type, count in type_counts.items() - ] + ] or ["0"] constexpr_expr = " + \\\n ".join(terms) cg.add_global( cg.RawStatement( @@ -592,14 +587,6 @@ async def _add_looping_components() -> None: ) ) - # Pre-init FixedVector with exact capacity so calculate_looping_components_() - # can skip the counting pass - cg.add( - cg.RawExpression( - "App.looping_components_.init(ESPHOME_LOOPING_COMPONENT_COUNT)" - ) - ) - @coroutine_with_priority(CoroPriority.CORE) async def to_code(config: ConfigType) -> None: @@ -642,6 +629,14 @@ async def to_code(config: ConfigType) -> None: # Define component count for static allocation cg.add_define("ESPHOME_COMPONENT_COUNT", len(CORE.component_ids)) + # Pre-init FixedVector with exact capacity so calculate_looping_components_() + # can skip the counting pass + cg.add( + cg.RawExpression( + "App.looping_components_.init(ESPHOME_LOOPING_COMPONENT_COUNT)" + ) + ) + CORE.add_job(_add_platform_defines) CORE.add_job(_add_controller_registry_define) CORE.add_job(_add_looping_components) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7c85bf753c..f36543b7cd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -501,14 +501,15 @@ async def _read_stream_lines( @asynccontextmanager -async def run_binary_and_wait_for_port( +async def run_binary( binary_path: Path, - host: str, - port: int, - timeout: float = PORT_WAIT_TIMEOUT, line_callback: Callable[[str], None] | None = None, -) -> AsyncGenerator[None]: - """Run a binary, wait for it to open a port, and clean up on exit.""" +) -> AsyncGenerator[tuple[asyncio.subprocess.Process, list[str]]]: + """Run a binary under a PTY, capture log output, and clean up on exit. + + Yields the running ``Process`` and a live list of captured log lines. + No port wait -- callers that need that should use + ``run_binary_and_wait_for_port``.""" # Create a pseudo-terminal to make the binary think it's running interactively # This is needed because the ESPHome host logger checks isatty() controller_fd, device_fd = pty.openpty() @@ -535,7 +536,6 @@ async def run_binary_and_wait_for_port( controller_transport, _ = await loop.connect_read_pipe( lambda: controller_protocol, os.fdopen(controller_fd, "rb", 0) ) - output_reader = controller_reader if process.returncode is not None: raise RuntimeError( @@ -543,27 +543,59 @@ async def run_binary_and_wait_for_port( "Ensure the binary is valid and can run successfully." ) - # Wait for the API server to start listening - loop = asyncio.get_running_loop() - start_time = loop.time() - - # Start collecting output stdout_lines: list[str] = [] - output_tasks: list[asyncio.Task] = [] + output_task = asyncio.create_task( + _read_stream_lines(controller_reader, stdout_lines, sys.stdout, line_callback) + ) try: - # Read from output stream - output_tasks = [ - asyncio.create_task( - _read_stream_lines( - output_reader, stdout_lines, sys.stdout, line_callback - ) - ) - ] - # Small yield to ensure the process has a chance to start await asyncio.sleep(0) + yield process, stdout_lines + finally: + output_task.cancel() + result = await asyncio.gather(output_task, return_exceptions=True) + if isinstance(result[0], Exception) and not isinstance( + result[0], asyncio.CancelledError + ): + print(f"Error reading from PTY: {result[0]}", file=sys.stderr) + # Close the PTY transport (Unix only) + if controller_transport is not None: + controller_transport.close() + + # Cleanup: terminate the process gracefully + if process.returncode is None: + # Send SIGINT (Ctrl+C) for graceful shutdown + process.send_signal(signal.SIGINT) + try: + await asyncio.wait_for(process.wait(), timeout=SIGINT_TIMEOUT) + except TimeoutError: + # If SIGINT didn't work, try SIGTERM + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=SIGTERM_TIMEOUT) + except TimeoutError: + # Last resort: SIGKILL + process.kill() + await process.wait() + + +@asynccontextmanager +async def run_binary_and_wait_for_port( + binary_path: Path, + host: str, + port: int, + timeout: float = PORT_WAIT_TIMEOUT, + line_callback: Callable[[str], None] | None = None, +) -> AsyncGenerator[None]: + """Run a binary, wait for it to open a port, and clean up on exit.""" + async with run_binary(binary_path, line_callback=line_callback) as ( + process, + stdout_lines, + ): + loop = asyncio.get_running_loop() + start_time = loop.time() while loop.time() - start_time < timeout: try: # Try to connect to the port @@ -593,41 +625,6 @@ async def run_binary_and_wait_for_port( raise TimeoutError(error_msg) - finally: - # Cancel output collection tasks - for task in output_tasks: - task.cancel() - # Wait for tasks to complete and check for exceptions - results = await asyncio.gather(*output_tasks, return_exceptions=True) - for i, result in enumerate(results): - if isinstance(result, Exception) and not isinstance( - result, asyncio.CancelledError - ): - print( - f"Error reading from PTY: {result}", - file=sys.stderr, - ) - - # Close the PTY transport (Unix only) - if controller_transport is not None: - controller_transport.close() - - # Cleanup: terminate the process gracefully - if process.returncode is None: - # Send SIGINT (Ctrl+C) for graceful shutdown - process.send_signal(signal.SIGINT) - try: - await asyncio.wait_for(process.wait(), timeout=SIGINT_TIMEOUT) - except TimeoutError: - # If SIGINT didn't work, try SIGTERM - process.terminate() - try: - await asyncio.wait_for(process.wait(), timeout=SIGTERM_TIMEOUT) - except TimeoutError: - # Last resort: SIGKILL - process.kill() - await process.wait() - @asynccontextmanager async def run_compiled_context( diff --git a/tests/integration/fixtures/safe_mode_loop_runs.yaml b/tests/integration/fixtures/safe_mode_loop_runs.yaml new file mode 100644 index 0000000000..342622428b --- /dev/null +++ b/tests/integration/fixtures/safe_mode_loop_runs.yaml @@ -0,0 +1,25 @@ +esphome: + name: safe-mode-loop-runs + +host: + +logger: + +safe_mode: + num_attempts: 10 + on_safe_mode: + - lambda: |- + // Spawn a detached thread that logs a unique marker. The + // non-main-thread log goes through the task log buffer, which + // is only drained by Logger::loop(). If looping components + // weren't initialized (the bug fixed in #16269), the buffer is + // never read and the marker never reaches the console. + struct MarkerThread { + static void *thread_func(void *) { + ESP_LOGI("safe_mode_test", "looping component ran in safe mode"); + return nullptr; + } + }; + pthread_t t; + pthread_create(&t, nullptr, MarkerThread::thread_func, nullptr); + pthread_detach(t); diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py new file mode 100644 index 0000000000..f835bee3bc --- /dev/null +++ b/tests/integration/host_prefs.py @@ -0,0 +1,39 @@ +"""Helpers for manipulating the host platform's preferences file. + +ESPHome's host platform stores preferences in +``~/.esphome/prefs/.prefs`` using a simple binary layout that +mirrors ``HostPreferences::sync()``: +``[uint32_t key][uint8_t len][uint8_t data[len]]`` per entry. + +Tests use these helpers to pre-populate state the binary will see at +boot (e.g. forcing safe mode) or to clear stale state between runs. +""" + +from __future__ import annotations + +from pathlib import Path +import struct + + +def host_prefs_path(device_name: str) -> Path: + """Return the on-disk prefs file path for a host-platform device.""" + return Path.home() / ".esphome" / "prefs" / f"{device_name}.prefs" + + +def clear_host_prefs(device_name: str) -> None: + """Delete the prefs file for a host-platform device, if it exists.""" + host_prefs_path(device_name).unlink(missing_ok=True) + + +def write_host_pref(device_name: str, key: int, data: bytes) -> Path: + """Write a single preference entry, replacing the file's contents. + + Returns the path that was written. + """ + if len(data) > 255: + raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") + path = host_prefs_path(device_name) + path.parent.mkdir(parents=True, exist_ok=True) + payload = struct.pack(" None: + """When safe mode is active, ``App.loop()`` must still iterate looping + components -- proven here by a thread-logged marker reaching the + console (which requires ``Logger::loop()`` to run).""" + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + + # Compile finished successfully; pre-populate prefs so the *next* run + # enters safe mode immediately. + write_host_pref( + DEVICE_NAME, SAFE_MODE_RTC_KEY, struct.pack(" None: + if not safe_mode_active.done() and safe_mode_pattern.search(line): + safe_mode_active.set_result(True) + if not thread_log_seen.done() and thread_log_pattern.search(line): + thread_log_seen.set_result(True) + + async with run_binary(binary_path, line_callback=on_log): + try: + await asyncio.wait_for(safe_mode_active, timeout=15.0) + except TimeoutError: + pytest.fail( + "Did not observe 'SAFE MODE IS ACTIVE' -- safe mode " + "didn't trigger, so this test isn't exercising the bug." + ) + try: + await asyncio.wait_for(thread_log_seen, timeout=10.0) + except TimeoutError: + pytest.fail( + f"Did not observe thread-logged marker {THREAD_LOG_MARKER!r} " + "within timeout. Logger::loop() never drained the task " + "log buffer, meaning App.looping_components_ was never " + "sized -- this is the regression #16269 fixed." + ) + finally: + clear_host_prefs(DEVICE_NAME) diff --git a/tests/integration/test_template_text_save.py b/tests/integration/test_template_text_save.py index 47c8e3188a..7e56209c50 100644 --- a/tests/integration/test_template_text_save.py +++ b/tests/integration/test_template_text_save.py @@ -9,7 +9,6 @@ Tests that: from __future__ import annotations import asyncio -from pathlib import Path import socket from typing import Any @@ -17,9 +16,12 @@ from aioesphomeapi import TextInfo, TextState import pytest from .conftest import run_binary_and_wait_for_port, wait_and_connect_api_client +from .host_prefs import clear_host_prefs from .state_utils import InitialStateHelper, require_entity from .types import CompileFunction, ConfigWriter +DEVICE_NAME = "host-template-text-save-test" + @pytest.mark.asyncio async def test_template_text_save( @@ -32,11 +34,7 @@ async def test_template_text_save( port, port_socket = reserved_tcp_port # Clean up any stale preference file from previous runs - prefs_file = ( - Path.home() / ".esphome" / "prefs" / "host-template-text-save-test.prefs" - ) - if prefs_file.exists(): - prefs_file.unlink() + clear_host_prefs(DEVICE_NAME) # Write and compile once config_path = await write_yaml_config(yaml_config) @@ -59,7 +57,7 @@ async def test_template_text_save( wait_and_connect_api_client(port=port) as client, ): device_info = await client.device_info() - assert device_info.name == "host-template-text-save-test" + assert device_info.name == DEVICE_NAME entities, _ = await client.list_entities_services() text_entity = require_entity( @@ -127,5 +125,4 @@ async def test_template_text_save( ) # Clean up preference file - if prefs_file.exists(): - prefs_file.unlink() + clear_host_prefs(DEVICE_NAME) diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 6fa8f7ed43..4ce862315d 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest from esphome import config_validation as cv, core +from esphome.components.safe_mode import to_code as safe_mode_to_code from esphome.const import ( CONF_AREA, CONF_AREAS, @@ -312,6 +313,75 @@ def test_add_platform_defines_priority() -> None: ) +def test_to_code_priority_above_safe_mode() -> None: + """Test that core to_code emits the looping_components_ init before safe_mode. + + Regression test for https://github.com/esphome/esphome/issues/16262. + safe_mode emits an `if (should_enter_safe_mode(...)) return;` line in main() + at APPLICATION priority. The `App.looping_components_.init(...)` call must be + emitted at a higher priority than APPLICATION so it lands in main() before + the early return; otherwise the FixedVector is never sized when safe mode is + active and loop() never runs (Wi-Fi never connects). + """ + assert config.to_code.priority > safe_mode_to_code.priority, ( + f"core to_code priority ({config.to_code.priority}) must be greater than " + f"safe_mode to_code priority ({safe_mode_to_code.priority}) so that " + "App.looping_components_.init() is emitted before safe_mode's early return" + ) + + +@pytest.mark.asyncio +async def test_add_looping_components_handles_empty_entries() -> None: + """Test that _add_looping_components emits a valid constexpr when there are + no looping component entries. + + With zero entries the generated constexpr must still be syntactically valid + C++ (`= 0;`), not an empty expression (`= ;`). This guards the empty-list + case that would otherwise produce uncompilable main.cpp output. + """ + CORE.data["looping_component_entries"] = [] + + await config._add_looping_components() + + constexpr_lines = [ + str(s) + for s in CORE.global_statements + if "ESPHOME_LOOPING_COMPONENT_COUNT" in str(s) + ] + assert len(constexpr_lines) == 1 + text = constexpr_lines[0] + assert "static constexpr size_t ESPHOME_LOOPING_COMPONENT_COUNT" in text + # The right-hand side must contain a literal `0`, not be empty. + rhs = text.split("=", 1)[1] + assert "0" in rhs + assert rhs.strip().rstrip(";").strip(), ( + f"constexpr right-hand side must not be empty, got: {text!r}" + ) + + +@pytest.mark.asyncio +async def test_add_looping_components_with_entries() -> None: + """Test that _add_looping_components builds a HasLoopOverride sum from entries.""" + CORE.data["looping_component_entries"] = [ + "esphome::wifi::WiFiComponent", + "esphome::logger::Logger", + "esphome::wifi::WiFiComponent", + ] + + await config._add_looping_components() + + constexpr_lines = [ + str(s) + for s in CORE.global_statements + if "ESPHOME_LOOPING_COMPONENT_COUNT" in str(s) + ] + assert len(constexpr_lines) == 1 + text = constexpr_lines[0] + # Deduplicated by type, with per-type counts as multiplier. + assert "(2 * HasLoopOverride::value)" in text + assert "(1 * HasLoopOverride::value)" in text + + def test_valid_include_with_angle_brackets() -> None: """Test valid_include accepts angle bracket includes.""" assert valid_include("") == "" From 2864922ac05311159596877c51fc7464cec8fa1b Mon Sep 17 00:00:00 2001 From: Mat931 <49403702+Mat931@users.noreply.github.com> Date: Wed, 6 May 2026 14:59:10 +0000 Subject: [PATCH 43/60] [ota] Partition table update: Fix log messages (#16241) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../components/esphome/ota/ota_esphome.cpp | 4 +-- .../components/ota/ota_backend_esp_idf.cpp | 3 +- .../components/ota/ota_partitions_esp_idf.cpp | 33 ++++++++++++------- esphome/espota2.py | 6 ++-- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 3ce3f2302d..5d3deca489 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -117,8 +117,8 @@ void ESPHomeOTAComponent::dump_config() { " Partition table:\n" " %-12s %-4s %-8s %-10s %-10s", "Name", "Type", "Subtype", "Address", "Size"); - esp_partition_iterator_t it = esp_partition_find(ESP_PARTITION_TYPE_ANY, ESP_PARTITION_SUBTYPE_ANY, NULL); - while (it != NULL) { + esp_partition_iterator_t it = esp_partition_find(ESP_PARTITION_TYPE_ANY, ESP_PARTITION_SUBTYPE_ANY, nullptr); + while (it != nullptr) { const esp_partition_t *partition = esp_partition_get(it); ESP_LOGCONFIG(TAG, " %-12s 0x%-2X 0x%-6X 0x%-8" PRIX32 " 0x%-8" PRIX32, partition->label, partition->type, partition->subtype, partition->address, partition->size); diff --git a/esphome/components/ota/ota_backend_esp_idf.cpp b/esphome/components/ota/ota_backend_esp_idf.cpp index 42d106bf1f..50a0988ba2 100644 --- a/esphome/components/ota/ota_backend_esp_idf.cpp +++ b/esphome/components/ota/ota_backend_esp_idf.cpp @@ -20,8 +20,7 @@ OTAResponseTypes IDFOTABackend::begin(size_t image_size, ota::OTAType ota_type) #ifdef USE_OTA_PARTITIONS this->ota_type_ = ota_type; if (this->ota_type_ == ota::OTA_TYPE_UPDATE_PARTITION_TABLE) { - // Reject any size other than ESP_PARTITION_TABLE_MAX_LEN: under- leaves stale bytes from the - // previous table; over- can't fit the reserved region. + // Reject any size other than ESP_PARTITION_TABLE_MAX_LEN if (image_size != ESP_PARTITION_TABLE_MAX_LEN) { ESP_LOGE(TAG, "Wrong partition table size: expected %u bytes, got %zu", ESP_PARTITION_TABLE_MAX_LEN, image_size); return OTA_RESPONSE_ERROR_PARTITION_TABLE_VERIFY; diff --git a/esphome/components/ota/ota_partitions_esp_idf.cpp b/esphome/components/ota/ota_partitions_esp_idf.cpp index 2a2ed577f1..f7fd529986 100644 --- a/esphome/components/ota/ota_partitions_esp_idf.cpp +++ b/esphome/components/ota/ota_partitions_esp_idf.cpp @@ -11,6 +11,7 @@ #include #include +#include #include namespace esphome::ota { @@ -135,10 +136,20 @@ OTAResponseTypes IDFOTABackend::validate_new_partition_table_(uint32_t running_a // Rejecting here is non-destructive (no flash op has run yet); the user can safely retry with // a different .bin. Log enough info that they can pick the right method without guessing. ESP_LOGE(TAG, - "Running app at 0x%X (%u bytes used) does not fit any compatible slot in the new " - "partition table. Pick a migration method whose size limit is at least %u bytes and " - "retry; no flash content was modified.", - running_app_offset, running_app_size, running_app_size); + "The new partition table must contain a compatible app partition with:\n" + " size: at least %" PRIu32 " bytes (0x%" PRIX32 ")\n" + " address: one of", + (uint32_t) running_app_size, (uint32_t) running_app_size); + esp_partition_iterator_t it = esp_partition_find(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_ANY, nullptr); + while (it != nullptr) { + const esp_partition_t *partition = esp_partition_get(it); + if (partition->size >= running_app_size) { + ESP_LOGE(TAG, " 0x%" PRIX32, partition->address); + } + it = esp_partition_next(it); + } + esp_partition_iterator_release(it); + ESP_LOGE(TAG, "Upload a different partition table. No flash content was modified."); return OTA_RESPONSE_ERROR_PARTITION_TABLE_VERIFY; } if (app_partitions_found < 2) { @@ -154,11 +165,11 @@ OTAResponseTypes IDFOTABackend::validate_new_partition_table_(uint32_t running_a return OTA_RESPONSE_ERROR_PARTITION_TABLE_VERIFY; } if (otadata_overlap) { + // Unlikely, the otadata partition is before the start of the first app partition in most cases ESP_LOGE(TAG, - "New otadata partition overlaps with the running app at 0x%X (size %u). The chosen " - "partition table is not compatible with this device's current flash layout; pick a " - "different migration method.", - running_app_offset, running_app_size); + "New otadata partition overlaps with the running app at address: 0x%" PRIX32 ", running app size: %" PRIu32 + " bytes", + running_app_offset, (uint32_t) running_app_size); return OTA_RESPONSE_ERROR_PARTITION_TABLE_VERIFY; } @@ -198,8 +209,8 @@ OTAResponseTypes IDFOTABackend::update_partition_table() { // can leave the device unbootable until it is recovered with a serial flash. ESP_LOGE(TAG, "Starting partition table update.\n" " DO NOT REMOVE POWER until the device reboots successfully.\n" - " Loss of power during this operation may render the device unable to boot until\n" - " it is recovered via a serial flash."); + " Loss of power during this operation may render the device\n" + " unable to boot until it is recovered via a serial flash."); // One guard over the whole critical section in case an IDF call takes longer than expected on // some chip variant. @@ -214,7 +225,7 @@ OTAResponseTypes IDFOTABackend::update_partition_table() { // which leaves esp_ota_get_running_partition() returning nullptr. const esp_partition_t *running_app_part = find_app_partition_at(running_app_offset, running_app_size); if (running_app_part == nullptr) { - ESP_LOGE(TAG, "Cannot resolve running app partition at offset 0x%X", running_app_offset); + ESP_LOGE(TAG, "Cannot resolve running app partition at address 0x%" PRIX32, running_app_offset); return OTA_RESPONSE_ERROR_PARTITION_TABLE_UPDATE; } ESP_LOGD(TAG, "Copying running app from 0x%X to 0x%X (size: 0x%X)", running_app_part->address, diff --git a/esphome/espota2.py b/esphome/espota2.py index a45a6ef234..b2a1fd2a40 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -139,9 +139,9 @@ _ERROR_MESSAGES: dict[int, str] = { ), RESPONSE_ERROR_PARTITION_TABLE_UPDATE: ( "An error occurred while updating the partition table. The device is now " - "in a degraded state (NVS handles are invalid; many components will fail) " - "and may not be able to boot. Check the logs, reboot the device, and " - "retry the update. If the device fails to boot, recover it via a serial flash." + "in a degraded state and may not be able to boot. Open the logs and retry " + "the partition table update without rebooting the device. If the device " + "fails to boot, recover it via a serial flash." ), RESPONSE_ERROR_UNKNOWN: "Unknown error from ESP", } From cfd2c9182c9cf7355cd4c536f64bc81e7073073c Mon Sep 17 00:00:00 2001 From: Didier A Date: Wed, 6 May 2026 18:34:55 +0200 Subject: [PATCH 44/60] [bl0942] Remove broken 24-bit overflow tracking (#15650) Co-authored-by: DidierA <1620015+didiera@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/bl0942/bl0942.cpp | 8 ++------ esphome/components/bl0942/bl0942.h | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/esphome/components/bl0942/bl0942.cpp b/esphome/components/bl0942/bl0942.cpp index 7d38597423..074aff9643 100644 --- a/esphome/components/bl0942/bl0942.cpp +++ b/esphome/components/bl0942/bl0942.cpp @@ -161,13 +161,9 @@ void BL0942::received_package_(DataPacket *data) { return; } - // cf_cnt is only 24 bits, so track overflows + // cf_cnt wraps at 24 bits; total_increasing on the energy sensor handles the + // wrap (and any spurious chip resets) downstream. uint32_t cf_cnt = (uint24_t) data->cf_cnt; - cf_cnt |= this->prev_cf_cnt_ & 0xff000000; - if (cf_cnt < this->prev_cf_cnt_) { - cf_cnt += 0x1000000; - } - this->prev_cf_cnt_ = cf_cnt; float v_rms = (uint24_t) data->v_rms / voltage_reference_; float i_rms = (uint24_t) data->i_rms / current_reference_; diff --git a/esphome/components/bl0942/bl0942.h b/esphome/components/bl0942/bl0942.h index 3c013f86e7..7604399c25 100644 --- a/esphome/components/bl0942/bl0942.h +++ b/esphome/components/bl0942/bl0942.h @@ -141,7 +141,6 @@ class BL0942 : public PollingComponent, public uart::UARTDevice { bool reset_ = false; LineFrequency line_freq_ = LINE_FREQUENCY_50HZ; optional rx_start_{}; - uint32_t prev_cf_cnt_ = 0; bool validate_checksum_(DataPacket *data); int read_reg_(uint8_t reg); From a4a57a540d38b009dff46996203ea7249fda10a8 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Wed, 6 May 2026 12:56:54 -0400 Subject: [PATCH 45/60] [core] Adds acquire and release methods to the ring buffer class (#16277) --- esphome/core/ring_buffer.cpp | 8 ++++++++ esphome/core/ring_buffer.h | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/esphome/core/ring_buffer.cpp b/esphome/core/ring_buffer.cpp index 2e0802eceb..486cf67f25 100644 --- a/esphome/core/ring_buffer.cpp +++ b/esphome/core/ring_buffer.cpp @@ -37,6 +37,14 @@ std::unique_ptr RingBuffer::create(size_t len, MemoryPreference pref return rb; } +void *RingBuffer::receive_acquire(size_t &length, size_t max_length, TickType_t ticks_to_wait) { + length = 0; + void *buffer_data = xRingbufferReceiveUpTo(this->handle_, &length, ticks_to_wait, max_length); + return buffer_data; +} + +void RingBuffer::receive_release(void *item) { vRingbufferReturnItem(this->handle_, item); } + size_t RingBuffer::read(void *data, size_t len, TickType_t ticks_to_wait) { size_t bytes_read = 0; diff --git a/esphome/core/ring_buffer.h b/esphome/core/ring_buffer.h index 4acd07d5b0..8ac3ff3811 100644 --- a/esphome/core/ring_buffer.h +++ b/esphome/core/ring_buffer.h @@ -27,6 +27,28 @@ class RingBuffer { */ size_t read(void *data, size_t len, TickType_t ticks_to_wait = 0); + /** + * @brief Acquires a pointer into the ring buffer's internal storage without copying. + * + * The returned pointer is valid until receive_release() is called. Only one item + * may be checked out at a time. + * + * @param[out] length Set to the number of bytes actually acquired (may be less than max_length at wrap boundary) + * @param max_length Maximum number of bytes to acquire + * @param ticks_to_wait Maximum number of FreeRTOS ticks to wait (default: 0) + * @return Pointer into the ring buffer's internal storage, or nullptr if no data is available + */ + void *receive_acquire(size_t &length, size_t max_length, TickType_t ticks_to_wait = 0); + + /** + * @brief Releases a previously acquired ring buffer item. + * + * Must be called exactly once for each successful receive_acquire(). + * + * @param item Pointer returned by receive_acquire() + */ + void receive_release(void *item); + /** * @brief Writes to the ring buffer, overwriting oldest data if necessary. * From fc25ab0246b1575d3e1bb2ea69dbc4546a72f239 Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Wed, 6 May 2026 12:57:03 -0400 Subject: [PATCH 46/60] [i2s_audio] Optimize software volume control (#16278) --- .../i2s_audio/speaker/i2s_audio_speaker.cpp | 51 ++++++------------- .../i2s_audio/speaker/i2s_audio_speaker.h | 2 +- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 836221e38a..58d17ea6c4 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -13,22 +13,16 @@ #include "esp_timer.h" +// esp-audio-libs +#include + namespace esphome::i2s_audio { static const char *const TAG = "i2s_audio.speaker"; -// Lists the Q15 fixed point scaling factor for volume reduction. -// Has 100 values representing silence and a reduction [49, 48.5, ... 0.5, 0] dB. -// dB to PCM scaling factor formula: floating_point_scale_factor = 2^(-db/6.014) -// float to Q15 fixed point formula: q15_scale_factor = floating_point_scale_factor * 2^(15) -static const std::vector Q15_VOLUME_SCALING_FACTORS = { - 0, 116, 122, 130, 137, 146, 154, 163, 173, 183, 194, 206, 218, 231, 244, - 259, 274, 291, 308, 326, 345, 366, 388, 411, 435, 461, 488, 517, 548, 580, - 615, 651, 690, 731, 774, 820, 868, 920, 974, 1032, 1094, 1158, 1227, 1300, 1377, - 1459, 1545, 1637, 1734, 1837, 1946, 2061, 2184, 2313, 2450, 2596, 2750, 2913, 3085, 3269, - 3462, 3668, 3885, 4116, 4360, 4619, 4893, 5183, 5490, 5816, 6161, 6527, 6914, 7324, 7758, - 8218, 8706, 9222, 9770, 10349, 10963, 11613, 12302, 13032, 13805, 14624, 15491, 16410, 17384, 18415, - 19508, 20665, 21891, 23189, 24565, 26022, 27566, 29201, 30933, 32767}; +// Software volume control maps the user-facing [0.0, 1.0] range to a Q31 scale factor. +// Volumes in (0.0, 1.0) map linearly to a dB reduction in [-49.0, 0.0] dB. +static constexpr float SOFTWARE_VOLUME_MIN_DB = -49.0f; void I2SAudioSpeakerBase::setup() { this->event_group_ = xEventGroupCreate(); @@ -147,14 +141,16 @@ void I2SAudioSpeakerBase::set_volume(float volume) { } else #endif // USE_AUDIO_DAC { - // Fallback to software volume control by using a Q15 fixed point scaling factor. - // At maximum volume (1.0), set to INT16_MAX to completely bypass volume processing + // Fallback to software volume control by using a Q31 fixed point scaling factor. + // At maximum volume (1.0), set to INT32_MAX to bypass volume processing entirely // and avoid any floating-point precision issues that could cause slight volume reduction. if (volume >= 1.0f) { - this->q15_volume_factor_ = INT16_MAX; + this->q31_volume_factor_ = INT32_MAX; + } else if (volume <= 0.0f) { + this->q31_volume_factor_ = 0; } else { - ssize_t decibel_index = remap(volume, 0.0f, 1.0f, 0, Q15_VOLUME_SCALING_FACTORS.size() - 1); - this->q15_volume_factor_ = Q15_VOLUME_SCALING_FACTORS[decibel_index]; + this->q31_volume_factor_ = + esp_audio_libs::gain::db_to_q31(remap(volume, 0.0f, 1.0f, SOFTWARE_VOLUME_MIN_DB, 0.0f)); } } } @@ -173,7 +169,7 @@ void I2SAudioSpeakerBase::set_mute_state(bool mute_state) { { if (mute_state) { // Fallback to software volume control and scale by 0 - this->q15_volume_factor_ = 0; + this->q31_volume_factor_ = 0; } else { // Revert to previous volume when unmuting this->set_volume(this->volume_); @@ -309,29 +305,14 @@ bool IRAM_ATTR I2SAudioSpeakerBase::i2s_on_sent_cb(i2s_chan_handle_t handle, i2s } void I2SAudioSpeakerBase::apply_software_volume_(uint8_t *data, size_t bytes_read) { - if (this->q15_volume_factor_ >= INT16_MAX) { + if (this->q31_volume_factor_ == INT32_MAX) { return; // Max volume, no processing needed } const size_t bytes_per_sample = this->current_stream_info_.samples_to_bytes(1); const uint32_t len = bytes_read / bytes_per_sample; - // Use Q16 for samples with 1 or 2 bytes: shifted_sample * gain_factor is Q16 * Q15 -> Q31 - int32_t shift = 15; // Q31 -> Q16 - int32_t gain_factor = this->q15_volume_factor_; // Q15 - - if (bytes_per_sample >= 3) { - // Use Q23 for samples with 3 or 4 bytes: shifted_sample * gain_factor is Q23 * Q8 -> Q31 - shift = 8; // Q31 -> Q23 - gain_factor >>= 7; // Q15 -> Q8 - } - - for (uint32_t i = 0; i < len; ++i) { - int32_t sample = audio::unpack_audio_sample_to_q31(&data[i * bytes_per_sample], bytes_per_sample); // Q31 - sample >>= shift; - sample *= gain_factor; // Q31 - audio::pack_q31_as_audio_sample(sample, &data[i * bytes_per_sample], bytes_per_sample); - } + esp_audio_libs::gain::apply(data, data, this->q31_volume_factor_, len, bytes_per_sample); } void I2SAudioSpeakerBase::swap_esp32_mono_samples_(uint8_t *data, size_t bytes_read) { diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h index b2644efd05..d9a228ef2c 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h @@ -151,7 +151,7 @@ class I2SAudioSpeakerBase : public I2SAudioOut, public speaker::Speaker, public bool pause_state_{false}; - int16_t q15_volume_factor_{INT16_MAX}; + int32_t q31_volume_factor_{INT32_MAX}; audio::AudioStreamInfo current_stream_info_; // The currently loaded driver's stream info From 9f49e3f80e75d2309eaf6766222aa622a458516f Mon Sep 17 00:00:00 2001 From: Kevin Ahrendt Date: Wed, 6 May 2026 13:22:18 -0400 Subject: [PATCH 47/60] [audio] Bump microFLAC to v0.2.0 (#16279) --- esphome/components/audio/__init__.py | 2 +- esphome/idf_component.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index cfb2ad4e75..67ef2e7d1a 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -386,7 +386,7 @@ async def to_code(config): # Adds a define and IDF component for legacy `audio_decoder.cpp`. if data.flac_support: cg.add_define("USE_AUDIO_FLAC_SUPPORT") - add_idf_component(name="esphome/micro-flac", ref="0.1.1") + add_idf_component(name="esphome/micro-flac", ref="0.2.0") _emit_memory_pair( data.flac.buffer_memory, "CONFIG_MICRO_FLAC_PREFER_PSRAM", diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 37c0da11f5..8ffcffa705 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -8,7 +8,7 @@ dependencies: esphome/micro-decoder: version: 0.2.0 esphome/micro-flac: - version: 0.1.1 + version: 0.2.0 esphome/micro-mp3: version: 0.2.0 esphome/micro-opus: From 4da62067cf5864ebcd8448c2c7ee8aed16ccdd11 Mon Sep 17 00:00:00 2001 From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com> Date: Wed, 6 May 2026 19:32:50 +0200 Subject: [PATCH 48/60] [nextion] Fix text sensor state not updated on string response (#16280) --- esphome/components/nextion/nextion.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp index e42f7ca216..b644cad507 100644 --- a/esphome/components/nextion/nextion.cpp +++ b/esphome/components/nextion/nextion.cpp @@ -641,6 +641,7 @@ void Nextion::process_nextion_commands_() { } else { ESP_LOGN(TAG, "String resp: '%s' id: %s type: %s", to_process.c_str(), component->get_variable_name().c_str(), component->get_queue_type_string()); + component->set_state_from_string(to_process, true, false); } delete nb; // NOLINT(cppcoreguidelines-owning-memory) From 0d94ffe15dbde9cd0bb8f852b7167e14417bdbda Mon Sep 17 00:00:00 2001 From: dbl-0 Date: Wed, 6 May 2026 11:48:38 -0600 Subject: [PATCH 49/60] [resolver] Make RESOLVE_TIMEOUT configurable via environment variable (#15951) Co-authored-by: Daniel Lowe Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/resolver.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/esphome/resolver.py b/esphome/resolver.py index 9fb596ce7b..f80a910afe 100644 --- a/esphome/resolver.py +++ b/esphome/resolver.py @@ -2,13 +2,28 @@ from __future__ import annotations +import logging +import os + from aioesphomeapi.core import ResolveAPIError, ResolveTimeoutAPIError import aioesphomeapi.host_resolver as hr from esphome.async_thread import AsyncThreadRunner from esphome.core import EsphomeError -RESOLVE_TIMEOUT = 10.0 # seconds +_LOGGER = logging.getLogger(__name__) + +_DEFAULT_RESOLVE_TIMEOUT = 20.0 +_env_timeout = os.environ.get("ESPHOME_RESOLVE_TIMEOUT", _DEFAULT_RESOLVE_TIMEOUT) +try: + RESOLVE_TIMEOUT = float(_env_timeout) +except ValueError: + _LOGGER.warning( + "ESPHOME_RESOLVE_TIMEOUT=%r is not a valid number; using default %.1fs", + _env_timeout, + _DEFAULT_RESOLVE_TIMEOUT, + ) + RESOLVE_TIMEOUT = _DEFAULT_RESOLVE_TIMEOUT class AsyncResolver: From 6173656bf8db35a563071cddfb3180304bf0a9fe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 12:49:00 -0500 Subject: [PATCH 50/60] [schema] Surface OnlyWith / OnlyWithout default + gate components in schema generator (#16276) --- script/build_language_schema.py | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 09ff999901..05ac47bfcc 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -1065,7 +1065,40 @@ def convert_keys(converted, schema, path): else: converted["key_type"] = str(k) - if hasattr(k, "default") and str(k.default) != "...": + # ``cv.OnlyWith`` / ``cv.OnlyWithout`` expose ``default`` as + # a property that returns ``vol.UNDEFINED`` when the gating + # component isn't loaded — and at schema-generation time + # ``CORE.loaded_integrations`` is always empty, so the + # property never resolves. The unconditional default lives + # on ``_default``; expose it under a *new* per-class field + # (``default_with`` for ``OnlyWith``, ``default_without`` for + # ``OnlyWithout``) that bundles the value with the gating + # component(s). Pure addition to the bundle — old consumers + # that read only ``default`` see these fields as + # default-less (same as today, no regression where they used + # to fall back to a hard-coded UI default); new consumers + # opt-in to the gated fields and apply the default + # *conditionally* on which integrations the user has + # loaded. Without the gate info, an ethernet-only config on + # ``cv.OnlyWith(K, "wifi", default=True)`` would otherwise + # render ``True`` even though ESPHome itself wouldn't apply + # the default for that config. + if isinstance(k, (cv.OnlyWith, cv.OnlyWithout)): + default_value = k._default() + if default_value is not None: + components = ( + list(k._component) + if isinstance(k._component, list) + else [k._component] + ) + gate_field = ( + "default_with" if isinstance(k, cv.OnlyWith) else "default_without" + ) + result[gate_field] = { + "value": str(default_value), + "components": components, + } + elif hasattr(k, "default") and str(k.default) != "...": default_value = k.default() if default_value is not None: result["default"] = str(default_value) From 1e58e8729a0a73117bbfa533f0a498ffb3e228f6 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 6 May 2026 14:53:48 -0400 Subject: [PATCH 51/60] [uart] Use `tcdrain` for flushing instead of`tcflush` on host (#14877) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/uart/uart_component_host.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/uart/uart_component_host.cpp b/esphome/components/uart/uart_component_host.cpp index 085610a983..5bb7a49726 100644 --- a/esphome/components/uart/uart_component_host.cpp +++ b/esphome/components/uart/uart_component_host.cpp @@ -276,9 +276,12 @@ UARTFlushResult HostUartComponent::flush() { if (this->file_descriptor_ == -1) { return UARTFlushResult::UART_FLUSH_RESULT_ASSUMED_SUCCESS; } - tcflush(this->file_descriptor_, TCIOFLUSH); ESP_LOGV(TAG, " Flushing"); - return UARTFlushResult::UART_FLUSH_RESULT_ASSUMED_SUCCESS; + if (tcdrain(this->file_descriptor_) == -1) { + this->update_error_(strerror(errno)); + return UARTFlushResult::UART_FLUSH_RESULT_FAILED; + } + return UARTFlushResult::UART_FLUSH_RESULT_SUCCESS; } void HostUartComponent::update_error_(const std::string &error) { From d2bbaeccf33359f9b96010aae17736f5224ba659 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 6 May 2026 21:42:11 +1200 Subject: [PATCH 52/60] [ha-addon] Add opt-in toggle for the new ESPHome Device Builder (#16247) --- .../etc/cont-init.d/40-device-builder.sh | 22 +++++++++++++++++++ .../etc/s6-overlay/s6-rc.d/esphome/run | 7 ++++++ .../etc/s6-overlay/s6-rc.d/init-nginx/run | 8 +++++++ .../etc/s6-overlay/s6-rc.d/nginx/run | 8 +++++++ 4 files changed, 45 insertions(+) create mode 100755 docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh diff --git a/docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh b/docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh new file mode 100755 index 0000000000..b990469762 --- /dev/null +++ b/docker/ha-addon-rootfs/etc/cont-init.d/40-device-builder.sh @@ -0,0 +1,22 @@ +#!/usr/bin/with-contenv bashio +# ============================================================================== +# Installs the latest prerelease of esphome-device-builder when the +# `use_new_device_builder` config option is enabled. +# This is a temporary install-on-boot step until esphome-device-builder +# becomes a direct dependency of esphome. +# ============================================================================== + +if ! bashio::config.true 'use_new_device_builder'; then + exit 0 +fi + +bashio::log.info "Installing latest prerelease of esphome-device-builder..." +if command -v uv > /dev/null; then + uv pip install --system --no-cache-dir --prerelease=allow --upgrade \ + esphome-device-builder || + bashio::exit.nok "Failed installing esphome-device-builder." +else + pip install --no-cache-dir --pre --upgrade esphome-device-builder || + bashio::exit.nok "Failed installing esphome-device-builder." +fi +bashio::log.info "Installed esphome-device-builder." diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run index cdbaff6c04..64ac0b18d2 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run @@ -49,5 +49,12 @@ if bashio::fs.directory_exists '/config/esphome/.esphome'; then rm -rf /config/esphome/.esphome fi +if bashio::config.true 'use_new_device_builder'; then + bashio::log.info "Starting ESPHome Device Builder..." + exec esphome-device-builder /config/esphome \ + --ha-addon \ + --ingress-port "$(bashio::addon.ingress_port)" +fi + bashio::log.info "Starting ESPHome dashboard..." exec esphome dashboard /config/esphome --socket /var/run/esphome.sock --ha-addon diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run index 2725f56670..18c75898ec 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/init-nginx/run @@ -4,6 +4,14 @@ # Community Hass.io Add-ons: ESPHome # Configures NGINX for use with ESPHome # ============================================================================== + +# When the new device builder is enabled it serves HA ingress directly, +# so nginx is not used at all -- skip configuration. +if bashio::config.true 'use_new_device_builder'; then + bashio::log.info "Skipping NGINX setup: new device builder serves ingress directly." + bashio::exit.ok +fi + mkdir -p /var/log/nginx # Generate Ingress configuration diff --git a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run index e96991cdd1..bb5f52e10c 100755 --- a/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run +++ b/docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/nginx/run @@ -5,6 +5,14 @@ # Runs the NGINX proxy # ============================================================================== +# The new device builder handles HA ingress itself, so nginx is bypassed. +# Block the longrun forever so s6 keeps the dependency satisfied and does +# not respawn it. +if bashio::config.true 'use_new_device_builder'; then + bashio::log.info "NGINX bypassed: new device builder serves ingress directly." + exec sleep infinity +fi + bashio::log.info "Waiting for ESPHome dashboard to come up..." while [[ ! -S /var/run/esphome.sock ]]; do From 016b509b551610d8fefd3fa90f1baf1b387940c2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 07:32:35 -0500 Subject: [PATCH 53/60] [bundle] Include secrets.yaml when `!secret` keys are quoted (#16271) --- esphome/bundle.py | 12 +++++---- tests/unit_tests/test_bundle.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/esphome/bundle.py b/esphome/bundle.py index efa80acc8c..70c4fad0fd 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -98,11 +98,13 @@ _KNOWN_FILE_EXTENSIONS = frozenset( ) -# Matches !secret references in YAML text. This is intentionally a simple -# regex scan rather than a YAML parse — it may match inside comments or -# multi-line strings, which is the conservative direction (include more -# secrets rather than fewer). -_SECRET_RE = re.compile(r"!secret\s+(\S+)") +# Matches !secret references in YAML text. An optional surrounding +# quote pair around the key is allowed and ignored: YAML treats +# ``!secret 'foo'`` and ``!secret foo`` as the same key. This is +# intentionally a simple regex scan rather than a YAML parse — it may +# match inside comments or multi-line strings, which is the conservative +# direction (include more secrets rather than fewer). +_SECRET_RE = re.compile(r"""!secret\s+['"]?([^\s'"]+)""") def _find_used_secret_keys(yaml_files: list[Path]) -> set[str]: diff --git a/tests/unit_tests/test_bundle.py b/tests/unit_tests/test_bundle.py index 89bf1a33b3..5d046252da 100644 --- a/tests/unit_tests/test_bundle.py +++ b/tests/unit_tests/test_bundle.py @@ -170,6 +170,23 @@ def test_find_used_secret_keys_deduplicates(tmp_path: Path) -> None: assert keys == {"key1"} +def test_find_used_secret_keys_quoted(tmp_path: Path) -> None: + """Quoted !secret keys should resolve to the same key as unquoted form. + + YAML strips surrounding quotes during parsing, so the secrets.yaml + lookup uses the unquoted key. The bundle scan must do the same. + """ + yaml1 = tmp_path / "a.yaml" + yaml1.write_text( + "single: !secret 'wifi_ssid'\n" + 'double: !secret "wifi_pw"\n' + "bare: !secret api_key\n" + ) + + keys = _find_used_secret_keys([yaml1]) + assert keys == {"wifi_ssid", "wifi_pw", "api_key"} + + # --------------------------------------------------------------------------- # _add_bytes_to_tar # --------------------------------------------------------------------------- @@ -1217,6 +1234,35 @@ def test_create_bundle_filters_secrets(tmp_path: Path) -> None: assert "should_not_appear" not in secrets_data +def test_create_bundle_filters_secrets_quoted(tmp_path: Path) -> None: + """Bundling must include secrets.yaml when !secret keys are quoted. + + Regression test for issue 16259: quoted !secret references previously + captured the quotes as part of the key, so no key matched secrets.yaml + entries and the secrets file was dropped from the bundle entirely. + """ + config_dir = _setup_config_dir(tmp_path) + + secrets = config_dir / "secrets.yaml" + secrets.write_text("ota_password: hunter2\nunused: should_not_appear\n") + + config_yaml = "ota:\n password: !secret 'ota_password'\n" + (config_dir / "test.yaml").write_text(config_yaml) + + creator = ConfigBundleCreator({}) + result = creator.create_bundle() + + assert result.manifest[ManifestKey.HAS_SECRETS] is True + + buf = io.BytesIO(result.data) + with tarfile.open(fileobj=buf, mode="r:gz") as tar: + secrets_data = tar.extractfile("secrets.yaml").read().decode() + + assert "ota_password" in secrets_data + assert "hunter2" in secrets_data + assert "unused" not in secrets_data + + def test_create_bundle_no_secrets(tmp_path: Path) -> None: _setup_config_dir(tmp_path) From 7f6aef4f33f882936ab2e68008ed6079c7a27508 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 6 May 2026 08:41:17 -0500 Subject: [PATCH 54/60] [substitutions] Fix sibling references inside dict-valued substitutions (#16273) --- esphome/components/substitutions/__init__.py | 13 ++++++++++++- .../18-dict_self_reference.approved.yaml | 16 ++++++++++++++++ .../18-dict_self_reference.input.yaml | 18 ++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 0144c13c01..af261fe2a3 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -297,7 +297,18 @@ def _push_context( """Resolve a variable, recursively resolving any dependencies it references.""" value = unresolved_vars.pop(key, Missing) if value is Missing: - return Missing + # Either already resolved (in resolved_vars) or currently being + # resolved (self-reference from inside a dict-valued substitution). + # Returning what we have lets sibling references inside a dict + # value, e.g. ``${device.manufacturer}`` inside ``device.name``, + # see literal sibling values during their own resolution. + return resolved_vars.get(key, Missing) + if isinstance(value, dict): + # Dict-valued substitutions form a namespace; eagerly publish the + # original mapping so its members can reference each other while + # the dict's own substitution pass is still running. The entry is + # replaced with the fully-substituted dict once recursion returns. + resolved_vars[key] = value try: value = substitute(value, [], resolver_context, True) except UndefinedError as err: diff --git a/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml new file mode 100644 index 0000000000..e5e6d4568e --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.approved.yaml @@ -0,0 +1,16 @@ +substitutions: + device: + manufacturer: espressif + model: esp32 + mac_suffix: ffffff + name: espressif-esp32-ffffff + network: + host: example.com + port: 8080 + url: http://example.com:8080/api +esphome: + name: espressif-esp32-ffffff +test_list: + - espressif-esp32-ffffff + - http://example.com:8080/api + - espressif/esp32 diff --git a/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml new file mode 100644 index 0000000000..b27c4b8c29 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/18-dict_self_reference.input.yaml @@ -0,0 +1,18 @@ +substitutions: + device: + manufacturer: "espressif" + model: "esp32" + mac_suffix: "ffffff" + name: ${device.manufacturer}-${device.model}-${device.mac_suffix} + network: + host: "example.com" + port: 8080 + url: "http://${network.host}:${network.port}/api" + +esphome: + name: ${device.name} + +test_list: + - ${device.name} + - ${network.url} + - "${device.manufacturer}/${device.model}" From b89c71c1eac2c99412540da1a3e9c1117275c093 Mon Sep 17 00:00:00 2001 From: Mat931 <49403702+Mat931@users.noreply.github.com> Date: Wed, 6 May 2026 14:56:33 +0000 Subject: [PATCH 55/60] [core] Fix WiFi connection in safe mode (#16269) Co-authored-by: J. Nick Koston Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/core/config.py | 27 ++--- tests/integration/conftest.py | 111 +++++++++--------- .../fixtures/safe_mode_loop_runs.yaml | 25 ++++ tests/integration/host_prefs.py | 39 ++++++ tests/integration/test_safe_mode_loop_runs.py | 94 +++++++++++++++ tests/integration/test_template_text_save.py | 15 +-- tests/unit_tests/core/test_config.py | 70 +++++++++++ 7 files changed, 299 insertions(+), 82 deletions(-) create mode 100644 tests/integration/fixtures/safe_mode_loop_runs.yaml create mode 100644 tests/integration/host_prefs.py create mode 100644 tests/integration/test_safe_mode_loop_runs.py diff --git a/esphome/core/config.py b/esphome/core/config.py index bf210876df..70c28a0368 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -562,14 +562,9 @@ async def _add_controller_registry_define() -> None: @coroutine_with_priority(CoroPriority.FINAL) async def _add_looping_components() -> None: - # Emit a constexpr that computes the looping component count at C++ compile time - # and pre-init the FixedVector with the exact capacity. Uses std::is_same_v to - # detect loop() overrides. The constexpr goes in main.cpp's global section where - # all component types are in scope. calculate_looping_components_() then skips - # the counting pass and only does the two population passes. + # Emit ESPHOME_LOOPING_COMPONENT_COUNT. Sizing of looping_components_ + # happens in core to_code() so it lands before safe_mode's early return. entries = CORE.data.get("looping_component_entries", []) - if not entries: - return # Build constexpr sum for the exact count, deduplicating by type # Uses HasLoopOverride which handles ambiguous &T::loop from multiple inheritance @@ -577,7 +572,7 @@ async def _add_looping_components() -> None: terms = [ f"({count} * HasLoopOverride<{cpp_type}>::value)" for cpp_type, count in type_counts.items() - ] + ] or ["0"] constexpr_expr = " + \\\n ".join(terms) cg.add_global( cg.RawStatement( @@ -586,14 +581,6 @@ async def _add_looping_components() -> None: ) ) - # Pre-init FixedVector with exact capacity so calculate_looping_components_() - # can skip the counting pass - cg.add( - cg.RawExpression( - "App.looping_components_.init(ESPHOME_LOOPING_COMPONENT_COUNT)" - ) - ) - @coroutine_with_priority(CoroPriority.CORE) async def to_code(config: ConfigType) -> None: @@ -636,6 +623,14 @@ async def to_code(config: ConfigType) -> None: # Define component count for static allocation cg.add_define("ESPHOME_COMPONENT_COUNT", len(CORE.component_ids)) + # Pre-init FixedVector with exact capacity so calculate_looping_components_() + # can skip the counting pass + cg.add( + cg.RawExpression( + "App.looping_components_.init(ESPHOME_LOOPING_COMPONENT_COUNT)" + ) + ) + CORE.add_job(_add_platform_defines) CORE.add_job(_add_controller_registry_define) CORE.add_job(_add_looping_components) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7c85bf753c..f36543b7cd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -501,14 +501,15 @@ async def _read_stream_lines( @asynccontextmanager -async def run_binary_and_wait_for_port( +async def run_binary( binary_path: Path, - host: str, - port: int, - timeout: float = PORT_WAIT_TIMEOUT, line_callback: Callable[[str], None] | None = None, -) -> AsyncGenerator[None]: - """Run a binary, wait for it to open a port, and clean up on exit.""" +) -> AsyncGenerator[tuple[asyncio.subprocess.Process, list[str]]]: + """Run a binary under a PTY, capture log output, and clean up on exit. + + Yields the running ``Process`` and a live list of captured log lines. + No port wait -- callers that need that should use + ``run_binary_and_wait_for_port``.""" # Create a pseudo-terminal to make the binary think it's running interactively # This is needed because the ESPHome host logger checks isatty() controller_fd, device_fd = pty.openpty() @@ -535,7 +536,6 @@ async def run_binary_and_wait_for_port( controller_transport, _ = await loop.connect_read_pipe( lambda: controller_protocol, os.fdopen(controller_fd, "rb", 0) ) - output_reader = controller_reader if process.returncode is not None: raise RuntimeError( @@ -543,27 +543,59 @@ async def run_binary_and_wait_for_port( "Ensure the binary is valid and can run successfully." ) - # Wait for the API server to start listening - loop = asyncio.get_running_loop() - start_time = loop.time() - - # Start collecting output stdout_lines: list[str] = [] - output_tasks: list[asyncio.Task] = [] + output_task = asyncio.create_task( + _read_stream_lines(controller_reader, stdout_lines, sys.stdout, line_callback) + ) try: - # Read from output stream - output_tasks = [ - asyncio.create_task( - _read_stream_lines( - output_reader, stdout_lines, sys.stdout, line_callback - ) - ) - ] - # Small yield to ensure the process has a chance to start await asyncio.sleep(0) + yield process, stdout_lines + finally: + output_task.cancel() + result = await asyncio.gather(output_task, return_exceptions=True) + if isinstance(result[0], Exception) and not isinstance( + result[0], asyncio.CancelledError + ): + print(f"Error reading from PTY: {result[0]}", file=sys.stderr) + # Close the PTY transport (Unix only) + if controller_transport is not None: + controller_transport.close() + + # Cleanup: terminate the process gracefully + if process.returncode is None: + # Send SIGINT (Ctrl+C) for graceful shutdown + process.send_signal(signal.SIGINT) + try: + await asyncio.wait_for(process.wait(), timeout=SIGINT_TIMEOUT) + except TimeoutError: + # If SIGINT didn't work, try SIGTERM + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=SIGTERM_TIMEOUT) + except TimeoutError: + # Last resort: SIGKILL + process.kill() + await process.wait() + + +@asynccontextmanager +async def run_binary_and_wait_for_port( + binary_path: Path, + host: str, + port: int, + timeout: float = PORT_WAIT_TIMEOUT, + line_callback: Callable[[str], None] | None = None, +) -> AsyncGenerator[None]: + """Run a binary, wait for it to open a port, and clean up on exit.""" + async with run_binary(binary_path, line_callback=line_callback) as ( + process, + stdout_lines, + ): + loop = asyncio.get_running_loop() + start_time = loop.time() while loop.time() - start_time < timeout: try: # Try to connect to the port @@ -593,41 +625,6 @@ async def run_binary_and_wait_for_port( raise TimeoutError(error_msg) - finally: - # Cancel output collection tasks - for task in output_tasks: - task.cancel() - # Wait for tasks to complete and check for exceptions - results = await asyncio.gather(*output_tasks, return_exceptions=True) - for i, result in enumerate(results): - if isinstance(result, Exception) and not isinstance( - result, asyncio.CancelledError - ): - print( - f"Error reading from PTY: {result}", - file=sys.stderr, - ) - - # Close the PTY transport (Unix only) - if controller_transport is not None: - controller_transport.close() - - # Cleanup: terminate the process gracefully - if process.returncode is None: - # Send SIGINT (Ctrl+C) for graceful shutdown - process.send_signal(signal.SIGINT) - try: - await asyncio.wait_for(process.wait(), timeout=SIGINT_TIMEOUT) - except TimeoutError: - # If SIGINT didn't work, try SIGTERM - process.terminate() - try: - await asyncio.wait_for(process.wait(), timeout=SIGTERM_TIMEOUT) - except TimeoutError: - # Last resort: SIGKILL - process.kill() - await process.wait() - @asynccontextmanager async def run_compiled_context( diff --git a/tests/integration/fixtures/safe_mode_loop_runs.yaml b/tests/integration/fixtures/safe_mode_loop_runs.yaml new file mode 100644 index 0000000000..342622428b --- /dev/null +++ b/tests/integration/fixtures/safe_mode_loop_runs.yaml @@ -0,0 +1,25 @@ +esphome: + name: safe-mode-loop-runs + +host: + +logger: + +safe_mode: + num_attempts: 10 + on_safe_mode: + - lambda: |- + // Spawn a detached thread that logs a unique marker. The + // non-main-thread log goes through the task log buffer, which + // is only drained by Logger::loop(). If looping components + // weren't initialized (the bug fixed in #16269), the buffer is + // never read and the marker never reaches the console. + struct MarkerThread { + static void *thread_func(void *) { + ESP_LOGI("safe_mode_test", "looping component ran in safe mode"); + return nullptr; + } + }; + pthread_t t; + pthread_create(&t, nullptr, MarkerThread::thread_func, nullptr); + pthread_detach(t); diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py new file mode 100644 index 0000000000..f835bee3bc --- /dev/null +++ b/tests/integration/host_prefs.py @@ -0,0 +1,39 @@ +"""Helpers for manipulating the host platform's preferences file. + +ESPHome's host platform stores preferences in +``~/.esphome/prefs/.prefs`` using a simple binary layout that +mirrors ``HostPreferences::sync()``: +``[uint32_t key][uint8_t len][uint8_t data[len]]`` per entry. + +Tests use these helpers to pre-populate state the binary will see at +boot (e.g. forcing safe mode) or to clear stale state between runs. +""" + +from __future__ import annotations + +from pathlib import Path +import struct + + +def host_prefs_path(device_name: str) -> Path: + """Return the on-disk prefs file path for a host-platform device.""" + return Path.home() / ".esphome" / "prefs" / f"{device_name}.prefs" + + +def clear_host_prefs(device_name: str) -> None: + """Delete the prefs file for a host-platform device, if it exists.""" + host_prefs_path(device_name).unlink(missing_ok=True) + + +def write_host_pref(device_name: str, key: int, data: bytes) -> Path: + """Write a single preference entry, replacing the file's contents. + + Returns the path that was written. + """ + if len(data) > 255: + raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") + path = host_prefs_path(device_name) + path.parent.mkdir(parents=True, exist_ok=True) + payload = struct.pack(" None: + """When safe mode is active, ``App.loop()`` must still iterate looping + components -- proven here by a thread-logged marker reaching the + console (which requires ``Logger::loop()`` to run).""" + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + + # Compile finished successfully; pre-populate prefs so the *next* run + # enters safe mode immediately. + write_host_pref( + DEVICE_NAME, SAFE_MODE_RTC_KEY, struct.pack(" None: + if not safe_mode_active.done() and safe_mode_pattern.search(line): + safe_mode_active.set_result(True) + if not thread_log_seen.done() and thread_log_pattern.search(line): + thread_log_seen.set_result(True) + + async with run_binary(binary_path, line_callback=on_log): + try: + await asyncio.wait_for(safe_mode_active, timeout=15.0) + except TimeoutError: + pytest.fail( + "Did not observe 'SAFE MODE IS ACTIVE' -- safe mode " + "didn't trigger, so this test isn't exercising the bug." + ) + try: + await asyncio.wait_for(thread_log_seen, timeout=10.0) + except TimeoutError: + pytest.fail( + f"Did not observe thread-logged marker {THREAD_LOG_MARKER!r} " + "within timeout. Logger::loop() never drained the task " + "log buffer, meaning App.looping_components_ was never " + "sized -- this is the regression #16269 fixed." + ) + finally: + clear_host_prefs(DEVICE_NAME) diff --git a/tests/integration/test_template_text_save.py b/tests/integration/test_template_text_save.py index 47c8e3188a..7e56209c50 100644 --- a/tests/integration/test_template_text_save.py +++ b/tests/integration/test_template_text_save.py @@ -9,7 +9,6 @@ Tests that: from __future__ import annotations import asyncio -from pathlib import Path import socket from typing import Any @@ -17,9 +16,12 @@ from aioesphomeapi import TextInfo, TextState import pytest from .conftest import run_binary_and_wait_for_port, wait_and_connect_api_client +from .host_prefs import clear_host_prefs from .state_utils import InitialStateHelper, require_entity from .types import CompileFunction, ConfigWriter +DEVICE_NAME = "host-template-text-save-test" + @pytest.mark.asyncio async def test_template_text_save( @@ -32,11 +34,7 @@ async def test_template_text_save( port, port_socket = reserved_tcp_port # Clean up any stale preference file from previous runs - prefs_file = ( - Path.home() / ".esphome" / "prefs" / "host-template-text-save-test.prefs" - ) - if prefs_file.exists(): - prefs_file.unlink() + clear_host_prefs(DEVICE_NAME) # Write and compile once config_path = await write_yaml_config(yaml_config) @@ -59,7 +57,7 @@ async def test_template_text_save( wait_and_connect_api_client(port=port) as client, ): device_info = await client.device_info() - assert device_info.name == "host-template-text-save-test" + assert device_info.name == DEVICE_NAME entities, _ = await client.list_entities_services() text_entity = require_entity( @@ -127,5 +125,4 @@ async def test_template_text_save( ) # Clean up preference file - if prefs_file.exists(): - prefs_file.unlink() + clear_host_prefs(DEVICE_NAME) diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 6fa8f7ed43..4ce862315d 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest from esphome import config_validation as cv, core +from esphome.components.safe_mode import to_code as safe_mode_to_code from esphome.const import ( CONF_AREA, CONF_AREAS, @@ -312,6 +313,75 @@ def test_add_platform_defines_priority() -> None: ) +def test_to_code_priority_above_safe_mode() -> None: + """Test that core to_code emits the looping_components_ init before safe_mode. + + Regression test for https://github.com/esphome/esphome/issues/16262. + safe_mode emits an `if (should_enter_safe_mode(...)) return;` line in main() + at APPLICATION priority. The `App.looping_components_.init(...)` call must be + emitted at a higher priority than APPLICATION so it lands in main() before + the early return; otherwise the FixedVector is never sized when safe mode is + active and loop() never runs (Wi-Fi never connects). + """ + assert config.to_code.priority > safe_mode_to_code.priority, ( + f"core to_code priority ({config.to_code.priority}) must be greater than " + f"safe_mode to_code priority ({safe_mode_to_code.priority}) so that " + "App.looping_components_.init() is emitted before safe_mode's early return" + ) + + +@pytest.mark.asyncio +async def test_add_looping_components_handles_empty_entries() -> None: + """Test that _add_looping_components emits a valid constexpr when there are + no looping component entries. + + With zero entries the generated constexpr must still be syntactically valid + C++ (`= 0;`), not an empty expression (`= ;`). This guards the empty-list + case that would otherwise produce uncompilable main.cpp output. + """ + CORE.data["looping_component_entries"] = [] + + await config._add_looping_components() + + constexpr_lines = [ + str(s) + for s in CORE.global_statements + if "ESPHOME_LOOPING_COMPONENT_COUNT" in str(s) + ] + assert len(constexpr_lines) == 1 + text = constexpr_lines[0] + assert "static constexpr size_t ESPHOME_LOOPING_COMPONENT_COUNT" in text + # The right-hand side must contain a literal `0`, not be empty. + rhs = text.split("=", 1)[1] + assert "0" in rhs + assert rhs.strip().rstrip(";").strip(), ( + f"constexpr right-hand side must not be empty, got: {text!r}" + ) + + +@pytest.mark.asyncio +async def test_add_looping_components_with_entries() -> None: + """Test that _add_looping_components builds a HasLoopOverride sum from entries.""" + CORE.data["looping_component_entries"] = [ + "esphome::wifi::WiFiComponent", + "esphome::logger::Logger", + "esphome::wifi::WiFiComponent", + ] + + await config._add_looping_components() + + constexpr_lines = [ + str(s) + for s in CORE.global_statements + if "ESPHOME_LOOPING_COMPONENT_COUNT" in str(s) + ] + assert len(constexpr_lines) == 1 + text = constexpr_lines[0] + # Deduplicated by type, with per-type counts as multiplier. + assert "(2 * HasLoopOverride::value)" in text + assert "(1 * HasLoopOverride::value)" in text + + def test_valid_include_with_angle_brackets() -> None: """Test valid_include accepts angle bracket includes.""" assert valid_include("") == "" From d9835c8705492923dfaa7bbb0413066b119a101b Mon Sep 17 00:00:00 2001 From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com> Date: Wed, 6 May 2026 19:32:50 +0200 Subject: [PATCH 56/60] [nextion] Fix text sensor state not updated on string response (#16280) --- esphome/components/nextion/nextion.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp index e42f7ca216..b644cad507 100644 --- a/esphome/components/nextion/nextion.cpp +++ b/esphome/components/nextion/nextion.cpp @@ -641,6 +641,7 @@ void Nextion::process_nextion_commands_() { } else { ESP_LOGN(TAG, "String resp: '%s' id: %s type: %s", to_process.c_str(), component->get_variable_name().c_str(), component->get_queue_type_string()); + component->set_state_from_string(to_process, true, false); } delete nb; // NOLINT(cppcoreguidelines-owning-memory) From 5283cdec124507957ac6d096d2102e4b86bb465a Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 7 May 2026 07:25:35 +1200 Subject: [PATCH 57/60] Bump version to 2026.4.5 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 2e86f54e96..98237cc228 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.4.4 +PROJECT_NUMBER = 2026.4.5 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index bc31ab36b5..2513a56635 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.4.4" +__version__ = "2026.4.5" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 3b6250bceecfa0a35388350f3483d3c2eee698d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 May 2026 15:23:58 -0500 Subject: [PATCH 58/60] Bump CodSpeedHQ/action from 4.15.0 to 4.15.1 (#16281) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87058e4fa5..f43c46dd00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -415,7 +415,7 @@ jobs: echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks - uses: CodSpeedHQ/action@c381be0bfd20e844fb45594f6aa182ffcd94545c # v4.15.0 + uses: CodSpeedHQ/action@3194d9a39c4d46684cb44bf7207fc56626aad8fd # v4.15.1 with: run: ${{ steps.build.outputs.binary }} mode: simulation From 004aa4913189ddc9ce1211d4ca5337df22d5b1fd Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 7 May 2026 06:57:53 +1000 Subject: [PATCH 59/60] [lvgl] Pass touch point to touch event lambdas (#16272) --- esphome/components/lvgl/defines.py | 8 ++++++++ esphome/components/lvgl/lvgl_esphome.cpp | 16 +++++++++++++++- esphome/components/lvgl/lvgl_esphome.h | 19 +++++++++++++------ esphome/components/lvgl/schemas.py | 13 +++++++++++-- esphome/components/lvgl/trigger.py | 17 +++++++++++------ esphome/components/lvgl/types.py | 2 ++ esphome/components/lvgl/widgets/__init__.py | 2 +- esphome/components/lvgl/widgets/canvas.py | 3 +-- esphome/components/lvgl/widgets/line.py | 4 ---- tests/components/lvgl/lvgl-package.yaml | 12 ++++++++++-- 10 files changed, 72 insertions(+), 24 deletions(-) diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index ef29a99ddd..03bbaf8ddb 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -309,6 +309,14 @@ LV_EVENT_MAP = { "STYLE_CHANGE": "STYLE_CHANGED", "TRIPLE_CLICK": "TRIPLE_CLICKED", } + +LV_PRESS_EVENTS = ("PRESS", "PRESSING", "RELEASE") + + +def is_press_event(event: str) -> bool: + return event.removeprefix("on_").upper() in LV_PRESS_EVENTS + + LV_SCREEN_EVENT_MAP = { "SCREEN_LOAD": "SCREEN_LOADED", "SCREEN_LOAD_START": "SCREEN_LOAD_START", diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index eb85faa16c..3141c5f93c 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -890,7 +890,21 @@ lv_color_t lv_grad_calculate_color(const lv_grad_dsc_t *dsc, int32_t pos) { int32_t offset = pos - stop1->frac; return lv_color_mix(stop2->color, stop1->color, range == 0 ? 0 : (offset * 255) / range); } -#endif +#endif // USE_LVGL_GRADIENT + +lv_point_t LvglComponent::get_touch_relative_to_obj(lv_obj_t *obj) { + auto *indev = lv_indev_get_act(); + if (indev == nullptr) { + return {INT32_MAX, INT32_MAX}; + } + lv_point_t point; + lv_indev_get_point(indev, &point); + lv_area_t coords; + lv_obj_get_coords(obj, &coords); + point.x -= coords.x1; + point.y -= coords.y1; + return point; +} static void lv_container_constructor(const lv_obj_class_t *class_p, lv_obj_t *obj) { LV_TRACE_OBJ_CREATE("begin"); diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index be1f150aff..32bf3ccac6 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -6,7 +6,7 @@ #endif // USE_BINARY_SENSOR #ifdef USE_IMAGE #include "esphome/components/image/image.h" -#endif // USE_LVGL_IMAGE +#endif // USE_IMAGE #ifdef USE_LVGL_ROTARY_ENCODER #include "esphome/components/rotary_encoder/rotary_encoder.h" #endif // USE_LVGL_ROTARY_ENCODER @@ -32,10 +32,10 @@ #ifdef USE_FONT #include "esphome/components/font/font.h" -#endif // USE_LVGL_FONT +#endif // USE_FONT #ifdef USE_TOUCHSCREEN #include "esphome/components/touchscreen/touchscreen.h" -#endif // USE_LVGL_TOUCHSCREEN +#endif // USE_TOUCHSCREEN #if defined(USE_LVGL_BUTTONMATRIX) || defined(USE_LVGL_KEYBOARD) #include "esphome/components/key_provider/key_provider.h" @@ -124,7 +124,8 @@ int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int32_t value); */ lv_color_t lv_grad_calculate_color(const lv_grad_dsc_t *dsc, int32_t pos); -#endif +#endif // USE_LVGL_GRADIENT + // Parent class for things that wrap an LVGL object class LvCompound { public: @@ -169,9 +170,9 @@ template class ObjUpdateAction : public Action { public: explicit ObjUpdateAction(std::function &&lamb) : lamb_(std::move(lamb)) {} + protected: void play(const Ts &...x) override { this->lamb_(x...); } - protected: std::function lamb_; }; #ifdef USE_LVGL_ANIMIMG @@ -190,6 +191,12 @@ class LvglComponent : public PollingComponent { LvglComponent(std::vector displays, float buffer_frac, bool full_refresh, int draw_rounding, bool resume_on_input, bool update_when_display_idle, RotationType rotation_type); static void static_flush_cb(lv_display_t *disp_drv, const lv_area_t *area, uint8_t *color_p); + /** + * + * @param obj A widget + * @return The position of the last indev point relative to the widget's origin. + */ + static lv_point_t get_touch_relative_to_obj(lv_obj_t *obj); float get_setup_priority() const override { return setup_priority::PROCESSOR; } void setup() override; @@ -311,9 +318,9 @@ class IdleTrigger : public Trigger<> { template class LvglAction : public Action, public Parented { public: explicit LvglAction(std::function &&lamb) : action_(std::move(lamb)) {} - void play(const Ts &...x) override { this->action_(this->parent_); } protected: + void play(const Ts &...x) override { this->action_(this->parent_); } std::function action_{}; }; diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 62117fbd32..a9427a9852 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -31,6 +31,7 @@ from .defines import ( CONF_TIME_FORMAT, LV_GRAD_DIR, get_remapped_uses, + is_press_event, ) from .helpers import CONF_IF_NAN, requires_component, validate_printf from .layout import ( @@ -46,6 +47,7 @@ from .types import ( LvType, lv_group_t, lv_obj_t, + lv_point_t, lv_pseudo_button_t, lv_style_t, ) @@ -370,13 +372,20 @@ def automation_schema(typ: LvType): if typ.has_on_value: events = events + (CONF_ON_VALUE,) args = typ.get_arg_type() - args.append(lv_event_t_ptr) + + def get_trigger_args(event): + result = args.copy() + if is_press_event(event): + result.append(lv_point_t) + result.append(lv_event_t_ptr) + return result + return { **{ cv.Optional(event): validate_automation( { cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( - Trigger.template(*args) + Trigger.template(*get_trigger_args(event)) ), } ) diff --git a/esphome/components/lvgl/trigger.py b/esphome/components/lvgl/trigger.py index f825999e8a..b3d12ed183 100644 --- a/esphome/components/lvgl/trigger.py +++ b/esphome/components/lvgl/trigger.py @@ -24,6 +24,7 @@ from .defines import ( LV_SCREEN_EVENT_MAP, LV_SCREEN_EVENT_TRIGGERS, SWIPE_TRIGGERS, + is_press_event, literal, ) from .lvcode import ( @@ -34,11 +35,10 @@ from .lvcode import ( LvConditional, lv, lv_add, - lv_event_t_ptr, lv_expr, lvgl_static, ) -from .types import LV_EVENT +from .types import LV_EVENT, lv_point_t from .widgets import LvScrActType, get_screen_active, widget_map @@ -133,19 +133,24 @@ def _get_event_literal(trigger: str | MockObj) -> MockObj: return literal("LV_EVENT_" + TRIGGER_MAP[trigger.upper()]) -async def add_trigger(conf, w, *events, is_selected=None): +async def add_trigger(conf, w, *events: str | MockObj, is_selected=None): is_selected = is_selected or w.is_selected() tid = conf[CONF_TRIGGER_ID] trigger = cg.new_Pvariable(tid) - args = w.get_args() + [(lv_event_t_ptr, "event")] - value = w.get_values() + args = w.get_args() + value: list = w.get_values() + if len(events) == 1 and is_press_event(str(events[0])): + # Make the touch point available for selected events + args.append((lv_point_t, "point")) + value.append(lvgl_static.get_touch_relative_to_obj(w.obj)) + args.extend(EVENT_ARG) await automation.build_automation(trigger, args, conf) async with LambdaContext(EVENT_ARG, where=tid) as context: with LvConditional(is_selected): lv_add(trigger.trigger(*value, literal("event"))) callback = await context.get_lambda() event_literals = [_get_event_literal(event) for event in events] - if isinstance(events[0], str) and events[0] in DISPLAY_TRIGGERS: + if str(events[0]) in DISPLAY_TRIGGERS: assert len(events) == 1 lv.display_add_event_cb( lv_expr.obj_get_display(w.obj), callback, event_literals[0], nullptr diff --git a/esphome/components/lvgl/types.py b/esphome/components/lvgl/types.py index 0c8ddfbfbd..1872ce2d32 100644 --- a/esphome/components/lvgl/types.py +++ b/esphome/components/lvgl/types.py @@ -70,6 +70,8 @@ lv_image_t = LvType("lv_image_t") lv_gradient_t = LvType("lv_grad_dsc_t") lv_event_t = LvType("lv_event_t") RotationType = lvgl_ns.enum("RotationType") +lv_point_t = cg.global_ns.struct("lv_point_t") +lv_point_precise_t = cg.global_ns.struct("lv_point_precise_t") LV_EVENT = MockObj(base="LV_EVENT_", op="") LV_STATE = MockObj(base="LV_STATE_", op="") diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index 0ac4062106..d35f84c4f2 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -366,7 +366,7 @@ class Widget: def get_args(self): if isinstance(self.type.w_type, LvType): - return self.type.w_type.args + return self.type.w_type.args.copy() return [(lv_obj_t_ptr, "obj")] def get_value(self): diff --git a/esphome/components/lvgl/widgets/canvas.py b/esphome/components/lvgl/widgets/canvas.py index 1308b82dcd..4427a3b00e 100644 --- a/esphome/components/lvgl/widgets/canvas.py +++ b/esphome/components/lvgl/widgets/canvas.py @@ -57,10 +57,9 @@ from ..lv_validation import ( ) from ..lvcode import LocalVariable, lv, lv_assign, lv_expr from ..schemas import STYLE_PROPS, TEXT_SCHEMA, point_schema, remap_property -from ..types import LvType, ObjUpdateAction +from ..types import LvType, ObjUpdateAction, lv_point_precise_t from . import Widget, WidgetType, get_widgets from .img import CONF_IMAGE -from .line import lv_point_precise_t CONF_CANVAS = "canvas" CONF_BUFFER_ID = "buffer_id" diff --git a/esphome/components/lvgl/widgets/line.py b/esphome/components/lvgl/widgets/line.py index 19f421cbbd..9d6aa7b4ad 100644 --- a/esphome/components/lvgl/widgets/line.py +++ b/esphome/components/lvgl/widgets/line.py @@ -1,4 +1,3 @@ -import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_X, CONF_Y @@ -13,9 +12,6 @@ CONF_LINE = "line" CONF_POINTS = "points" CONF_POINT_LIST_ID = "point_list_id" -lv_point_t = cg.global_ns.struct("lv_point_t") -lv_point_precise_t = cg.global_ns.struct("lv_point_precise_t") - class LineType(WidgetType): def __init__(self): diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 39d7472054..4bf5b9d494 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -649,11 +649,15 @@ lvgl: on_scroll_begin: logger.log: Button clicked on_release: - logger.log: Button clicked + logger.log: + format: Button released at %d/%d + args: [point.x, point.y] on_long_press_repeat: logger.log: Button clicked on_pressing: - logger.log: Button pressing + logger.log: + format: Button pressing at %d/%d + args: [point.x, point.y] on_press_lost: logger.log: Button press lost on_single_click: @@ -925,6 +929,10 @@ lvgl: value: !lambda |- static float yyy = 83.0; return yyy + .8; + on_release: + logger.log: + format: Slider released at %d/%d with value %.0f + args: [point.x, point.y, x] - button: styles: spin_button id: spin_up From 9301f76482de58997712cefbe4391d5b9bca74ac Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 7 May 2026 06:59:22 +1000 Subject: [PATCH 60/60] [sensor] Add alternate calibration format for ntc (#15937) --- esphome/components/const/__init__.py | 5 ++- esphome/components/lc709203f/sensor.py | 3 +- esphome/components/ntc/sensor.py | 2 +- esphome/components/sensor/__init__.py | 49 ++++++++++++++++++---- tests/components/template/common-base.yaml | 7 +++- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 846d3fd883..6f418b48ea 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -2,11 +2,12 @@ CODEOWNERS = ["@esphome/core"] -CONF_BYTE_ORDER = "byte_order" -CONF_CLIMATE_ID = "climate_id" BYTE_ORDER_LITTLE = "little_endian" BYTE_ORDER_BIG = "big_endian" +CONF_B_CONSTANT = "b_constant" +CONF_BYTE_ORDER = "byte_order" +CONF_CLIMATE_ID = "climate_id" CONF_COLOR_DEPTH = "color_depth" CONF_CRC_ENABLE = "crc_enable" CONF_DATA_BITS = "data_bits" diff --git a/esphome/components/lc709203f/sensor.py b/esphome/components/lc709203f/sensor.py index 75ae703638..d4e6213425 100644 --- a/esphome/components/lc709203f/sensor.py +++ b/esphome/components/lc709203f/sensor.py @@ -1,5 +1,6 @@ import esphome.codegen as cg from esphome.components import i2c, sensor +from esphome.components.const import CONF_B_CONSTANT import esphome.config_validation as cv from esphome.const import ( CONF_BATTERY_LEVEL, @@ -22,8 +23,6 @@ DEPENDENCIES = ["i2c"] lc709203f_ns = cg.esphome_ns.namespace("lc709203f") -CONF_B_CONSTANT = "b_constant" - LC709203FBatteryVoltage = lc709203f_ns.enum("LC709203FBatteryVoltage") BATTERY_VOLTAGE_OPTIONS = { "3.7": LC709203FBatteryVoltage.LC709203F_BATTERY_VOLTAGE_3_7, diff --git a/esphome/components/ntc/sensor.py b/esphome/components/ntc/sensor.py index d47052cac6..dd7d1bd35d 100644 --- a/esphome/components/ntc/sensor.py +++ b/esphome/components/ntc/sensor.py @@ -2,6 +2,7 @@ from math import log import esphome.codegen as cg from esphome.components import sensor +from esphome.components.const import CONF_B_CONSTANT import esphome.config_validation as cv from esphome.const import ( CONF_CALIBRATION, @@ -18,7 +19,6 @@ from esphome.const import ( ntc_ns = cg.esphome_ns.namespace("ntc") NTC = ntc_ns.class_("NTC", cg.Component, sensor.Sensor) -CONF_B_CONSTANT = "b_constant" CONF_A = "a" CONF_B = "b" CONF_C = "c" diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index ed02cc2543..f076c7f17b 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -4,6 +4,7 @@ import math from esphome import automation import esphome.codegen as cg from esphome.components import mqtt, web_server, zigbee +from esphome.components.const import CONF_B_CONSTANT import esphome.config_validation as cv from esphome.const import ( CONF_ABOVE, @@ -32,6 +33,8 @@ from esphome.const import ( CONF_OPTIMISTIC, CONF_PERIOD, CONF_QUANTILE, + CONF_REFERENCE_RESISTANCE, + CONF_REFERENCE_TEMPERATURE, CONF_SEND_EVERY, CONF_SEND_FIRST_AT, CONF_STATE_CLASS, @@ -1078,16 +1081,44 @@ def ntc_get_abc(value): return a, b, c +def ntc_calc_b_constant(value): + beta = value[CONF_B_CONSTANT] + t0 = value[CONF_REFERENCE_TEMPERATURE] + ZERO_POINT + r0 = value[CONF_REFERENCE_RESISTANCE] + + a = (1 / t0) - (1 / beta) * math.log(r0) + b = 1 / beta + c = 0 + return a, b, c + + def ntc_process_calibration(value): if isinstance(value, dict): - value = cv.Schema( - { - cv.Required(CONF_A): cv.float_, - cv.Required(CONF_B): cv.float_, - cv.Required(CONF_C): cv.float_, - } - )(value) - a, b, c = ntc_get_abc(value) + if CONF_B_CONSTANT in value: + value = cv.Schema( + { + cv.Required(CONF_B_CONSTANT): cv.All( + cv.float_, cv.Range(min=0, min_included=False) + ), + cv.Required(CONF_REFERENCE_TEMPERATURE): cv.All( + cv.temperature, + cv.Range(min=-ZERO_POINT, min_included=False), + ), + cv.Required(CONF_REFERENCE_RESISTANCE): cv.All( + cv.resistance, cv.Range(min=0, min_included=False) + ), + } + )(value) + a, b, c = ntc_calc_b_constant(value) + else: + value = cv.Schema( + { + cv.Required(CONF_A): cv.float_, + cv.Required(CONF_B): cv.float_, + cv.Required(CONF_C): cv.float_, + } + )(value) + a, b, c = ntc_get_abc(value) elif isinstance(value, list): if len(value) != 3: raise cv.Invalid( @@ -1097,7 +1128,7 @@ def ntc_process_calibration(value): a, b, c = ntc_calc_steinhart_hart(value) else: raise cv.Invalid( - f"Calibration parameter accepts either a list for steinhart-hart calibration, or mapping for b-constant calibration, not {type(value)}" + f"Calibration parameter accepts either a list for steinhart-hart calibration, or mapping for b-constant or precomputed (a, b, c) calibration, not {type(value)}" ) _LOGGER.info("Coefficient: a:%s, b:%s, c:%s", a, b, c) return { diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index b97cafd25c..d3985a848b 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -202,6 +202,11 @@ sensor: value: last - timeout: timeout: 1d + - to_ntc_temperature: + calibration: + b_constant: 3950 + reference_temperature: 25.0°C + reference_resistance: 10kOhm - to_ntc_resistance: calibration: - 10.0kOhm -> 25°C @@ -270,8 +275,6 @@ cover: stop_action: - logger.log: stop_action optimistic: true - on_open: - - logger.log: "Cover on_open (deprecated)" on_opened: - logger.log: "Cover fully opened" on_closed: