diff --git a/.clang-tidy.hash b/.clang-tidy.hash index 5c7eab517b..cd61d9ec48 100644 --- a/.clang-tidy.hash +++ b/.clang-tidy.hash @@ -1 +1 @@ -f31f13994768b5b07e29624406c9b053bf4bb26e1623ac2bc1e9d4a9477502d6 +d48687d988ae2a94a9973226df773478a7db1d52133545f07aa05e34fc678dcf diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index f27690c97b..cd38c82dd8 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -671,11 +671,12 @@ def _is_framework_url(source: str) -> bool: # The default/recommended arduino framework version # - https://github.com/espressif/arduino-esp32/releases ARDUINO_FRAMEWORK_VERSION_LOOKUP = { - "recommended": cv.Version(3, 3, 7), - "latest": cv.Version(3, 3, 7), - "dev": cv.Version(3, 3, 7), + "recommended": cv.Version(3, 3, 8), + "latest": cv.Version(3, 3, 8), + "dev": cv.Version(3, 3, 8), } ARDUINO_PLATFORM_VERSION_LOOKUP = { + cv.Version(3, 3, 8): cv.Version(55, 3, 38), cv.Version(3, 3, 7): cv.Version(55, 3, 37), cv.Version(3, 3, 6): cv.Version(55, 3, 36), cv.Version(3, 3, 5): cv.Version(55, 3, 35), @@ -695,6 +696,7 @@ ARDUINO_PLATFORM_VERSION_LOOKUP = { # These versions correspond to pioarduino/esp-idf releases # See: https://github.com/pioarduino/esp-idf/releases ARDUINO_IDF_VERSION_LOOKUP = { + cv.Version(3, 3, 8): cv.Version(5, 5, 4), cv.Version(3, 3, 7): cv.Version(5, 5, 3, "1"), cv.Version(3, 3, 6): cv.Version(5, 5, 2), cv.Version(3, 3, 5): cv.Version(5, 5, 2), @@ -714,17 +716,15 @@ ARDUINO_IDF_VERSION_LOOKUP = { # The default/recommended esp-idf framework version # - https://github.com/espressif/esp-idf/releases ESP_IDF_FRAMEWORK_VERSION_LOOKUP = { - "recommended": cv.Version(5, 5, 3, "1"), - "latest": cv.Version(5, 5, 3, "1"), + "recommended": cv.Version(5, 5, 4), + "latest": cv.Version(5, 5, 4), "dev": cv.Version(5, 5, 4), } ESP_IDF_PLATFORM_VERSION_LOOKUP = { cv.Version( 6, 0, 0 ): "https://github.com/pioarduino/platform-espressif32.git#prep_IDF6", - cv.Version( - 5, 5, 4 - ): "https://github.com/pioarduino/platform-espressif32.git#develop", + cv.Version(5, 5, 4): cv.Version(55, 3, 38), cv.Version(5, 5, 3, "1"): cv.Version(55, 3, 37), cv.Version(5, 5, 3): cv.Version(55, 3, 37), cv.Version(5, 5, 2): cv.Version(55, 3, 37), @@ -744,8 +744,8 @@ ESP_IDF_PLATFORM_VERSION_LOOKUP = { # The platform-espressif32 version # - https://github.com/pioarduino/platform-espressif32/releases PLATFORM_VERSION_LOOKUP = { - "recommended": cv.Version(55, 3, 37), - "latest": cv.Version(55, 3, 37), + "recommended": cv.Version(55, 3, 38), + "latest": cv.Version(55, 3, 38), "dev": "https://github.com/pioarduino/platform-espressif32.git#develop", } diff --git a/esphome/components/esp32/boards.py b/esphome/components/esp32/boards.py index 2bd08e7c39..2c73fe7d08 100644 --- a/esphome/components/esp32/boards.py +++ b/esphome/components/esp32/boards.py @@ -1960,6 +1960,10 @@ BOARDS = { "name": "Hornbill ESP32 Minima", "variant": VARIANT_ESP32, }, + "huidu_hd_wf1": { + "name": "Huidu HD-WF1", + "variant": VARIANT_ESP32S2, + }, "huidu_hd_wf2": { "name": "Huidu HD-WF2", "variant": VARIANT_ESP32S3, @@ -2028,6 +2032,10 @@ BOARDS = { "name": "LilyGo T-Display-S3", "variant": VARIANT_ESP32S3, }, + "lilygo-t-energy-s3": { + "name": "LilyGo T-Energy-S3", + "variant": VARIANT_ESP32S3, + }, "lilygo-t3-s3": { "name": "LilyGo T3-S3", "variant": VARIANT_ESP32S3, @@ -2289,10 +2297,18 @@ BOARDS = { "name": "S.ODI Ultra v1", "variant": VARIANT_ESP32, }, + "seeed_xiao_esp32_s3_plus": { + "name": "Seeed Studio XIAO ESP32S3 Plus", + "variant": VARIANT_ESP32S3, + }, "seeed_xiao_esp32c3": { "name": "Seeed Studio XIAO ESP32C3", "variant": VARIANT_ESP32C3, }, + "seeed_xiao_esp32c5": { + "name": "Seeed Studio XIAO ESP32C5", + "variant": VARIANT_ESP32C5, + }, "seeed_xiao_esp32c6": { "name": "Seeed Studio XIAO ESP32C6", "variant": VARIANT_ESP32C6, diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index cb080b2a95..e9719f7dcd 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -5,104 +5,15 @@ import os from pathlib import Path import re import subprocess -import time -from typing import Any +import sys from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE from esphome.core import CORE, EsphomeError -from esphome.util import run_external_command, run_external_process +from esphome.util import run_external_process _LOGGER = logging.getLogger(__name__) -def patch_structhash(): - # Patch platformio's structhash to not recompile the entire project when files are - # removed/added. This might have unintended consequences, but this improves compile - # times greatly when adding/removing components and a simple clean build solves - # all issues - from platformio.run import cli, helpers - - def patched_clean_build_dir(build_dir, *args): - from platformio import fs - from platformio.project.helpers import get_project_dir - - platformio_ini = Path(get_project_dir()) / "platformio.ini" - - build_dir = Path(build_dir) - - # if project's config is modified - if ( - build_dir.is_dir() - and platformio_ini.stat().st_mtime > build_dir.stat().st_mtime - ): - fs.rmtree(build_dir) - - if not build_dir.is_dir(): - build_dir.mkdir(parents=True) - - helpers.clean_build_dir = patched_clean_build_dir - cli.clean_build_dir = patched_clean_build_dir - - -def patch_file_downloader(): - """Patch PlatformIO's FileDownloader to retry on PackageException errors. - - PlatformIO's FileDownloader uses HTTPSession which lacks built-in retry - for 502/503 errors. We add retries with exponential backoff and close the - session between attempts to force a fresh TCP connection, which may route - to a different CDN edge node. - """ - from platformio.package.download import FileDownloader - from platformio.package.exception import PackageException - - if getattr(FileDownloader.__init__, "_esphome_patched", False): - return - - original_init = FileDownloader.__init__ - - def patched_init(self, *args: Any, **kwargs: Any) -> None: - max_retries = 5 - - for attempt in range(max_retries): - try: - original_init(self, *args, **kwargs) - return - except PackageException as e: - if attempt < max_retries - 1: - # Exponential backoff: 2, 4, 8, 16 seconds - delay = 2 ** (attempt + 1) - _LOGGER.warning( - "Package download failed: %s. " - "Retrying in %d seconds... (attempt %d/%d)", - str(e), - delay, - attempt + 1, - max_retries, - ) - # Close the response and session to free resources - # and force a new TCP connection on retry, which may - # route to a different CDN edge node - # pylint: disable=protected-access,broad-except - try: - if ( - hasattr(self, "_http_response") - and self._http_response is not None - ): - self._http_response.close() - if hasattr(self, "_http_session"): - self._http_session.close() - except Exception: - pass - # pylint: enable=protected-access,broad-except - time.sleep(delay) - else: - # Final attempt - re-raise - raise - - patched_init._esphome_patched = True # type: ignore[attr-defined] # pylint: disable=protected-access - FileDownloader.__init__ = patched_init - - IGNORE_LIB_WARNINGS = f"(?:{'|'.join(['Hash', 'Update'])})" FILTER_PLATFORMIO_LINES = [ r"Verbose mode can be enabled via `-v, --verbose` option.*", @@ -142,20 +53,6 @@ FILTER_PLATFORMIO_LINES = [ ] -class PlatformioLogFilter(logging.Filter): - """Filter to suppress noisy platformio log messages.""" - - _PATTERN = re.compile( - r"|".join(r"(?:" + pattern + r")" for pattern in FILTER_PLATFORMIO_LINES) - ) - - def filter(self, record: logging.LogRecord) -> bool: - # Only filter messages from platformio-related loggers - if "platformio" not in record.name.lower(): - return True - return self._PATTERN.match(record.getMessage()) is None - - def run_platformio_cli(*args, **kwargs) -> str | int: os.environ["PLATFORMIO_FORCE_COLOR"] = "true" os.environ["PLATFORMIO_BUILD_DIR"] = str(CORE.relative_pioenvs_path().absolute()) @@ -166,30 +63,12 @@ def run_platformio_cli(*args, **kwargs) -> str | int: os.environ.setdefault("PYTHONWARNINGS", "ignore::SyntaxWarning") # Increase uv retry count to handle transient network errors (default is 3) os.environ.setdefault("UV_HTTP_RETRIES", "10") - cmd = ["platformio"] + list(args) + cmd = [sys.executable, "-m", "esphome.platformio_runner"] + list(args) if not CORE.verbose: kwargs["filter_lines"] = FILTER_PLATFORMIO_LINES - if os.environ.get("ESPHOME_USE_SUBPROCESS") is not None: - return run_external_process(*cmd, **kwargs) - - import platformio.__main__ - - patch_structhash() - patch_file_downloader() - - # Add log filter to suppress noisy platformio messages - log_filter = PlatformioLogFilter() if not CORE.verbose else None - if log_filter: - for handler in logging.getLogger().handlers: - handler.addFilter(log_filter) - try: - return run_external_command(platformio.__main__.main, *cmd, **kwargs) - finally: - if log_filter: - for handler in logging.getLogger().handlers: - handler.removeFilter(log_filter) + return run_external_process(*cmd, **kwargs) def run_platformio_cli_run(config, verbose, *args, **kwargs) -> str | int: diff --git a/esphome/platformio_runner.py b/esphome/platformio_runner.py new file mode 100644 index 0000000000..408d49d1a6 --- /dev/null +++ b/esphome/platformio_runner.py @@ -0,0 +1,114 @@ +"""Subprocess entry point that applies ESPHome's PlatformIO patches. + +Invoked via ``python -m esphome.platformio_runner`` instead of +``python -m platformio`` so that the patches (incremental rebuild +preservation, download retries) apply inside the subprocess. Running +PlatformIO in a subprocess keeps its ``sys.path`` mutations and other +global state from leaking into the ESPHome process. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +import sys +import time +from typing import Any + +_LOGGER = logging.getLogger(__name__) + + +def patch_structhash() -> None: + """Avoid full rebuilds when files are added or removed. + + PlatformIO clears the build dir whenever its structure hash changes. + We replace that with an mtime check against ``platformio.ini`` so + incremental builds are preserved unless the project config changed. + """ + from platformio.run import cli, helpers + + def patched_clean_build_dir(build_dir, *_args): + from platformio import fs + from platformio.project.helpers import get_project_dir + + platformio_ini = Path(get_project_dir()) / "platformio.ini" + build_dir = Path(build_dir) + + if ( + build_dir.is_dir() + and platformio_ini.stat().st_mtime > build_dir.stat().st_mtime + ): + fs.rmtree(build_dir) + + if not build_dir.is_dir(): + build_dir.mkdir(parents=True) + + helpers.clean_build_dir = patched_clean_build_dir + cli.clean_build_dir = patched_clean_build_dir + + +def patch_file_downloader() -> None: + """Retry PlatformIO package downloads with exponential backoff. + + PlatformIO's ``FileDownloader`` uses an ``HTTPSession`` without built-in + retry for 502/503 errors. We wrap ``__init__`` to retry on + ``PackageException`` and close the session between attempts so a new + TCP connection can route to a different CDN edge node. + """ + from platformio.package.download import FileDownloader + from platformio.package.exception import PackageException + + if getattr(FileDownloader.__init__, "_esphome_patched", False): + return + + original_init = FileDownloader.__init__ + + def patched_init(self, *args: Any, **kwargs: Any) -> None: + max_retries = 5 + + for attempt in range(max_retries): + try: + original_init(self, *args, **kwargs) + return + except PackageException as e: + if attempt < max_retries - 1: + delay = 2 ** (attempt + 1) + _LOGGER.warning( + "Package download failed: %s. " + "Retrying in %d seconds... (attempt %d/%d)", + str(e), + delay, + attempt + 1, + max_retries, + ) + # pylint: disable=protected-access,broad-except + try: + if ( + hasattr(self, "_http_response") + and self._http_response is not None + ): + self._http_response.close() + if hasattr(self, "_http_session"): + self._http_session.close() + except Exception: + pass + # pylint: enable=protected-access,broad-except + time.sleep(delay) + else: + raise + + patched_init._esphome_patched = True # type: ignore[attr-defined] # pylint: disable=protected-access + FileDownloader.__init__ = patched_init + + +def main() -> int: + patch_structhash() + patch_file_downloader() + + import platformio.__main__ + + return platformio.__main__.main() or 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platformio.ini b/platformio.ini index e0f7c7d443..7d17628a8f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -133,10 +133,10 @@ extra_scripts = post:esphome/components/esp8266/post_build.py.script ; This are common settings for the ESP32 (all variants) using Arduino. [common:esp32-arduino] extends = common:arduino -platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.37/platform-espressif32.zip +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.38/platform-espressif32.zip platform_packages = - pioarduino/framework-arduinoespressif32@https://github.com/espressif/arduino-esp32/releases/download/3.3.7/esp32-core-3.3.7.tar.xz - pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.3.1/esp-idf-v5.5.3.1.tar.xz + pioarduino/framework-arduinoespressif32@https://github.com/espressif/arduino-esp32/releases/download/3.3.8/esp32-core-3.3.8.tar.xz + pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz framework = arduino, espidf ; Arduino as an ESP-IDF component lib_deps = @@ -169,9 +169,9 @@ extra_scripts = post:esphome/components/esp32/post_build.py.script ; This are common settings for the ESP32 (all variants) using IDF. [common:esp32-idf] extends = common:idf -platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.37/platform-espressif32.zip +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.38/platform-espressif32.zip platform_packages = - pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.3.1/esp-idf-v5.5.3.1.tar.xz + pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz framework = espidf lib_deps = diff --git a/pyproject.toml b/pyproject.toml index 2e3a247768..a744286e88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ classifiers = [ "Topic :: Home Automation", ] -# Python 3.14 is not supported on Windows, see https://github.com/zephyrproject-rtos/windows-curses/issues/76 requires-python = ">=3.11.0,<3.15" dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 1a1bfffd03..dfd4305c4d 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -84,9 +84,9 @@ def mock_decode_pc() -> Generator[Mock, None, None]: @pytest.fixture -def mock_run_external_command() -> Generator[Mock, None, None]: - """Mock run_external_command for platformio_api.""" - with patch("esphome.platformio_api.run_external_command") as mock: +def mock_run_external_process() -> Generator[Mock, None, None]: + """Mock run_external_process for platformio_api.""" + with patch("esphome.platformio_api.run_external_process") as mock: yield mock diff --git a/tests/unit_tests/test_platformio_api.py b/tests/unit_tests/test_platformio_api.py index e1b3908c24..ddc4e45c84 100644 --- a/tests/unit_tests/test_platformio_api.py +++ b/tests/unit_tests/test_platformio_api.py @@ -1,7 +1,8 @@ """Tests for platformio_api.py path functions.""" +# pylint: disable=protected-access + import json -import logging import os from pathlib import Path import shutil @@ -10,7 +11,7 @@ from unittest.mock import MagicMock, Mock, call, patch import pytest -from esphome import platformio_api +from esphome import platformio_api, platformio_runner from esphome.core import CORE, EsphomeError @@ -281,13 +282,13 @@ def test_run_idedata_raises_on_invalid_json( def test_run_platformio_cli_sets_environment_variables( - setup_core: Path, mock_run_external_command: Mock + setup_core: Path, mock_run_external_process: Mock ) -> None: """Test run_platformio_cli sets correct environment variables.""" CORE.build_path = str(setup_core / "build" / "test") with patch.dict(os.environ, {}, clear=False): - mock_run_external_command.return_value = 0 + mock_run_external_process.return_value = 0 platformio_api.run_platformio_cli("test", "arg") # Check environment variables were set @@ -300,10 +301,12 @@ def test_run_platformio_cli_sets_environment_variables( assert "PLATFORMIO_LIBDEPS_DIR" in os.environ assert "PYTHONWARNINGS" in os.environ - # Check command was called correctly - mock_run_external_command.assert_called_once() - args = mock_run_external_command.call_args[0] - assert "platformio" in args + # Check command was called correctly — runs PlatformIO as a subprocess + # via the esphome.platformio_runner entry point. + mock_run_external_process.assert_called_once() + args = mock_run_external_process.call_args[0] + assert "-m" in args + assert "esphome.platformio_runner" in args assert "test" in args assert "arg" in args @@ -444,7 +447,7 @@ def test_patch_structhash(setup_core: Path) -> None: }, ): # Call patch_structhash - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Verify both modules had clean_build_dir patched # Check that clean_build_dir was set on both modules @@ -496,7 +499,7 @@ def test_patched_clean_build_dir_removes_outdated(setup_core: Path) -> None: }, ): # Call patch_structhash to install the patched function - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Call the patched function mock_helpers.clean_build_dir(str(build_dir), []) @@ -546,7 +549,7 @@ def test_patched_clean_build_dir_keeps_updated(setup_core: Path) -> None: }, ): # Call patch_structhash to install the patched function - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Call the patched function mock_helpers.clean_build_dir(str(build_dir), []) @@ -594,7 +597,7 @@ def test_patched_clean_build_dir_creates_missing(setup_core: Path) -> None: }, ): # Call patch_structhash to install the patched function - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Call the patched function mock_helpers.clean_build_dir(str(build_dir), []) @@ -719,7 +722,7 @@ def test_patch_file_downloader_succeeds_first_try() -> None: ), }, ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -758,7 +761,7 @@ def test_patch_file_downloader_retries_on_failure() -> None: ), patch("time.sleep") as mock_sleep, ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -799,7 +802,7 @@ def test_patch_file_downloader_raises_after_max_retries() -> None: ), patch("time.sleep") as mock_sleep, ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -847,7 +850,7 @@ def test_patch_file_downloader_closes_session_and_response_between_retries() -> ), patch("time.sleep"), ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -882,9 +885,9 @@ def test_patch_file_downloader_idempotent() -> None: }, ): # Patch multiple times - platformio_api.patch_file_downloader() - platformio_api.patch_file_downloader() - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() + platformio_runner.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -895,19 +898,18 @@ def test_patch_file_downloader_idempotent() -> None: assert call_count == 1 -def test_platformio_log_filter_allows_non_platformio_messages() -> None: - """Test that non-platformio logger messages are allowed through.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name="esphome.core", - level=logging.INFO, - pathname="", - lineno=0, - msg="Some esphome message", - args=(), - exc_info=None, +def _filter_through_redirect(line: str) -> str: + """Write a line through RedirectText with FILTER_PLATFORMIO_LINES and return what passes.""" + import io + + from esphome.util import RedirectText + + captured = io.StringIO() + redirect = RedirectText( + captured, filter_lines=platformio_api.FILTER_PLATFORMIO_LINES ) - assert log_filter.filter(record) is True + redirect.write(line + "\n") + return captured.getvalue() @pytest.mark.parametrize( @@ -930,19 +932,9 @@ def test_platformio_log_filter_allows_non_platformio_messages() -> None: "Memory Usage -> https://bit.ly/pio-memory-usage", ], ) -def test_platformio_log_filter_blocks_noisy_messages(msg: str) -> None: - """Test that noisy platformio messages are filtered out.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name="platformio.builder", - level=logging.INFO, - pathname="", - lineno=0, - msg=msg, - args=(), - exc_info=None, - ) - assert log_filter.filter(record) is False +def test_filter_platformio_lines_blocks_noisy_messages(msg: str) -> None: + """Test that noisy platformio output lines are filtered out by RedirectText.""" + assert _filter_through_redirect(msg) == "" @pytest.mark.parametrize( @@ -954,39 +946,6 @@ def test_platformio_log_filter_blocks_noisy_messages(msg: str) -> None: "warning: unused variable", ], ) -def test_platformio_log_filter_allows_other_platformio_messages(msg: str) -> None: - """Test that non-noisy platformio messages are allowed through.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name="platformio.builder", - level=logging.INFO, - pathname="", - lineno=0, - msg=msg, - args=(), - exc_info=None, - ) - assert log_filter.filter(record) is True - - -@pytest.mark.parametrize( - "logger_name", - [ - "PLATFORMIO.builder", - "PlatformIO.core", - "platformio.run", - ], -) -def test_platformio_log_filter_case_insensitive_logger_name(logger_name: str) -> None: - """Test that platformio logger name matching is case insensitive.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name=logger_name, - level=logging.INFO, - pathname="", - lineno=0, - msg="Found 5 compatible libraries", - args=(), - exc_info=None, - ) - assert log_filter.filter(record) is False +def test_filter_platformio_lines_allows_other_messages(msg: str) -> None: + """Test that non-noisy platformio output lines pass through RedirectText.""" + assert _filter_through_redirect(msg) == msg + "\n"