[espidf] Install native ESP-IDF into a machine-global cache dir (#17306)

docker/docker_entrypoint.sh

@@ -21,6 +21,10 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms"
 export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages"
 export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache"
 
+# Keep the native ESP-IDF install on the persistent cache root, not the
+# container's ephemeral user cache dir (re-downloaded on every restart).
+export ESPHOME_ESP_IDF_PREFIX="$(dirname "${pio_cache_base}")/idf"
+
 # If /build is mounted, use that as the build path
 # otherwise use path in /config (so that builds aren't lost on container restart)
 if [[ -d /build ]]; then

docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run

@@ -15,6 +15,10 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms"
 export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages"
 export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache"
 
+# Keep the native ESP-IDF install on the persistent /data volume, not the
+# container's ephemeral user cache dir (wiped on every add-on update/restart).
+export ESPHOME_ESP_IDF_PREFIX=/data/cache/idf
+
 if bashio::config.true 'leave_front_door_open'; then
     export DISABLE_HA_AUTHENTICATION=true
 fi

esphome/__main__.py

@@ -2386,7 +2386,10 @@ def parse_args(argv):
     )
 
     parser_clean_all = subparsers.add_parser(
-        "clean-all", help="Clean all build and platform files."
+        "clean-all",
+        help="Clean all build and platform files, including machine-global "
+        "toolchain caches shared by all configurations, so other projects will "
+        "re-download them on next build.",
     )
     parser_clean_all.add_argument(
         "configuration", help="Your YAML file or configuration directory.", nargs="*"

esphome/espidf/clang_tidy.py

@@ -147,9 +147,9 @@ def _setup_core(work_dir: Path, settings: _Settings) -> None:
     from esphome.core import CORE
 
     CORE.name = TIDY_PROJECT_NAME
-    # config_path's parent is the data dir root: the IDF install lives at
-    # ``<parent>/.esphome/idf`` -- keep it beside (not inside) the per-run
-    # project dir so clearing the project doesn't force an IDF re-download.
+    # config_path's parent is the data dir root for per-run artifacts (idedata,
+    # converted pio_components). The IDF install is in the global cache dir,
+    # independent of this path.
     CORE.config_path = work_dir.parent / "tidy.yaml"
     CORE.build_path = work_dir
     esp32 = CORE.data.setdefault(KEY_ESP32, {})

esphome/espidf/framework.py

@@ -9,6 +9,8 @@ import re
 import shutil
 import tempfile
 
+import platformdirs
+
 from esphome.config_validation import Version
 from esphome.core import CORE
 from esphome.framework_helpers import (
@@ -80,10 +82,18 @@ def _get_idf_tools_path() -> Path:
     Returns:
         Path object pointing to the ESP-IDF tools directory
     """
-    if "ESPHOME_ESP_IDF_PREFIX" in os.environ:
-        path = Path(get_str_env("ESPHOME_ESP_IDF_PREFIX", None)).expanduser()
+    # Treat an empty/whitespace ESPHOME_ESP_IDF_PREFIX as unset: Path("")
+    # resolves to the CWD, which would install into (and let clean-all delete)
+    # the working directory by accident.
+    if prefix := get_str_env("ESPHOME_ESP_IDF_PREFIX", "").strip():
+        path = Path(prefix).expanduser()
     else:
-        path = CORE.data_dir / "idf"
+        # Machine-global so all projects share the multi-GB install instead of
+        # a per-config-directory copy. The user cache dir (not ~/.esphome)
+        # avoids colliding with data_dir when configs live in the home dir.
+        # appauthor=False drops the redundant <author>\ segment on Windows
+        # (which otherwise repeats "esphome\esphome\") to keep the path short.
+        path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf"
     # Resolve so an unnormalized config path (e.g. compiling ``../config/x.yaml``)
     # doesn't leave ``..`` segments in the IDF_TOOLS_PATH handed to idf.py, which
     # otherwise warns that the venv interpreter path doesn't match the install.
@@ -145,10 +155,11 @@ def _check_windows_path_length() -> None:
         "  fatal error: bits/c++config.h: No such file or directory\n"
         "  cannot execute 'as': CreateProcess: No such file or directory\n"
         "To fix, either:\n"
-        "  - Enable Windows long path support: set\n"
-        "    HKLM\\SYSTEM\\CurrentControlSet\\Control\\FileSystem\\LongPathsEnabled\n"
-        "    to 1 and reboot, or\n"
-        "  - Move your ESPHome project to a shorter path\n"
+        "  - Enable Windows long path support, then reboot. In an elevated\n"
+        "    PowerShell run:\n"
+        "      Set-ItemProperty 'HKLM:\\SYSTEM\\CurrentControlSet\\Control\\FileSystem' LongPathsEnabled 1\n"
+        "    Details: https://learn.microsoft.com/windows/win32/fileio/maximum-file-path-limitation\n"
+        "  - Or set ESPHOME_ESP_IDF_PREFIX to a shorter path (e.g. C:\\ESPHome\\idf)\n"
         "Then delete the ESP-IDF tools directory above so the toolchain "
         "reinstalls cleanly.",
         tools_path,
@@ -553,7 +564,7 @@ def _check_esphome_idf_framework_install(
     # Logged every invocation (not just on install) so the user can verify the
     # override. A changed URL needs ``esphome clean-all`` to force a re-download
     # (``esphome clean`` only wipes the build dir, not the extracted framework
-    # under <data_dir>/idf/frameworks/<version>).
+    # under the global install dir's ``frameworks/<version>``).
     if source_url:
         _LOGGER.info("Using framework source override: %s", source_url)
 
@@ -822,11 +833,9 @@ def _ccache_env() -> dict[str, str]:
 
     Enabled by default whenever the ``ccache`` binary is on PATH; set
     ``IDF_CCACHE_ENABLE=0`` in the environment to opt out. The cache lives under
-    the IDF tools path. How widely it is shared depends on where that resolves:
-    across projects (and surviving ``clean-all``) when it is a common location
-    (``ESPHOME_ESP_IDF_PREFIX`` or the add-on ``/data``), but per-project under
-    ``.esphome/idf`` for a default pip install, where ``clean-all`` clears it
-    along with the framework.
+    the IDF tools path (the machine-global cache dir, or
+    ``ESPHOME_ESP_IDF_PREFIX``), so it is shared across all projects and removed
+    by ``esphome clean-all`` along with the framework.
 
     Depend mode keeps cache-miss overhead low (hashes the compiler's depfiles
     instead of preprocessing). ``CCACHE_BASEDIR`` rewrites the per-build

esphome/writer.py

@@ -653,6 +653,15 @@ def clean_all(configuration: list[str]):
                 elif item.is_dir() and item.name != "storage":
                     rmtree(item)
 
+    # The native ESP-IDF install lives in a machine-global cache dir, outside
+    # any .esphome data dir, so the per-config loop above won't reach it.
+    from esphome.espidf.framework import _get_idf_tools_path
+
+    idf_install_path = _get_idf_tools_path()
+    if idf_install_path.is_dir():
+        _LOGGER.info("Deleting %s", idf_install_path)
+        rmtree(idf_install_path)
+
     # Clean PlatformIO project files
     try:
         from platformio.project.config import ProjectConfig

requirements.txt

@@ -23,6 +23,7 @@ bleak==2.1.1
 smpclient==6.0.0
 requests==2.34.2
 py7zr==1.1.3
+platformdirs==4.9.4  # native esp-idf toolchain global cache dir
 
 # esp-idf >= 5.0 requires this
 pyparsing >= 3.3.2

tests/unit_tests/test_espidf_framework.py

@@ -36,6 +36,19 @@ from esphome.espidf.framework import (
 from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path
 
 
+@pytest.fixture(autouse=True)
+def _isolate_idf_install_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Pin the ESP-IDF install root to a tmp dir for every test.
+
+    The default location is the OS user cache dir, so without this any test
+    that builds framework paths or pre-creates the framework dir would touch
+    the real ``~/.cache/esphome`` on the developer's machine. Tests that need
+    to exercise the override or default-resolution logic clear/override the env
+    themselves.
+    """
+    monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", str(tmp_path / "idf_install"))
+
+
 @pytest.mark.parametrize(
     ("source", "expected"),
     [
@@ -791,6 +804,38 @@ def test_get_idf_tools_path_env_override(tmp_path: Path) -> None:
         assert _get_idf_tools_path() == Path(override)
 
 
+@pytest.mark.parametrize("value", ["", "   "])
+def test_get_idf_tools_path_blank_env_falls_back_to_default(
+    value: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A blank ESPHOME_ESP_IDF_PREFIX is treated as unset, not as CWD.
+
+    Path("") would resolve to the working directory, which clean-all could then
+    delete by accident.
+    """
+    import platformdirs
+
+    monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", value)
+    expected = (
+        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf"
+    ).resolve()
+    assert _get_idf_tools_path() == expected
+
+
+def test_get_idf_tools_path_default_uses_user_cache(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Without the env override the install root is the machine-global OS user
+    cache dir, not the per-config ``<data_dir>/idf``."""
+    import platformdirs
+
+    monkeypatch.delenv("ESPHOME_ESP_IDF_PREFIX", raising=False)
+    expected = (
+        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf"
+    ).resolve()
+    assert _get_idf_tools_path() == expected
+
+
 def test_write_idf_version_txt_warns_on_write_error(tmp_path: Path) -> None:
     with patch("pathlib.Path.write_text", side_effect=OSError("denied")):
         # write failure is caught and warned, not raised
@@ -908,3 +953,5 @@ def test_check_windows_path_length_long_path_warns(
     message = caplog.records[0].getMessage()
     assert _LONG_IDF_PATH in message
     assert "long path support" in message
+    # The install is global now; the remedy is the prefix env, not moving the project.
+    assert "ESPHOME_ESP_IDF_PREFIX" in message

tests/unit_tests/test_writer.py

@@ -67,15 +67,23 @@ def _isolate_platformio_paths(tmp_path_factory: pytest.TempPathFactory) -> Any:
     want to verify the PIO-cleanup branch (e.g. test_clean_all,
     test_clean_all_partial_exists) install their own inner patch which
     stacks on top of this one and wins for the duration of their block.
+
+    Also pin ``ESPHOME_ESP_IDF_PREFIX`` to a nonexistent tmp dir for the
+    same reason: ``clean_all`` removes the now machine-global ESP-IDF
+    install, which otherwise defaults to the real ``~/.cache/esphome``.
     """
     pio_root = tmp_path_factory.mktemp("isolated_pio") / "nonexistent"
+    idf_root = tmp_path_factory.mktemp("isolated_idf") / "nonexistent"
     mock_cfg = MagicMock()
     mock_cfg.get.side_effect = lambda section, option: (
         str(pio_root / option) if section == "platformio" else ""
     )
-    with patch(
-        "platformio.project.config.ProjectConfig.get_instance",
-        return_value=mock_cfg,
+    with (
+        patch(
+            "platformio.project.config.ProjectConfig.get_instance",
+            return_value=mock_cfg,
+        ),
+        patch.dict("os.environ", {"ESPHOME_ESP_IDF_PREFIX": str(idf_root)}),
     ):
         yield
 
@@ -990,6 +998,30 @@ def test_clean_all_with_yaml_file(
     assert str(build_dir) in caplog.text
 
 
+@patch("esphome.writer.CORE")
+def test_clean_all_removes_global_idf_install(
+    mock_core: MagicMock,
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """clean_all removes the machine-global native ESP-IDF install dir."""
+    idf_install = tmp_path / "idf_install"
+    (idf_install / "frameworks").mkdir(parents=True)
+    monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", str(idf_install))
+
+    config_dir = tmp_path / "config"
+    config_dir.mkdir()
+
+    from esphome.writer import clean_all
+
+    with caplog.at_level("INFO"):
+        clean_all([str(config_dir)])
+
+    assert not idf_install.exists()
+    assert str(idf_install.resolve()) in caplog.text
+
+
 @patch("esphome.writer.CORE")
 def test_clean_all_with_yaml_build_path(
     mock_core: MagicMock,