[nrf52] Install native sdk-nrf into a machine-global cache dir (#17353)

docker/docker_entrypoint.sh

@@ -21,9 +21,10 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms"
 export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages"
 export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache"
 
-# Keep the native ESP-IDF install on the persistent cache root, not the
+# Keep the native toolchain installs on the persistent cache root, not the
 # container's ephemeral user cache dir (re-downloaded on every restart).
 export ESPHOME_ESP_IDF_PREFIX="$(dirname "${pio_cache_base}")/idf"
+export ESPHOME_SDK_NRF_PREFIX="$(dirname "${pio_cache_base}")/sdk-nrf"
 
 # If /build is mounted, use that as the build path
 # otherwise use path in /config (so that builds aren't lost on container restart)

docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run

@@ -15,9 +15,10 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms"
 export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages"
 export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache"
 
-# Keep the native ESP-IDF install on the persistent /data volume, not the
+# Keep the native toolchain installs on the persistent /data volume, not the
 # container's ephemeral user cache dir (wiped on every add-on update/restart).
 export ESPHOME_ESP_IDF_PREFIX=/data/cache/idf
+export ESPHOME_SDK_NRF_PREFIX=/data/cache/sdk-nrf
 
 if bashio::config.true 'leave_front_door_open'; then
     export DISABLE_HA_AUTHENTICATION=true

esphome/components/nrf52/framework.py

@@ -4,6 +4,8 @@ from pathlib import Path
 import platform
 import tempfile
 
+import platformdirs
+
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, EsphomeError
 from esphome.framework_helpers import (
@@ -15,6 +17,7 @@ from esphome.framework_helpers import (
     run_command_ok,
     str_to_lst_of_str,
 )
+from esphome.helpers import get_str_env
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -38,20 +41,28 @@ SDK_NG_MINIMAL_MIRRORS = str_to_lst_of_str(
 )
 
 
-def _get_tools_path() -> Path:
-    return CORE.data_dir / "sdk-nrf"
+def get_sdk_nrf_tools_path() -> Path:
+    # A blank ESPHOME_SDK_NRF_PREFIX must be treated as unset: Path("")
+    # resolves to the CWD, which clean-all would then delete.
+    if prefix := get_str_env("ESPHOME_SDK_NRF_PREFIX", "").strip():
+        path = Path(prefix).expanduser()
+    else:
+        # Machine-global (OS user cache dir) so all projects share one install;
+        # see espidf.framework.get_idf_tools_path for the location rationale.
+        path = Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "sdk-nrf"
+    return path.resolve()
 
 
 def _get_python_env_path(version: str) -> Path:
-    return _get_tools_path() / "penvs" / version
+    return get_sdk_nrf_tools_path() / "penvs" / version
 
 
 def _get_framework_path(version: str) -> Path:
-    return _get_tools_path() / "frameworks" / version
+    return get_sdk_nrf_tools_path() / "frameworks" / version
 
 
 def _get_toolchain_path(version: str) -> Path:
-    return _get_tools_path() / "toolchains" / version
+    return get_sdk_nrf_tools_path() / "toolchains" / version
 
 
 _SITECUSTOMIZE = """\

esphome/espidf/framework.py

@@ -75,7 +75,7 @@ ESP_IDF_CONSTRAINTS_MIRRORS = str_to_lst_of_str(
 )
 
 
-def _get_idf_tools_path() -> Path:
+def get_idf_tools_path() -> Path:
     """
     Get the path to the ESP-IDF tools directory.
 
@@ -141,7 +141,7 @@ def _check_windows_path_length() -> None:
     """
     if platform.system() != "Windows" or _windows_long_paths_enabled():
         return
-    tools_path = str(_get_idf_tools_path())
+    tools_path = str(get_idf_tools_path())
     projected = len(tools_path) + _TOOLCHAIN_NESTED_PATH_LEN
     if projected <= _WINDOWS_MAX_PATH:
         return
@@ -180,7 +180,7 @@ def _get_framework_path(version: str) -> Path:
     Returns:
         Path object pointing to the framework directory
     """
-    return _get_idf_tools_path() / "frameworks" / f"{version}"
+    return get_idf_tools_path() / "frameworks" / f"{version}"
 
 
 def _get_python_env_path(version: str) -> Path:
@@ -193,7 +193,7 @@ def _get_python_env_path(version: str) -> Path:
     Returns:
         Path object pointing to the Python environment directory
     """
-    return _get_idf_tools_path() / "penvs" / f"{version}"
+    return get_idf_tools_path() / "penvs" / f"{version}"
 
 
 def _check_stamp(file: PathType, data: dict[str, str]) -> bool:
@@ -707,7 +707,7 @@ def _check_esp_idf_python_env_install(
 
         esp_idf_version = _get_idf_version(framework_path, env=env)
         constraint_file_path = (
-            _get_idf_tools_path() / f"espidf.constraints.v{esp_idf_version}.txt"
+            get_idf_tools_path() / f"espidf.constraints.v{esp_idf_version}.txt"
         )
         _LOGGER.debug("ESP-IDF version %s", esp_idf_version)
 
@@ -798,7 +798,7 @@ def check_esp_idf_install(
     _check_windows_path_length()
 
     env = {}
-    env["IDF_TOOLS_PATH"] = str(_get_idf_tools_path())
+    env["IDF_TOOLS_PATH"] = str(get_idf_tools_path())
     env["IDF_PATH"] = ""
 
     targets = targets or ESPHOME_IDF_DEFAULT_TARGETS
@@ -867,7 +867,7 @@ def _ccache_env() -> dict[str, str]:
 
     defaults = {
         "IDF_CCACHE_ENABLE": "1",
-        "CCACHE_DIR": str(_get_idf_tools_path() / "ccache"),
+        "CCACHE_DIR": str(get_idf_tools_path() / "ccache"),
         "CCACHE_NOHASHDIR": "true",
         "CCACHE_DEPEND": "1",
         "CCACHE_BASEDIR": str(Path(CORE.build_path).resolve()),
@@ -894,7 +894,7 @@ def get_framework_env(
     """
     # 1. Initialize base environment with extra ESP-IDF environment variables
     env = env.copy() if env else {}
-    env["IDF_TOOLS_PATH"] = str(_get_idf_tools_path())
+    env["IDF_TOOLS_PATH"] = str(get_idf_tools_path())
     env["IDF_PATH"] = ""
 
     # 2. Get existing PATH from env or os.environ

esphome/writer.py

@@ -653,14 +653,21 @@ def clean_all(configuration: list[str]):
                 elif item.is_dir() and item.name != "storage":
                     rmtree(item)
 
-    # The native ESP-IDF install lives in a machine-global cache dir, outside
-    # any .esphome data dir, so the per-config loop above won't reach it.
-    from esphome.espidf.framework import _get_idf_tools_path
+    # The native toolchain installs live in a machine-global cache dir that
+    # the per-config loop above can't reach. Wipe the default cache root
+    # (also catches leftovers from older install layouts), then the resolved
+    # install paths for the ESPHOME_*_PREFIX overrides (docker/add-on/CI)
+    # that live outside it.
+    import platformdirs
 
-    idf_install_path = _get_idf_tools_path()
-    if idf_install_path.is_dir():
-        _LOGGER.info("Deleting %s", idf_install_path)
-        rmtree(idf_install_path)
+    from esphome.components.nrf52.framework import get_sdk_nrf_tools_path
+    from esphome.espidf.framework import get_idf_tools_path
+
+    cache_root = Path(platformdirs.user_cache_dir("esphome", appauthor=False)).resolve()
+    for install_path in (cache_root, get_idf_tools_path(), get_sdk_nrf_tools_path()):
+        if install_path.is_dir():
+            _LOGGER.info("Deleting %s", install_path)
+            rmtree(install_path)
 
     # Clean PlatformIO project files
     try:

tests/unit_tests/test_espidf_framework.py

@@ -21,7 +21,6 @@ from esphome.espidf.framework import (
     _clone_idf_with_submodules,
     _get_framework_path,
     _get_idf_tool_paths,
-    _get_idf_tools_path,
     _get_idf_version,
     _get_python_env_path,
     _get_python_version,
@@ -32,6 +31,7 @@ from esphome.espidf.framework import (
     _write_stamp,
     check_esp_idf_install,
     get_framework_env,
+    get_idf_tools_path,
 )
 from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path
 
@@ -639,7 +639,7 @@ def test_write_stamp_writes_json(tmp_path: Path) -> None:
 def test_get_framework_env_with_python_env(tmp_path: Path) -> None:
     with (
         patch(
-            "esphome.espidf.framework._get_idf_tools_path",
+            "esphome.espidf.framework.get_idf_tools_path",
             return_value=tmp_path / "tools",
         ),
         patch("esphome.espidf.framework._get_idf_version", return_value="5.1.2"),
@@ -664,7 +664,7 @@ def test_get_framework_env_with_python_env(tmp_path: Path) -> None:
 def test_get_framework_env_without_python_env_uses_os_path(tmp_path: Path) -> None:
     with (
         patch(
-            "esphome.espidf.framework._get_idf_tools_path",
+            "esphome.espidf.framework.get_idf_tools_path",
             return_value=tmp_path / "tools",
         ),
         patch("esphome.espidf.framework._get_idf_version", return_value="5.1.2"),
@@ -687,7 +687,7 @@ def _ccache_patches(tmp_path: Path, which: str | None, build_path: Path | None):
     return (
         patch("esphome.espidf.framework.shutil.which", return_value=which),
         patch(
-            "esphome.espidf.framework._get_idf_tools_path",
+            "esphome.espidf.framework.get_idf_tools_path",
             return_value=tmp_path / "tools",
         ),
         patch(
@@ -761,7 +761,7 @@ def test_ccache_env_raises_without_build_path(tmp_path: Path) -> None:
 
 
 # ---------------------------------------------------------------------------
-# _check_stamp / _write_idf_version_txt / _get_idf_tools_path
+# _check_stamp / _write_idf_version_txt / get_idf_tools_path
 # ---------------------------------------------------------------------------
 
 
@@ -798,14 +798,14 @@ def test_write_idf_version_txt_skips_when_present(tmp_path: Path) -> None:
     assert (tmp_path / "version.txt").read_text(encoding="utf-8") == "existing\n"
 
 
-def test_get_idf_tools_path_env_override(tmp_path: Path) -> None:
+def testget_idf_tools_path_env_override(tmp_path: Path) -> None:
     override = str(tmp_path / "custom-idf")
     with patch.dict("os.environ", {"ESPHOME_ESP_IDF_PREFIX": override}):
-        assert _get_idf_tools_path() == Path(override)
+        assert get_idf_tools_path() == Path(override)
 
 
 @pytest.mark.parametrize("value", ["", "   "])
-def test_get_idf_tools_path_blank_env_falls_back_to_default(
+def testget_idf_tools_path_blank_env_falls_back_to_default(
     value: str, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     """A blank ESPHOME_ESP_IDF_PREFIX is treated as unset, not as CWD.
@@ -819,10 +819,10 @@ def test_get_idf_tools_path_blank_env_falls_back_to_default(
     expected = (
         Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf"
     ).resolve()
-    assert _get_idf_tools_path() == expected
+    assert get_idf_tools_path() == expected
 
 
-def test_get_idf_tools_path_default_uses_user_cache(
+def testget_idf_tools_path_default_uses_user_cache(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     """Without the env override the install root is the machine-global OS user
@@ -833,7 +833,7 @@ def test_get_idf_tools_path_default_uses_user_cache(
     expected = (
         Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "idf"
     ).resolve()
-    assert _get_idf_tools_path() == expected
+    assert get_idf_tools_path() == expected
 
 
 def test_write_idf_version_txt_warns_on_write_error(tmp_path: Path) -> None:
@@ -908,7 +908,7 @@ def test_check_windows_path_length_noop_when_long_paths_enabled(
         patch(
             "esphome.espidf.framework._windows_long_paths_enabled", return_value=True
         ),
-        patch("esphome.espidf.framework._get_idf_tools_path") as get_path_mock,
+        patch("esphome.espidf.framework.get_idf_tools_path") as get_path_mock,
         caplog.at_level(logging.WARNING),
     ):
         _check_windows_path_length()
@@ -925,7 +925,7 @@ def test_check_windows_path_length_short_path_silent(
             "esphome.espidf.framework._windows_long_paths_enabled", return_value=False
         ),
         patch(
-            "esphome.espidf.framework._get_idf_tools_path",
+            "esphome.espidf.framework.get_idf_tools_path",
             return_value=_SHORT_IDF_PATH,
         ),
         caplog.at_level(logging.WARNING),
@@ -943,7 +943,7 @@ def test_check_windows_path_length_long_path_warns(
             "esphome.espidf.framework._windows_long_paths_enabled", return_value=False
         ),
         patch(
-            "esphome.espidf.framework._get_idf_tools_path",
+            "esphome.espidf.framework.get_idf_tools_path",
             return_value=_LONG_IDF_PATH,
         ),
         caplog.at_level(logging.WARNING),

tests/unit_tests/test_nrf52_framework.py

@@ -10,12 +10,28 @@ from esphome.components.nrf52.framework import (
     _TOOLCHAIN_VERSION,
     _get_toolchain_platform_info,
     check_and_install,
+    get_sdk_nrf_tools_path,
 )
 from esphome.config_validation import Version
 from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION
 from esphome.core import CORE, EsphomeError
 
 
+@pytest.fixture(autouse=True)
+def _isolate_sdk_nrf_install_path(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Pin the sdk-nrf install root to a tmp dir for every test.
+
+    The default location is the OS user cache dir, so without this any test
+    that builds framework paths or pre-creates the install dir would touch
+    the real ``~/.cache/esphome`` on the developer's machine. Tests that need
+    to exercise the override or default-resolution logic clear/override the
+    env themselves.
+    """
+    monkeypatch.setenv("ESPHOME_SDK_NRF_PREFIX", str(tmp_path / "sdk_nrf_install"))
+
+
 @pytest.mark.parametrize(
     ("system", "machine", "expected"),
     [
@@ -52,7 +68,7 @@ _TEST_SDK_VERSION = "2.9.0"
 def nrf52_dirs(setup_core: Path) -> SimpleNamespace:
     """Populate CORE and pre-create SDK directories so sentinel.touch() succeeds."""
     CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse(_TEST_SDK_VERSION)}
-    tools = CORE.data_dir / "sdk-nrf"
+    tools = get_sdk_nrf_tools_path()
     python_env = tools / "penvs" / f"v{_TEST_SDK_VERSION}"
     framework = tools / "frameworks" / f"v{_TEST_SDK_VERSION}"
     toolchain_dir = tools / "toolchains" / _TOOLCHAIN_VERSION
@@ -226,3 +242,46 @@ class TestCheckAndInstall:
         assert substitutions["sysname"] == "linux"
         assert substitutions["machine"] == "x86_64"
         assert substitutions["extension"] == "tar.xz"
+
+
+# ---------------------------------------------------------------------------
+# get_sdk_nrf_tools_path tests
+# ---------------------------------------------------------------------------
+
+
+def testget_tools_path_env_override(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    override = tmp_path / "custom" / "sdk-nrf"
+    monkeypatch.setenv("ESPHOME_SDK_NRF_PREFIX", str(override))
+    assert get_sdk_nrf_tools_path() == override.resolve()
+
+
+@pytest.mark.parametrize("value", ["", "   "])
+def testget_tools_path_blank_env_falls_back_to_default(
+    value: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """A blank ESPHOME_SDK_NRF_PREFIX is treated as unset, not as CWD.
+
+    Path("") would resolve to the working directory, which clean-all could
+    then delete by accident.
+    """
+    import platformdirs
+
+    monkeypatch.setenv("ESPHOME_SDK_NRF_PREFIX", value)
+    expected = (
+        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "sdk-nrf"
+    ).resolve()
+    assert get_sdk_nrf_tools_path() == expected
+
+
+def testget_tools_path_default_is_global_cache(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import platformdirs
+
+    monkeypatch.delenv("ESPHOME_SDK_NRF_PREFIX", raising=False)
+    expected = (
+        Path(platformdirs.user_cache_dir("esphome", appauthor=False)) / "sdk-nrf"
+    ).resolve()
+    assert get_sdk_nrf_tools_path() == expected

tests/unit_tests/test_writer.py

@@ -68,12 +68,16 @@ def _isolate_platformio_paths(tmp_path_factory: pytest.TempPathFactory) -> Any:
     test_clean_all_partial_exists) install their own inner patch which
     stacks on top of this one and wins for the duration of their block.
 
-    Also pin ``ESPHOME_ESP_IDF_PREFIX`` to a nonexistent tmp dir for the
-    same reason: ``clean_all`` removes the now machine-global ESP-IDF
-    install, which otherwise defaults to the real ``~/.cache/esphome``.
+    Also pin ``ESPHOME_ESP_IDF_PREFIX`` and ``ESPHOME_SDK_NRF_PREFIX`` to
+    nonexistent tmp dirs, and patch ``platformdirs.user_cache_dir``, for the
+    same reason: ``clean_all`` removes the machine-global toolchain installs
+    and their default cache root, which otherwise resolve to the real
+    ``~/.cache/esphome``.
     """
     pio_root = tmp_path_factory.mktemp("isolated_pio") / "nonexistent"
     idf_root = tmp_path_factory.mktemp("isolated_idf") / "nonexistent"
+    sdk_nrf_root = tmp_path_factory.mktemp("isolated_sdk_nrf") / "nonexistent"
+    cache_root = tmp_path_factory.mktemp("isolated_cache") / "nonexistent"
     mock_cfg = MagicMock()
     mock_cfg.get.side_effect = lambda section, option: (
         str(pio_root / option) if section == "platformio" else ""
@@ -83,7 +87,14 @@ def _isolate_platformio_paths(tmp_path_factory: pytest.TempPathFactory) -> Any:
             "platformio.project.config.ProjectConfig.get_instance",
             return_value=mock_cfg,
         ),
-        patch.dict("os.environ", {"ESPHOME_ESP_IDF_PREFIX": str(idf_root)}),
+        patch.dict(
+            "os.environ",
+            {
+                "ESPHOME_ESP_IDF_PREFIX": str(idf_root),
+                "ESPHOME_SDK_NRF_PREFIX": str(sdk_nrf_root),
+            },
+        ),
+        patch("platformdirs.user_cache_dir", return_value=str(cache_root)),
     ):
         yield
 
@@ -1022,6 +1033,55 @@ def test_clean_all_removes_global_idf_install(
     assert str(idf_install.resolve()) in caplog.text
 
 
+@patch("esphome.writer.CORE")
+def test_clean_all_removes_global_sdk_nrf_install(
+    mock_core: MagicMock,
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """clean_all removes the machine-global native sdk-nrf install dir."""
+    sdk_nrf_install = tmp_path / "sdk_nrf_install"
+    (sdk_nrf_install / "frameworks").mkdir(parents=True)
+    monkeypatch.setenv("ESPHOME_SDK_NRF_PREFIX", str(sdk_nrf_install))
+
+    config_dir = tmp_path / "config"
+    config_dir.mkdir()
+
+    from esphome.writer import clean_all
+
+    with caplog.at_level("INFO"):
+        clean_all([str(config_dir)])
+
+    assert not sdk_nrf_install.exists()
+    assert str(sdk_nrf_install.resolve()) in caplog.text
+
+
+@patch("esphome.writer.CORE")
+def test_clean_all_removes_default_cache_root(
+    mock_core: MagicMock,
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """clean_all removes the default cache root (stale/orphaned installs)."""
+    cache_root = tmp_path / "cache_root"
+    (cache_root / "some-old-toolchain").mkdir(parents=True)
+
+    config_dir = tmp_path / "config"
+    config_dir.mkdir()
+
+    from esphome.writer import clean_all
+
+    with (
+        patch("platformdirs.user_cache_dir", return_value=str(cache_root)),
+        caplog.at_level("INFO"),
+    ):
+        clean_all([str(config_dir)])
+
+    assert not cache_root.exists()
+    assert str(cache_root.resolve()) in caplog.text
+
+
 @patch("esphome.writer.CORE")
 def test_clean_all_with_yaml_build_path(
     mock_core: MagicMock,