[espidf] Run PIO extraScript with SCons-env shim (#16404)

esphome/espidf/component.py

@@ -322,6 +322,36 @@ def _patch_component(component: IDFComponent, first_pass: bool):
         (component.path / "idf_component.yml").write_text("")
 
 
+def _apply_extra_script(component: IDFComponent) -> None:
+    """Run a PIO ``extraScript`` and fold its captured env vars into
+    ``component.data["build"]["flags"]`` so the existing -L/-l/-D
+    extraction in ``generate_cmakelists_txt`` picks them up."""
+    extra_script = component.data.get("build", {}).get("extraScript")
+    if not extra_script:
+        return
+    # Resolve and confine to the component dir so a malicious library.json
+    # can't escape (e.g. ``"extraScript": "../../etc/passwd"``).
+    library_root = component.path.resolve()
+    script_path = (component.path / extra_script).resolve()
+    if not script_path.is_relative_to(library_root) or not script_path.is_file():
+        return
+    from esphome.components.esp32 import get_esp32_variant
+    from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script
+
+    idf_target = get_esp32_variant().lower().replace("-", "")
+    result = run_extra_script(
+        script_path, library_dir=component.path, idf_target=idf_target
+    )
+    extra_flags = captured_as_build_flags(result, library_dir=component.path)
+    if not extra_flags:
+        return
+    flags = component.data.setdefault("build", {}).setdefault("flags", [])
+    if isinstance(flags, str):
+        flags = [flags]
+    flags.extend(extra_flags)
+    component.data["build"]["flags"] = flags
+
+
 T = TypeVar("T")
 
 
@@ -748,13 +778,6 @@ def _check_library_data(data: dict):
     if not valid_framework:
         raise InvalidIDFComponent(f"Unsupported library frameworks: {frameworks}")
 
-    extra_script = data.get("build", {}).get("extraScript", None)
-    if extra_script:
-        _LOGGER.warning(
-            'Extra scripts are not supported. The script "%s" will not be executed.',
-            extra_script,
-        )
-
 
 def _process_dependencies(component: IDFComponent):
     """
@@ -899,9 +922,17 @@ def _generate_idf_component(library: Library, force: bool = False) -> IDFCompone
     # Apply additional patches to the library metadata
     _patch_component(component, False)
 
-    # Check if the component is usable with ESP-IDF
+    # Check if the component is usable with ESP-IDF before executing any
+    # third-party Python from the library (``_apply_extra_script`` below).
     _check_library_data(component.data)
 
+    # If the library declares a PIO ``extraScript``, run it against a
+    # fake SCons env so we can fold its captured LIBPATH/LIBS/etc into
+    # the build-flag pipeline ``generate_cmakelists_txt`` consumes
+    # below. Without this, libraries that wire per-MCU archive linking
+    # via extraScript fail to link under native ESP-IDF.
+    _apply_extra_script(component)
+
     # Handle the dependencies (convert PlatformIO library to ESP-IDF component if needed)
     _process_dependencies(component)
 

esphome/espidf/extra_script.py

@@ -0,0 +1,161 @@
+"""Run a PlatformIO ``extraScript`` against a captured SCons-env stand-in.
+
+PlatformIO libraries occasionally configure per-target link/build state
+via a Python ``extraScript`` declared in ``library.json``'s ``build``
+section instead of static fields. The script runs under SCons during
+PIO's build and mutates the active ``Environment`` (``env.Append``,
+``env.Replace``, …) — chiefly to set ``LIBPATH``/``LIBS`` per chip MCU.
+
+ESPHome's PIO→IDF converter (``_generate_idf_component``) doesn't run
+SCons, so these scripts were previously ignored and any library
+relying on them failed to link under ``toolchain: esp-idf``. This
+module provides a small shim that ``exec``s an extra-script with a
+fake ``env`` object, captures the common ``env.Append(...)`` calls,
+and returns the captured vars so the caller can fold them back into
+the library's generated CMakeLists.
+
+Caveats
+-------
+* Only the ``env.Append`` API is captured. ``env.Replace``,
+  ``env.Prepend``, ``env.AddPreAction``, SCons file generators, and any
+  arbitrary I/O are silently no-ops. Scripts that depend on those will
+  produce incomplete output.
+* Running arbitrary Python from third-party libraries is a non-trivial
+  trust decision. The shim does no sandboxing — anything in the
+  script's process can run. Use only with libraries whose source you
+  trust.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import logging
+import os
+from pathlib import Path
+
+_LOGGER = logging.getLogger(__name__)
+
+# Keys we know how to translate back into ESPHome's build-flag pipeline.
+# Other env.Append kwargs are recorded but ignored downstream.
+_CAPTURED_KEYS = frozenset({"LIBPATH", "LIBS", "CPPDEFINES", "LINKFLAGS", "CPPFLAGS"})
+
+
+@dataclass
+class ExtraScriptResult:
+    """Build-var deltas captured from a PIO extra-script ``env.Append`` call."""
+
+    libpath: list[str] = field(default_factory=list)
+    libs: list[str] = field(default_factory=list)
+    cppdefines: list[str | tuple[str, str]] = field(default_factory=list)
+    linkflags: list[str] = field(default_factory=list)
+    cppflags: list[str] = field(default_factory=list)
+
+
+class _FakeSConsEnv:
+    """Minimal stand-in for SCons ``Environment`` exposed to extra-scripts.
+
+    Implements just enough surface area to let scripts query ``BOARD_MCU``
+    / ``PIOENV`` and call ``env.Append(LIBPATH=…, LIBS=…, …)``. Every
+    other env method swallows silently so unrelated calls don't raise
+    ``AttributeError`` and abort the script.
+    """
+
+    def __init__(self, *, board_mcu: str, pio_env: str) -> None:
+        self._vars: dict[str, str] = {
+            "BOARD_MCU": board_mcu,
+            "PIOPLATFORM": "espressif32",
+            "PIOENV": pio_env,
+        }
+        self.result = ExtraScriptResult()
+
+    # ----- SCons env API the common scripts use -----
+
+    def get(self, key: str, default: str | None = None) -> str | None:
+        return self._vars.get(key, default)
+
+    def Append(self, **kwargs) -> None:  # noqa: N802 (SCons API name)
+        for key, value in kwargs.items():
+            if key not in _CAPTURED_KEYS:
+                continue
+            items = list(value) if isinstance(value, (list, tuple)) else [value]
+            bucket = getattr(self.result, key.lower())
+            bucket.extend(items)
+
+    # ----- Everything else is a no-op so unsupported scripts don't crash -----
+
+    def __getattr__(self, name: str):
+        def _noop(*args, **kwargs):
+            return None
+
+        return _noop
+
+
+def run_extra_script(
+    script_path: Path, *, library_dir: Path, idf_target: str
+) -> ExtraScriptResult:
+    """Execute ``script_path`` with a fake SCons env and return captured vars.
+
+    ``idf_target`` is the active ESP-IDF target name (e.g. ``esp32``,
+    ``esp32s3``); it's exposed to the script as PlatformIO's
+    ``BOARD_MCU`` so chip-conditional logic resolves the same way it
+    would under PIO. The script runs with ``library_dir`` as the
+    process CWD so relative-path lookups (``join``, ``realpath``,
+    ``open``) resolve against the library tree.
+
+    On any exception inside the script we log at debug level and return
+    an empty result — extra-scripts are best-effort, and an unsupported
+    script shouldn't block the build.
+    """
+    env = _FakeSConsEnv(board_mcu=idf_target, pio_env=f"esphome_{idf_target}")
+    code = compile(script_path.read_text(), str(script_path), "exec")
+    old_cwd = os.getcwd()
+    try:
+        os.chdir(library_dir)
+        exec(  # noqa: S102 pylint: disable=exec-used
+            code,
+            {
+                "Import": lambda *_args: None,  # SCons-side import; harmless here
+                "env": env,
+                "__file__": str(script_path),
+                "__name__": "__pio_extra_script__",
+            },
+        )
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        _LOGGER.warning("PIO extra-script %s raised %s; skipping", script_path, e)
+        return ExtraScriptResult()
+    finally:
+        os.chdir(old_cwd)
+    return env.result
+
+
+def captured_as_build_flags(
+    result: ExtraScriptResult, *, library_dir: Path
+) -> list[str]:
+    """Translate captured env vars into the ``-L`` / ``-l`` / ``-D`` /
+    raw-flag form ``_generate_cmakelists_txt`` already knows how to consume.
+
+    ``LIBPATH`` entries are made relative to ``library_dir`` so the
+    generated CMakeLists is portable; absolute paths outside the library
+    tree are kept as-is (CMake handles absolute paths in
+    ``target_link_directories`` fine).
+    """
+    flags: list[str] = []
+    library_root = library_dir.resolve()
+    for path in result.libpath:
+        # Anchor relative paths to library_dir (not the current CWD, which
+        # has been restored by the time we get here). Joining an absolute
+        # path against library_dir returns the absolute path unchanged.
+        resolved = (library_dir / path).resolve()
+        try:
+            flags.append(f"-L{resolved.relative_to(library_root)}")
+        except ValueError:
+            flags.append(f"-L{resolved}")
+    flags.extend(f"-l{lib}" for lib in result.libs)
+    for define in result.cppdefines:
+        if isinstance(define, tuple) and len(define) == 2:
+            flags.append(f"-D{define[0]}={define[1]}")
+        else:
+            flags.append(f"-D{define}")
+    flags.extend(result.linkflags)
+    flags.extend(result.cppflags)
+    return flags

tests/unit_tests/test_espidf_component.py

@@ -250,14 +250,126 @@ def test_check_library_data_invalid_framework(esp32_idf_core):
         _check_library_data({"platforms": "*", "frameworks": ["other"]})
 
 
-def test_extra_script_logs_warning(caplog, esp32_idf_core):
-    extra_script = "myscript.sh"
+def test_extra_script_captures_libpath_libs_and_defines(tmp_path):
+    from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script
+
+    (tmp_path / "src" / "esp32").mkdir(parents=True)
+    script = tmp_path / "extra_script.py"
+    script.write_text(
+        "Import('env')\n"
+        "mcu = env.get('BOARD_MCU')\n"
+        "env.Append(\n"
+        "    LIBPATH=[join('src', mcu)],\n"
+        "    LIBS=['algobsec'],\n"
+        "    CPPDEFINES=['FOO', ('BAR', '1')],\n"
+        "    LINKFLAGS=['-Wl,--gc-sections'],\n"
+        ")\n"
+    )
+    # The script uses bare ``join`` (PIO's extra-scripts run inside SCons
+    # where this is in scope). Inject it via the script header so the
+    # shim's exec namespace can resolve it.
+    script.write_text("from os.path import join\n" + script.read_text())
+
+    result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
+
+    assert result.libpath == [os.path.join("src", "esp32")]
+    assert result.libs == ["algobsec"]
+    assert ("BAR", "1") in result.cppdefines
+    assert "FOO" in result.cppdefines
+    assert result.linkflags == ["-Wl,--gc-sections"]
+
+    flags = captured_as_build_flags(result, library_dir=tmp_path)
+    sep = os.sep
+    assert f"-Lsrc{sep}esp32" in flags
+    assert "-lalgobsec" in flags
+    assert "-DFOO" in flags
+    assert "-DBAR=1" in flags
+    assert "-Wl,--gc-sections" in flags
+
+
+def test_extra_script_libpath_relative_resolves_against_library_dir(
+    tmp_path, monkeypatch
+):
+    """Relative LIBPATH entries must resolve against ``library_dir``, not the
+    caller's CWD (the shim restores CWD before ``captured_as_build_flags``
+    runs)."""
+    from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags
+
+    (tmp_path / "lib" / "esp32").mkdir(parents=True)
+    elsewhere = tmp_path.parent / "not_the_library_dir"
+    elsewhere.mkdir(exist_ok=True)
+    monkeypatch.chdir(elsewhere)
+
+    result = ExtraScriptResult(libpath=["lib/esp32"])
+    flags = captured_as_build_flags(result, library_dir=tmp_path)
+
+    sep = os.sep
+    assert flags == [f"-Llib{sep}esp32"]
+
+
+def test_extra_script_libpath_absolute_outside_library_dir(tmp_path):
+    from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags
+
+    outside = tmp_path.parent / "system_lib"
+    outside.mkdir(exist_ok=True)
+    result = ExtraScriptResult(libpath=[str(outside)])
+
+    flags = captured_as_build_flags(result, library_dir=tmp_path)
+    assert flags == [f"-L{outside.resolve()}"]
+
+
+def test_extra_script_failure_returns_empty_result(tmp_path, caplog):
+    from esphome.espidf.extra_script import run_extra_script
+
+    script = tmp_path / "broken.py"
+    script.write_text("raise RuntimeError('boom')\n")
 
     with caplog.at_level("WARNING"):
-        _check_library_data({"build": {"extraScript": extra_script}})
+        result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32")
 
-    assert "not supported" in caplog.text
-    assert "myscript.sh" in caplog.text
+    assert result.libpath == []
+    assert result.libs == []
+    assert "broken.py" in caplog.text
+
+
+def test_apply_extra_script_path_traversal_is_rejected(tmp_path):
+    from esphome.espidf.component import _apply_extra_script
+
+    library_dir = tmp_path / "lib"
+    library_dir.mkdir()
+    outside = tmp_path / "evil.py"
+    outside.write_text("env.Append(LIBS=['pwned'])\n")
+
+    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
+    c.path = library_dir
+    c.data = {"build": {"extraScript": "../evil.py"}}
+
+    _apply_extra_script(c)
+
+    # Nothing was folded into flags: the traversal was rejected before
+    # the script could run.
+    assert "flags" not in c.data["build"]
+
+
+def test_apply_extra_script_merges_into_existing_flags(tmp_path, monkeypatch):
+    from esphome.components import esp32 as esp32_module
+
+    monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32")
+
+    from esphome.espidf.component import _apply_extra_script
+
+    (tmp_path / "src").mkdir()
+    script = tmp_path / "extra.py"
+    script.write_text("env.Append(LIBS=['algobsec'])\n")
+
+    c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy"))
+    c.path = tmp_path
+    c.data = {"build": {"extraScript": "extra.py", "flags": ["-DEXISTING"]}}
+
+    _apply_extra_script(c)
+
+    assert "-DEXISTING" in c.data["build"]["flags"]
+    assert "-lalgobsec" in c.data["build"]["flags"]
 
 
 def test_parse_library_json(tmp_path):