diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index 8cd77dc6d1b..af8640949d1 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -322,6 +322,36 @@ def _patch_component(component: IDFComponent, first_pass: bool): (component.path / "idf_component.yml").write_text("") +def _apply_extra_script(component: IDFComponent) -> None: + """Run a PIO ``extraScript`` and fold its captured env vars into + ``component.data["build"]["flags"]`` so the existing -L/-l/-D + extraction in ``generate_cmakelists_txt`` picks them up.""" + extra_script = component.data.get("build", {}).get("extraScript") + if not extra_script: + return + # Resolve and confine to the component dir so a malicious library.json + # can't escape (e.g. ``"extraScript": "../../etc/passwd"``). + library_root = component.path.resolve() + script_path = (component.path / extra_script).resolve() + if not script_path.is_relative_to(library_root) or not script_path.is_file(): + return + from esphome.components.esp32 import get_esp32_variant + from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script + + idf_target = get_esp32_variant().lower().replace("-", "") + result = run_extra_script( + script_path, library_dir=component.path, idf_target=idf_target + ) + extra_flags = captured_as_build_flags(result, library_dir=component.path) + if not extra_flags: + return + flags = component.data.setdefault("build", {}).setdefault("flags", []) + if isinstance(flags, str): + flags = [flags] + flags.extend(extra_flags) + component.data["build"]["flags"] = flags + + T = TypeVar("T") @@ -748,13 +778,6 @@ def _check_library_data(data: dict): if not valid_framework: raise InvalidIDFComponent(f"Unsupported library frameworks: {frameworks}") - extra_script = data.get("build", {}).get("extraScript", None) - if extra_script: - _LOGGER.warning( - 'Extra scripts are not supported. The script "%s" will not be executed.', - extra_script, - ) - def _process_dependencies(component: IDFComponent): """ @@ -899,9 +922,17 @@ def _generate_idf_component(library: Library, force: bool = False) -> IDFCompone # Apply additional patches to the library metadata _patch_component(component, False) - # Check if the component is usable with ESP-IDF + # Check if the component is usable with ESP-IDF before executing any + # third-party Python from the library (``_apply_extra_script`` below). _check_library_data(component.data) + # If the library declares a PIO ``extraScript``, run it against a + # fake SCons env so we can fold its captured LIBPATH/LIBS/etc into + # the build-flag pipeline ``generate_cmakelists_txt`` consumes + # below. Without this, libraries that wire per-MCU archive linking + # via extraScript fail to link under native ESP-IDF. + _apply_extra_script(component) + # Handle the dependencies (convert PlatformIO library to ESP-IDF component if needed) _process_dependencies(component) diff --git a/esphome/espidf/extra_script.py b/esphome/espidf/extra_script.py new file mode 100644 index 00000000000..2f22f23c100 --- /dev/null +++ b/esphome/espidf/extra_script.py @@ -0,0 +1,161 @@ +"""Run a PlatformIO ``extraScript`` against a captured SCons-env stand-in. + +PlatformIO libraries occasionally configure per-target link/build state +via a Python ``extraScript`` declared in ``library.json``'s ``build`` +section instead of static fields. The script runs under SCons during +PIO's build and mutates the active ``Environment`` (``env.Append``, +``env.Replace``, …) — chiefly to set ``LIBPATH``/``LIBS`` per chip MCU. + +ESPHome's PIO→IDF converter (``_generate_idf_component``) doesn't run +SCons, so these scripts were previously ignored and any library +relying on them failed to link under ``toolchain: esp-idf``. This +module provides a small shim that ``exec``s an extra-script with a +fake ``env`` object, captures the common ``env.Append(...)`` calls, +and returns the captured vars so the caller can fold them back into +the library's generated CMakeLists. + +Caveats +------- +* Only the ``env.Append`` API is captured. ``env.Replace``, + ``env.Prepend``, ``env.AddPreAction``, SCons file generators, and any + arbitrary I/O are silently no-ops. Scripts that depend on those will + produce incomplete output. +* Running arbitrary Python from third-party libraries is a non-trivial + trust decision. The shim does no sandboxing — anything in the + script's process can run. Use only with libraries whose source you + trust. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +import os +from pathlib import Path + +_LOGGER = logging.getLogger(__name__) + +# Keys we know how to translate back into ESPHome's build-flag pipeline. +# Other env.Append kwargs are recorded but ignored downstream. +_CAPTURED_KEYS = frozenset({"LIBPATH", "LIBS", "CPPDEFINES", "LINKFLAGS", "CPPFLAGS"}) + + +@dataclass +class ExtraScriptResult: + """Build-var deltas captured from a PIO extra-script ``env.Append`` call.""" + + libpath: list[str] = field(default_factory=list) + libs: list[str] = field(default_factory=list) + cppdefines: list[str | tuple[str, str]] = field(default_factory=list) + linkflags: list[str] = field(default_factory=list) + cppflags: list[str] = field(default_factory=list) + + +class _FakeSConsEnv: + """Minimal stand-in for SCons ``Environment`` exposed to extra-scripts. + + Implements just enough surface area to let scripts query ``BOARD_MCU`` + / ``PIOENV`` and call ``env.Append(LIBPATH=…, LIBS=…, …)``. Every + other env method swallows silently so unrelated calls don't raise + ``AttributeError`` and abort the script. + """ + + def __init__(self, *, board_mcu: str, pio_env: str) -> None: + self._vars: dict[str, str] = { + "BOARD_MCU": board_mcu, + "PIOPLATFORM": "espressif32", + "PIOENV": pio_env, + } + self.result = ExtraScriptResult() + + # ----- SCons env API the common scripts use ----- + + def get(self, key: str, default: str | None = None) -> str | None: + return self._vars.get(key, default) + + def Append(self, **kwargs) -> None: # noqa: N802 (SCons API name) + for key, value in kwargs.items(): + if key not in _CAPTURED_KEYS: + continue + items = list(value) if isinstance(value, (list, tuple)) else [value] + bucket = getattr(self.result, key.lower()) + bucket.extend(items) + + # ----- Everything else is a no-op so unsupported scripts don't crash ----- + + def __getattr__(self, name: str): + def _noop(*args, **kwargs): + return None + + return _noop + + +def run_extra_script( + script_path: Path, *, library_dir: Path, idf_target: str +) -> ExtraScriptResult: + """Execute ``script_path`` with a fake SCons env and return captured vars. + + ``idf_target`` is the active ESP-IDF target name (e.g. ``esp32``, + ``esp32s3``); it's exposed to the script as PlatformIO's + ``BOARD_MCU`` so chip-conditional logic resolves the same way it + would under PIO. The script runs with ``library_dir`` as the + process CWD so relative-path lookups (``join``, ``realpath``, + ``open``) resolve against the library tree. + + On any exception inside the script we log at debug level and return + an empty result — extra-scripts are best-effort, and an unsupported + script shouldn't block the build. + """ + env = _FakeSConsEnv(board_mcu=idf_target, pio_env=f"esphome_{idf_target}") + code = compile(script_path.read_text(), str(script_path), "exec") + old_cwd = os.getcwd() + try: + os.chdir(library_dir) + exec( # noqa: S102 pylint: disable=exec-used + code, + { + "Import": lambda *_args: None, # SCons-side import; harmless here + "env": env, + "__file__": str(script_path), + "__name__": "__pio_extra_script__", + }, + ) + except Exception as e: # pylint: disable=broad-exception-caught + _LOGGER.warning("PIO extra-script %s raised %s; skipping", script_path, e) + return ExtraScriptResult() + finally: + os.chdir(old_cwd) + return env.result + + +def captured_as_build_flags( + result: ExtraScriptResult, *, library_dir: Path +) -> list[str]: + """Translate captured env vars into the ``-L`` / ``-l`` / ``-D`` / + raw-flag form ``_generate_cmakelists_txt`` already knows how to consume. + + ``LIBPATH`` entries are made relative to ``library_dir`` so the + generated CMakeLists is portable; absolute paths outside the library + tree are kept as-is (CMake handles absolute paths in + ``target_link_directories`` fine). + """ + flags: list[str] = [] + library_root = library_dir.resolve() + for path in result.libpath: + # Anchor relative paths to library_dir (not the current CWD, which + # has been restored by the time we get here). Joining an absolute + # path against library_dir returns the absolute path unchanged. + resolved = (library_dir / path).resolve() + try: + flags.append(f"-L{resolved.relative_to(library_root)}") + except ValueError: + flags.append(f"-L{resolved}") + flags.extend(f"-l{lib}" for lib in result.libs) + for define in result.cppdefines: + if isinstance(define, tuple) and len(define) == 2: + flags.append(f"-D{define[0]}={define[1]}") + else: + flags.append(f"-D{define}") + flags.extend(result.linkflags) + flags.extend(result.cppflags) + return flags diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index caef10eea32..3988c997a76 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -250,14 +250,126 @@ def test_check_library_data_invalid_framework(esp32_idf_core): _check_library_data({"platforms": "*", "frameworks": ["other"]}) -def test_extra_script_logs_warning(caplog, esp32_idf_core): - extra_script = "myscript.sh" +def test_extra_script_captures_libpath_libs_and_defines(tmp_path): + from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script + + (tmp_path / "src" / "esp32").mkdir(parents=True) + script = tmp_path / "extra_script.py" + script.write_text( + "Import('env')\n" + "mcu = env.get('BOARD_MCU')\n" + "env.Append(\n" + " LIBPATH=[join('src', mcu)],\n" + " LIBS=['algobsec'],\n" + " CPPDEFINES=['FOO', ('BAR', '1')],\n" + " LINKFLAGS=['-Wl,--gc-sections'],\n" + ")\n" + ) + # The script uses bare ``join`` (PIO's extra-scripts run inside SCons + # where this is in scope). Inject it via the script header so the + # shim's exec namespace can resolve it. + script.write_text("from os.path import join\n" + script.read_text()) + + result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32") + + assert result.libpath == [os.path.join("src", "esp32")] + assert result.libs == ["algobsec"] + assert ("BAR", "1") in result.cppdefines + assert "FOO" in result.cppdefines + assert result.linkflags == ["-Wl,--gc-sections"] + + flags = captured_as_build_flags(result, library_dir=tmp_path) + sep = os.sep + assert f"-Lsrc{sep}esp32" in flags + assert "-lalgobsec" in flags + assert "-DFOO" in flags + assert "-DBAR=1" in flags + assert "-Wl,--gc-sections" in flags + + +def test_extra_script_libpath_relative_resolves_against_library_dir( + tmp_path, monkeypatch +): + """Relative LIBPATH entries must resolve against ``library_dir``, not the + caller's CWD (the shim restores CWD before ``captured_as_build_flags`` + runs).""" + from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags + + (tmp_path / "lib" / "esp32").mkdir(parents=True) + elsewhere = tmp_path.parent / "not_the_library_dir" + elsewhere.mkdir(exist_ok=True) + monkeypatch.chdir(elsewhere) + + result = ExtraScriptResult(libpath=["lib/esp32"]) + flags = captured_as_build_flags(result, library_dir=tmp_path) + + sep = os.sep + assert flags == [f"-Llib{sep}esp32"] + + +def test_extra_script_libpath_absolute_outside_library_dir(tmp_path): + from esphome.espidf.extra_script import ExtraScriptResult, captured_as_build_flags + + outside = tmp_path.parent / "system_lib" + outside.mkdir(exist_ok=True) + result = ExtraScriptResult(libpath=[str(outside)]) + + flags = captured_as_build_flags(result, library_dir=tmp_path) + assert flags == [f"-L{outside.resolve()}"] + + +def test_extra_script_failure_returns_empty_result(tmp_path, caplog): + from esphome.espidf.extra_script import run_extra_script + + script = tmp_path / "broken.py" + script.write_text("raise RuntimeError('boom')\n") with caplog.at_level("WARNING"): - _check_library_data({"build": {"extraScript": extra_script}}) + result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32") - assert "not supported" in caplog.text - assert "myscript.sh" in caplog.text + assert result.libpath == [] + assert result.libs == [] + assert "broken.py" in caplog.text + + +def test_apply_extra_script_path_traversal_is_rejected(tmp_path): + from esphome.espidf.component import _apply_extra_script + + library_dir = tmp_path / "lib" + library_dir.mkdir() + outside = tmp_path / "evil.py" + outside.write_text("env.Append(LIBS=['pwned'])\n") + + c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy")) + c.path = library_dir + c.data = {"build": {"extraScript": "../evil.py"}} + + _apply_extra_script(c) + + # Nothing was folded into flags: the traversal was rejected before + # the script could run. + assert "flags" not in c.data["build"] + + +def test_apply_extra_script_merges_into_existing_flags(tmp_path, monkeypatch): + from esphome.components import esp32 as esp32_module + + monkeypatch.setattr(esp32_module, "get_esp32_variant", lambda: "ESP32") + + from esphome.espidf.component import _apply_extra_script + + (tmp_path / "src").mkdir() + script = tmp_path / "extra.py" + script.write_text("env.Append(LIBS=['algobsec'])\n") + + c = IDFComponent("owner/name", "1.0", source=URLSource("http://dummy")) + c.path = tmp_path + c.data = {"build": {"extraScript": "extra.py", "flags": ["-DEXISTING"]}} + + _apply_extra_script(c) + + assert "-DEXISTING" in c.data["build"]["flags"] + assert "-lalgobsec" in c.data["build"]["flags"] def test_parse_library_json(tmp_path):