[substitutions] Fix substitutions: !include file.yaml regression (#15850)

esphome/components/packages/__init__.py

@@ -10,6 +10,7 @@ from esphome.components.substitutions import (
     ContextVars,
     push_context,
     resolve_include,
+    resolve_substitutions_block,
     substitute,
 )
 from esphome.components.substitutions.jinja import has_jinja
@@ -516,7 +517,12 @@ def do_packages_pass(
     if CONF_PACKAGES not in config:
         return config
 
-    substitutions = UserDict(config.pop(CONF_SUBSTITUTIONS, {}))
+    with cv.prepend_path(CONF_SUBSTITUTIONS):
+        substitutions = UserDict(
+            resolve_substitutions_block(
+                config.pop(CONF_SUBSTITUTIONS, {}), command_line_substitutions
+            )
+        )
     processor = _PackageProcessor(
         substitutions, command_line_substitutions, skip_update
     )

esphome/components/substitutions/__init__.py

@@ -414,6 +414,34 @@ def _warn_unresolved_variables(errors: ErrList) -> None:
         )
 
 
+def resolve_substitutions_block(
+    substitutions: Any,
+    command_line_substitutions: dict[str, Any] | None,
+) -> dict[str, Any]:
+    """Resolve a deferred ``substitutions: !include file.yaml`` and validate the shape.
+
+    The caller is responsible for wrapping the call in
+    ``cv.prepend_path(CONF_SUBSTITUTIONS)`` for error reporting.
+    ``command_line_substitutions`` seeds the filename context so
+    ``substitutions: !include ${var}.yaml`` can reference CLI-provided vars.
+    """
+    if isinstance(substitutions, IncludeFile):
+        # Single-shot resolution — matches ``_walk_packages`` for the
+        # ``packages: !include`` entry point.  Chained includes (an include that
+        # itself loads another ``!include`` at the top level) are not supported.
+        substitutions, _ = resolve_include(
+            substitutions,
+            [],
+            ContextVars(command_line_substitutions or {}),
+            strict_undefined=False,
+        )
+    if not isinstance(substitutions, dict):
+        raise cv.Invalid(
+            f"Substitutions must be a key to value mapping, got {type(substitutions)}"
+        )
+    return substitutions
+
+
 def do_substitution_pass(
     config: OrderedDict, command_line_substitutions: dict[str, Any] | None = None
 ) -> OrderedDict:
@@ -429,10 +457,9 @@ def do_substitution_pass(
     # Use merge_dicts_ordered to preserve OrderedDict type for move_to_end()
     substitutions = config.pop(CONF_SUBSTITUTIONS, {})
     with cv.prepend_path(CONF_SUBSTITUTIONS):
-        if not isinstance(substitutions, dict):
-            raise cv.Invalid(
-                f"Substitutions must be a key to value mapping, got {type(substitutions)}"
-            )
+        substitutions = resolve_substitutions_block(
+            substitutions, command_line_substitutions
+        )
         substitutions = merge_dicts_ordered(
             substitutions, command_line_substitutions or {}
         )

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml

@@ -0,0 +1,5 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml

@@ -0,0 +1 @@
+wifi_password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml

@@ -0,0 +1,5 @@
+substitutions:
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml

@@ -0,0 +1,9 @@
+substitutions: !include 15-substitutions_inc.yaml
+
+packages:
+  wifi_pkg:
+    wifi:
+      password: $wifi_password
+
+wifi:
+  ssid: main_ssid

tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml

@@ -0,0 +1,6 @@
+substitutions:
+  subs_file: 15-substitutions_inc
+  wifi_password: sub_password
+wifi:
+  ssid: main_ssid
+  password: sub_password

tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml

@@ -0,0 +1,8 @@
+command_line_substitutions:
+  subs_file: 15-substitutions_inc
+
+substitutions: !include ${subs_file}.yaml
+
+wifi:
+  ssid: main_ssid
+  password: $wifi_password

tests/unit_tests/test_substitutions.py

@@ -675,6 +675,57 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None:
         substitutions.do_substitution_pass(config)
 
 
+def test_do_substitution_pass_included_substitutions_must_be_mapping(
+    tmp_path: Path,
+) -> None:
+    """`substitutions: !include list.yaml` where the file holds a list raises cv.Invalid.
+
+    Locks in the shape check that runs after the deferred IncludeFile has been
+    resolved.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        return ["not", "a", "mapping"]
+
+    include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
+    config = OrderedDict({CONF_SUBSTITUTIONS: include})
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        substitutions.do_substitution_pass(config)
+
+
+def test_do_packages_pass_included_substitutions_must_be_mapping(
+    tmp_path: Path,
+) -> None:
+    """`substitutions: !include list.yaml` alongside `packages:` raises cv.Invalid.
+
+    Without the shape check, ``UserDict(...)`` would surface a low-level
+    ``TypeError``; the explicit ``cv.Invalid`` points at the substitutions path.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        return ["not", "a", "mapping"]
+
+    include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
+    config = OrderedDict(
+        {
+            CONF_SUBSTITUTIONS: include,
+            "packages": {"noop": {"wifi": {"ssid": "main"}}},
+        }
+    )
+
+    with pytest.raises(
+        cv.Invalid, match="Substitutions must be a key to value mapping"
+    ):
+        do_packages_pass(config)
+
+
 def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None:
     """An undefined substitution in a package include filename raises cv.Invalid.