[substitutions] !include ${filename}, Substitutions in include filename paths (package refactor part 5) (#12213)

Co-authored-by: J. Nick Koston <nick@home-assistant.io>
Co-authored-by: J. Nick Koston <nick@koston.org>

tests/unit_tests/fixtures/substitutions/11-include_path.approved.yaml

@@ -0,0 +1,15 @@
+values:
+  - var1: 4
+  - a: 5
+  - b: 6
+  - c: The value of C is 7
+  - This value comes from inc2.yaml. x is 3, y is 4
+  - From main config, x is 3, y is 2
+  - $a $b $c are out of scope here
+  - keys_in_inc3:
+      x: 3
+      y: 2
+substitutions:
+  x: 3
+  y: 2
+  include_file: inc1

tests/unit_tests/fixtures/substitutions/11-include_path.input.yaml

@@ -0,0 +1,21 @@
+substitutions:
+  include_file: inc1
+  x: 3  # override x from inc2.yaml
+
+packages:
+  my_package: !include
+    file: ${include_file + ".yaml"}  # includes inc1.yaml
+    vars:
+      var1: 4
+      a: ${x+2}
+      b: ${a+1}
+      c: 7
+  other_package: !include
+    file: inc${1+1}.yaml  # includes inc2.yaml
+    vars:
+      y: 4
+
+values:
+  - From main config, x is $x, y is $y
+  - $a $b $c are out of scope here
+  - !include ${"inc" + "3.yaml"}  # includes inc3.yaml here (not a package)

tests/unit_tests/fixtures/substitutions/12-yaml-merge.approved.yaml

@@ -0,0 +1,9 @@
+substitutions:
+  x: 7
+test_list:
+  - content:
+      before: Content before
+      after: Content after
+      keys_in_inc3:
+        x: 7
+        y: 8

tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml

@@ -0,0 +1,10 @@
+substitutions:
+  x: 7
+test_list:
+  - content:
+      before: Content before
+      <<: !include
+        file: inc3.yaml
+        vars:
+          y: 8
+      after: Content after

tests/unit_tests/fixtures/substitutions/inc2.yaml

@@ -0,0 +1,6 @@
+substitutions:
+  x: 1
+  y: 2
+
+values:
+  - This value comes from inc2.yaml. x is $x, y is $y

tests/unit_tests/fixtures/substitutions/inc3.yaml

@@ -0,0 +1,3 @@
+keys_in_inc3:
+  x: ${x}
+  y: ${y}

tests/unit_tests/test_substitutions.py

@@ -8,12 +8,17 @@ import pytest
 
 from esphome import config as config_module, yaml_util
 from esphome.components import substitutions
-from esphome.components.packages import do_packages_pass, merge_packages
+from esphome.components.packages import (
+    MAX_INCLUDE_DEPTH,
+    _PackageProcessor,
+    do_packages_pass,
+    merge_packages,
+)
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, merge_config
 import esphome.config_validation as cv
 from esphome.const import CONF_SUBSTITUTIONS
-from esphome.core import CORE, Lambda
+from esphome.core import CORE, EsphomeError, Lambda
 from esphome.util import OrderedDict
 
 _LOGGER = logging.getLogger(__name__)
@@ -630,3 +635,62 @@ def test_do_substitution_pass_substitutions_must_be_mapping_from_config() -> Non
         cv.Invalid, match="Substitutions must be a key to value mapping"
     ):
         substitutions.do_substitution_pass(config)
+
+
+# ── IncludeFile / package loading tests ────────────────────────────────────
+
+
+def test_resolve_package_max_depth_exceeded(tmp_path: Path) -> None:
+    """A yaml_loader that always returns another IncludeFile triggers the depth guard."""
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    # Each call to the loader returns a fresh IncludeFile pointing at itself,
+    # so PACKAGE_SCHEMA always sees an IncludeFile and never a dict.
+    def always_returns_include(path: Path) -> yaml_util.IncludeFile:
+        return yaml_util.IncludeFile(parent, path.name, None, always_returns_include)
+
+    package_config = yaml_util.IncludeFile(
+        parent, "test.yaml", None, always_returns_include
+    )
+    processor = _PackageProcessor({}, None, False)
+    with pytest.raises(
+        cv.Invalid,
+        match=f"Maximum include nesting depth \\({MAX_INCLUDE_DEPTH}\\) exceeded",
+    ):
+        processor.resolve_package(package_config, substitutions.ContextVars())
+
+
+def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None:
+    """!include with an undefined substitution variable raises cv.Invalid.
+
+    The error message must reference the unresolved filename template so the
+    user knows which include failed, rather than seeing a bare file-not-found.
+    """
+    main_file = tmp_path / "main.yaml"
+    main_file.write_text("result: !include ${undefined_var}.yaml\n")
+
+    config = yaml_util.load_yaml(main_file)
+    with pytest.raises(cv.Invalid, match=r"\$\{undefined_var\}"):
+        substitutions.do_substitution_pass(config)
+
+
+def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None:
+    """An undefined substitution in a package include filename raises cv.Invalid.
+
+    Previously this would raise an unhandled UndefinedError. With
+    strict_undefined=False, the unresolved filename passes through to
+    file loading which produces a clean cv.Invalid error.
+    """
+    parent = tmp_path / "main.yaml"
+    parent.write_text("")
+
+    def loader(path: Path):
+        raise EsphomeError(f"Error reading file {path}: No such file")
+
+    package_config = yaml_util.IncludeFile(
+        parent, "${undefined_var}.yaml", None, loader
+    )
+    processor = _PackageProcessor({}, None, False)
+    with pytest.raises(cv.Invalid, match="unresolved substitutions"):
+        processor.resolve_package(package_config, substitutions.ContextVars())

tests/unit_tests/test_yaml_util.py

@@ -1,3 +1,4 @@
+import io
 from pathlib import Path
 import shutil
 from unittest.mock import patch
@@ -7,6 +8,7 @@ import pytest
 from esphome import core, yaml_util
 from esphome.components import substitutions
 from esphome.config_helpers import Extend, Remove
+import esphome.config_validation as cv
 from esphome.core import EsphomeError
 from esphome.util import OrderedDict
 
@@ -74,7 +76,9 @@ def test_parsing_with_custom_loader(fixture_path):
         loader_calls.append(fname)
 
     with yaml_file.open(encoding="utf-8") as f_handle:
-        yaml_util.parse_yaml(yaml_file, f_handle, custom_loader)
+        config = yaml_util.parse_yaml(yaml_file, f_handle, custom_loader)
+        # substitute config to expand includes:
+        substitutions.substitute(config, [], substitutions.ContextVars(), False)
 
     assert len(loader_calls) == 3
     assert loader_calls[0].parts[-2:] == ("includes", "included.yaml")
@@ -348,7 +352,9 @@ def test_track_yaml_loads_records_includes(tmp_path: Path) -> None:
     main.write_text("child: !include included.yaml\n")
 
     with yaml_util.track_yaml_loads() as loaded:
-        yaml_util.load_yaml(main)
+        result = yaml_util.load_yaml(main)
+        # !include is deferred; resolve it to trigger the nested load
+        result["child"].load()
 
     resolved = [p.name for p in loaded]
     assert "main.yaml" in resolved
@@ -500,3 +506,161 @@ def test_represent_extend() -> None:
 def test_represent_remove() -> None:
     """Test that Remove objects are dumped as plain !remove scalars."""
     assert yaml_util.dump({"key": Remove("my_id")}) == "key: !remove 'my_id'\n"
+
+
+# ── IncludeFile unit tests ──────────────────────────────────────────────────
+
+
+def test_include_file_repr(tmp_path: Path) -> None:
+    """repr() includes the filename so it appears usefully in error messages."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, "some/nested.yaml", None, lambda _: {})
+    assert repr(include) == "IncludeFile(some/nested.yaml)"
+
+
+def test_include_file_load_caches_result(tmp_path: Path) -> None:
+    """load() invokes the yaml_loader only once; subsequent calls return the cached object."""
+    parent = tmp_path / "main.yaml"
+    content = {"key": "value"}
+    call_count = 0
+
+    def counting_loader(_):
+        nonlocal call_count
+        call_count += 1
+        return content
+
+    include = yaml_util.IncludeFile(parent, "child.yaml", None, counting_loader)
+    first = include.load()
+    second = include.load()
+
+    assert call_count == 1
+    assert first is second
+
+
+def test_include_file_load_caches_none_result(tmp_path: Path) -> None:
+    """load() caches None content (empty YAML files) and does not re-invoke the loader."""
+    parent = tmp_path / "main.yaml"
+    call_count = 0
+
+    def counting_loader(_):
+        nonlocal call_count
+        call_count += 1
+
+    include = yaml_util.IncludeFile(parent, "empty.yaml", None, counting_loader)
+    first = include.load()
+    second = include.load()
+
+    assert call_count == 1
+    assert first is None
+    assert second is None
+
+
+def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> None:
+    """load() raises if the filename contains unresolved substitutions or expressions."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", None, lambda _: {})
+    with pytest.raises(cv.Invalid, match="unresolved"):
+        include.load()
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected"),
+    [
+        ("device-${platform}.yaml", True),
+        ("$platform.yaml", True),
+        ("${a + b}.yaml", True),  # Jinja expression
+        ("device.yaml", False),
+        ("path/to/device.yaml", False),
+        ("my$file.yaml", True),  # $file is a valid substitution
+        ("price-100$.yaml", False),  # $ at end, not followed by valid substitution
+    ],
+)
+def test_include_file_has_unresolved_expressions(
+    tmp_path: Path, filename: str, expected: bool
+) -> None:
+    """has_unresolved_expressions() detects substitution patterns in the filename."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, filename, None, lambda _: {})
+    assert include.has_unresolved_expressions() == expected
+
+
+def test_include_in_list_context() -> None:
+    """!include of a file returning a list is handled correctly,
+    including when that list itself contains a nested IncludeFile."""
+    parent = Path("/fake/main.yaml")
+
+    # The nested IncludeFile resolves to a plain string value
+    inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: "gamma")
+
+    # The outer IncludeFile returns a list whose last element is itself an IncludeFile,
+    # exercising the substitution pass's ability to recurse into loaded content.
+    outer = yaml_util.IncludeFile(
+        parent, "items.yaml", None, lambda _: ["alpha", "beta", inner]
+    )
+
+    config = OrderedDict({"values": outer})
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["values"] == ["alpha", "beta", "gamma"]
+
+
+def test_include_plain_filename_loads_after_deferred_refactor() -> None:
+    """!include with a plain filename (no $ expressions) still loads correctly.
+
+    Regression guard: the deferred-loading refactor must not break the simple case.
+    """
+    parent = Path("/fake/main.yaml")
+    include = yaml_util.IncludeFile(
+        parent, "child.yaml", None, lambda _: {"answer": 42}
+    )
+
+    config = OrderedDict({"result": include})
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["result"]["answer"] == 42
+
+
+def test_yaml_merge_include_with_filename_substitution_raises() -> None:
+    """<<: !include ${expr} raises a clear error — substitutions in merge-key filenames
+    are not yet supported, and the error message must say so."""
+    yaml_text = "base:\n  existing: value\n  <<: !include ${filename}.yaml\n"
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        yaml_util.parse_yaml(
+            Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
+        )
+
+
+def test_yaml_merge_list_include_with_filename_substitution_raises() -> None:
+    """Substitutions in include filenames within merge-key lists raise a clear error."""
+    yaml_text = "base:\n  existing: value\n  <<:\n    - !include ${filename}.yaml\n"
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        yaml_util.parse_yaml(
+            Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
+        )
+
+
+def test_yaml_merge_chain_include_resolves() -> None:
+    """Chained includes in merge keys resolve through multiple IncludeFile layers."""
+    parent = Path("/fake/main.yaml")
+
+    inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: {"x": 1})
+    outer = yaml_util.IncludeFile(parent, "outer.yaml", None, lambda _: inner)
+
+    yaml_text = "base:\n  existing: value\n  <<: !include outer.yaml\n"
+    config = yaml_util.parse_yaml(parent, io.StringIO(yaml_text), lambda _: outer)
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["base"]["x"] == 1
+    assert config["base"]["existing"] == "value"
+
+
+def test_yaml_merge_chain_include_depth_exceeded() -> None:
+    """Chain includes in merge keys exceeding depth limit raise a clear error."""
+    parent = Path("/fake/main.yaml")
+
+    def self_referencing_loader(path: Path) -> yaml_util.IncludeFile:
+        return yaml_util.IncludeFile(parent, path.name, None, self_referencing_loader)
+
+    yaml_text = "base:\n  <<: !include loop.yaml\n"
+    with pytest.raises(EsphomeError, match="Maximum include chain depth"):
+        yaml_util.parse_yaml(parent, io.StringIO(yaml_text), self_referencing_loader)