[substitutions] !include ${filename}, Substitutions in include filename paths (package refactor part 5) (#12213)

Co-authored-by: J. Nick Koston <nick@home-assistant.io>
Co-authored-by: J. Nick Koston <nick@koston.org>

tests/unit_tests/test_yaml_util.py

@@ -1,3 +1,4 @@
+import io
 from pathlib import Path
 import shutil
 from unittest.mock import patch
@@ -7,6 +8,7 @@ import pytest
 from esphome import core, yaml_util
 from esphome.components import substitutions
 from esphome.config_helpers import Extend, Remove
+import esphome.config_validation as cv
 from esphome.core import EsphomeError
 from esphome.util import OrderedDict
 
@@ -74,7 +76,9 @@ def test_parsing_with_custom_loader(fixture_path):
         loader_calls.append(fname)
 
     with yaml_file.open(encoding="utf-8") as f_handle:
-        yaml_util.parse_yaml(yaml_file, f_handle, custom_loader)
+        config = yaml_util.parse_yaml(yaml_file, f_handle, custom_loader)
+        # substitute config to expand includes:
+        substitutions.substitute(config, [], substitutions.ContextVars(), False)
 
     assert len(loader_calls) == 3
     assert loader_calls[0].parts[-2:] == ("includes", "included.yaml")
@@ -348,7 +352,9 @@ def test_track_yaml_loads_records_includes(tmp_path: Path) -> None:
     main.write_text("child: !include included.yaml\n")
 
     with yaml_util.track_yaml_loads() as loaded:
-        yaml_util.load_yaml(main)
+        result = yaml_util.load_yaml(main)
+        # !include is deferred; resolve it to trigger the nested load
+        result["child"].load()
 
     resolved = [p.name for p in loaded]
     assert "main.yaml" in resolved
@@ -500,3 +506,161 @@ def test_represent_extend() -> None:
 def test_represent_remove() -> None:
     """Test that Remove objects are dumped as plain !remove scalars."""
     assert yaml_util.dump({"key": Remove("my_id")}) == "key: !remove 'my_id'\n"
+
+
+# ── IncludeFile unit tests ──────────────────────────────────────────────────
+
+
+def test_include_file_repr(tmp_path: Path) -> None:
+    """repr() includes the filename so it appears usefully in error messages."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, "some/nested.yaml", None, lambda _: {})
+    assert repr(include) == "IncludeFile(some/nested.yaml)"
+
+
+def test_include_file_load_caches_result(tmp_path: Path) -> None:
+    """load() invokes the yaml_loader only once; subsequent calls return the cached object."""
+    parent = tmp_path / "main.yaml"
+    content = {"key": "value"}
+    call_count = 0
+
+    def counting_loader(_):
+        nonlocal call_count
+        call_count += 1
+        return content
+
+    include = yaml_util.IncludeFile(parent, "child.yaml", None, counting_loader)
+    first = include.load()
+    second = include.load()
+
+    assert call_count == 1
+    assert first is second
+
+
+def test_include_file_load_caches_none_result(tmp_path: Path) -> None:
+    """load() caches None content (empty YAML files) and does not re-invoke the loader."""
+    parent = tmp_path / "main.yaml"
+    call_count = 0
+
+    def counting_loader(_):
+        nonlocal call_count
+        call_count += 1
+
+    include = yaml_util.IncludeFile(parent, "empty.yaml", None, counting_loader)
+    first = include.load()
+    second = include.load()
+
+    assert call_count == 1
+    assert first is None
+    assert second is None
+
+
+def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> None:
+    """load() raises if the filename contains unresolved substitutions or expressions."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", None, lambda _: {})
+    with pytest.raises(cv.Invalid, match="unresolved"):
+        include.load()
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected"),
+    [
+        ("device-${platform}.yaml", True),
+        ("$platform.yaml", True),
+        ("${a + b}.yaml", True),  # Jinja expression
+        ("device.yaml", False),
+        ("path/to/device.yaml", False),
+        ("my$file.yaml", True),  # $file is a valid substitution
+        ("price-100$.yaml", False),  # $ at end, not followed by valid substitution
+    ],
+)
+def test_include_file_has_unresolved_expressions(
+    tmp_path: Path, filename: str, expected: bool
+) -> None:
+    """has_unresolved_expressions() detects substitution patterns in the filename."""
+    parent = tmp_path / "main.yaml"
+    include = yaml_util.IncludeFile(parent, filename, None, lambda _: {})
+    assert include.has_unresolved_expressions() == expected
+
+
+def test_include_in_list_context() -> None:
+    """!include of a file returning a list is handled correctly,
+    including when that list itself contains a nested IncludeFile."""
+    parent = Path("/fake/main.yaml")
+
+    # The nested IncludeFile resolves to a plain string value
+    inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: "gamma")
+
+    # The outer IncludeFile returns a list whose last element is itself an IncludeFile,
+    # exercising the substitution pass's ability to recurse into loaded content.
+    outer = yaml_util.IncludeFile(
+        parent, "items.yaml", None, lambda _: ["alpha", "beta", inner]
+    )
+
+    config = OrderedDict({"values": outer})
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["values"] == ["alpha", "beta", "gamma"]
+
+
+def test_include_plain_filename_loads_after_deferred_refactor() -> None:
+    """!include with a plain filename (no $ expressions) still loads correctly.
+
+    Regression guard: the deferred-loading refactor must not break the simple case.
+    """
+    parent = Path("/fake/main.yaml")
+    include = yaml_util.IncludeFile(
+        parent, "child.yaml", None, lambda _: {"answer": 42}
+    )
+
+    config = OrderedDict({"result": include})
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["result"]["answer"] == 42
+
+
+def test_yaml_merge_include_with_filename_substitution_raises() -> None:
+    """<<: !include ${expr} raises a clear error — substitutions in merge-key filenames
+    are not yet supported, and the error message must say so."""
+    yaml_text = "base:\n  existing: value\n  <<: !include ${filename}.yaml\n"
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        yaml_util.parse_yaml(
+            Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
+        )
+
+
+def test_yaml_merge_list_include_with_filename_substitution_raises() -> None:
+    """Substitutions in include filenames within merge-key lists raise a clear error."""
+    yaml_text = "base:\n  existing: value\n  <<:\n    - !include ${filename}.yaml\n"
+    with pytest.raises(EsphomeError, match="not supported yet"):
+        yaml_util.parse_yaml(
+            Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
+        )
+
+
+def test_yaml_merge_chain_include_resolves() -> None:
+    """Chained includes in merge keys resolve through multiple IncludeFile layers."""
+    parent = Path("/fake/main.yaml")
+
+    inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: {"x": 1})
+    outer = yaml_util.IncludeFile(parent, "outer.yaml", None, lambda _: inner)
+
+    yaml_text = "base:\n  existing: value\n  <<: !include outer.yaml\n"
+    config = yaml_util.parse_yaml(parent, io.StringIO(yaml_text), lambda _: outer)
+    config = substitutions.do_substitution_pass(config)
+
+    assert config["base"]["x"] == 1
+    assert config["base"]["existing"] == "value"
+
+
+def test_yaml_merge_chain_include_depth_exceeded() -> None:
+    """Chain includes in merge keys exceeding depth limit raise a clear error."""
+    parent = Path("/fake/main.yaml")
+
+    def self_referencing_loader(path: Path) -> yaml_util.IncludeFile:
+        return yaml_util.IncludeFile(parent, path.name, None, self_referencing_loader)
+
+    yaml_text = "base:\n  <<: !include loop.yaml\n"
+    with pytest.raises(EsphomeError, match="Maximum include chain depth"):
+        yaml_util.parse_yaml(parent, io.StringIO(yaml_text), self_referencing_loader)