diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf9d474d7a2..ad39b3f3467 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -261,6 +261,7 @@ jobs: cpp-unit-tests-run-all: ${{ steps.determine.outputs.cpp-unit-tests-run-all }} cpp-unit-tests-components: ${{ steps.determine.outputs.cpp-unit-tests-components }} component-test-batches: ${{ steps.determine.outputs.component-test-batches }} + validate-only-components: ${{ steps.determine.outputs.validate-only-components }} benchmarks: ${{ steps.determine.outputs.benchmarks }} steps: - name: Check out code from GitHub @@ -305,6 +306,7 @@ jobs: echo "cpp-unit-tests-run-all=$(echo "$output" | jq -r '.cpp_unit_tests_run_all')" >> $GITHUB_OUTPUT echo "cpp-unit-tests-components=$(echo "$output" | jq -c '.cpp_unit_tests_components')" >> $GITHUB_OUTPUT echo "component-test-batches=$(echo "$output" | jq -c '.component_test_batches')" >> $GITHUB_OUTPUT + echo "validate-only-components=$(echo "$output" | jq -c '.validate_only_components')" >> $GITHUB_OUTPUT echo "benchmarks=$(echo "$output" | jq -r '.benchmarks')" >> $GITHUB_OUTPUT - name: Save components graph cache if: github.ref == 'refs/heads/dev' @@ -775,13 +777,45 @@ jobs: echo "Config validation passed! Starting compilation..." echo "" + # Compute the compile-stage component list. Components whose only + # changes are validate.*.yaml files are config-only -- their source + # and test fixtures didn't move, so rebuilding firmware adds no + # signal. Subtract them from this batch before invoking compile. + validate_only_json='${{ needs.determine-jobs.outputs.validate-only-components }}' + if [ -z "$validate_only_json" ]; then + validate_only_json='[]' + fi + if ! validate_only_csv=$(echo "$validate_only_json" | jq -r 'join(",")'); then + echo "::error::Failed to render validate-only-components as CSV from: $validate_only_json" + exit 1 + fi + if [ -z "$validate_only_csv" ]; then + compile_csv="$components_csv" + else + components_sorted=$(echo "$components_csv" | tr ',' '\n' | sort -u) + validate_sorted=$(echo "$validate_only_csv" | tr ',' '\n' | sort -u) + if ! diff_out=$(comm -23 <(echo "$components_sorted") <(echo "$validate_sorted")); then + echo "::error::Failed to compute compile component subset." + exit 1 + fi + compile_csv=$(echo "$diff_out" | paste -sd ',' -) + skipped=$(comm -12 <(echo "$components_sorted") <(echo "$validate_sorted") | paste -sd ',' -) + if [ -n "$skipped" ]; then + echo "Validate-only components in this batch (skipping compile): $skipped" + fi + fi + # Show disk space before compilation echo "Disk space before compilation:" df -h echo "" - # Run compilation with grouping and isolation - python3 script/test_build_components.py -e compile -c "$components_csv" -f --isolate "$directly_changed_csv" + if [ -n "$compile_csv" ]; then + # Run compilation with grouping and isolation + python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv" + else + echo "All components in this batch are validate-only -- skipping compile stage." + fi test-native-idf: name: Test components with native ESP-IDF diff --git a/AGENTS.md b/AGENTS.md index 86f554e9cec..2139a2b796d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -398,13 +398,23 @@ This document provides essential context for AI models interacting with this pro │ ├── i2c/ # I2C bus │ └── spi/ # SPI bus └── components/[component]/ - ├── common.yaml # Component-only config (no bus definitions) - ├── test.esp32-idf.yaml - ├── test.esp8266-ard.yaml - └── test.rp2040-ard.yaml + ├── common.yaml # Component-only config (no bus definitions) + ├── test.esp32-idf.yaml # config + compile + ├── test.esp8266-ard.yaml # config + compile + ├── test-variant.esp32-idf.yaml # variant test, config + compile + ├── validate.esp32-idf.yaml # config-only (never compiled) + └── validate-legacy.esp32-idf.yaml # config-only variant ``` Run them using `script/test_build_components`. Use `-c ` to test specific components and `-t ` for specific platforms. + * **Config-only test files (`validate.*.yaml`):** Use this prefix when a YAML file only needs to exercise schema/validation paths and does not need to be compiled. CI runs `validate.*.yaml` files with `esphome config` only and skips them during compile. The grammar mirrors `test.*.yaml`: + - `validate..yaml` — base config-only test + - `validate-..yaml` — config-only variant + + Use this for things like deprecated-syntax migration tests, schema edge cases, or platform-specific validation branches where building firmware adds no signal. A component may have any mix of `test.*.yaml` and `validate.*.yaml` files. Validate files never participate in bus-grouping; each one runs as its own `esphome config` invocation. + + When a PR's only edits to a component are `validate.*.yaml` files (no source changes, no `test.*.yaml` changes, and the component isn't pulled in as a dependency of another changed component), CI skips the compile stage for that component entirely and only runs config validation. This is decided in `script/determine-jobs.py` via `_component_change_is_validate_only` and surfaced as the `validate_only_components` output that the `test-build-components-split` job consumes. + * **Test Grouping with Packages:** Components that use shared bus packages can be grouped together in CI to reduce build count. **Never define buses (uart, i2c, spi, modbus) directly in test YAML files** — always use packages from `test_build_components/common/`: ```yaml # test.esp32-idf.yaml — use packages for buses diff --git a/script/analyze_component_buses.py b/script/analyze_component_buses.py index 17af7af5771..1d86d5c71ca 100755 --- a/script/analyze_component_buses.py +++ b/script/analyze_component_buses.py @@ -34,7 +34,7 @@ from typing import Any # Add esphome to path sys.path.insert(0, str(Path(__file__).parent.parent)) -from helpers import BASE_BUS_COMPONENTS +from helpers import BASE_BUS_COMPONENTS, is_validate_only_file from esphome import yaml_util from esphome.config_helpers import Extend, Remove @@ -283,6 +283,13 @@ def analyze_component(component_dir: Path) -> tuple[dict[str, list[str]], bool, # Analyze all YAML files in the component directory for yaml_file in component_dir.glob("*.yaml"): + # validate.*.yaml files are config-only -- they don't compile, so + # their contents must not influence compile-time grouping decisions + # (e.g. a !extend used only to exercise schema validation must not + # disqualify the whole component from being grouped). + if is_validate_only_file(yaml_file): + continue + analysis = analyze_yaml_file(yaml_file) # Track if any file uses extend/remove diff --git a/script/ci-custom.py b/script/ci-custom.py index 25db32105c6..56ca0d0355e 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -1068,7 +1068,12 @@ PACKAGE_BUS_RE = re.compile( ) -@lint_content_check(include=["tests/components/*/test.*.yaml"]) +@lint_content_check( + include=[ + "tests/components/*/test.*.yaml", + "tests/components/*/validate.*.yaml", + ] +) def lint_test_package_key_matches_bus(fname, content): """Ensure package keys match the common bus directory name. diff --git a/script/determine-jobs.py b/script/determine-jobs.py index b8f324784d8..57b3c6eb88e 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -29,6 +29,8 @@ The CI workflow uses this information to: - Skip or run downstream esphome/device-builder tests against the PR's Python code - Determine which components to test individually - Decide how to split component tests (if there are many) +- Identify directly-changed components whose only edits are validate.*.yaml files, + so CI can skip the compile stage for them and run config validation only - Run memory impact analysis whenever there are changed components (merged config), and also for core-only changes Usage: @@ -68,6 +70,7 @@ from helpers import ( get_integration_test_files_for_components, get_target_branch, git_ls_files, + is_validate_only_file, parse_test_filename, root_path, ) @@ -600,14 +603,41 @@ def _component_has_tests(component: str) -> bool: """Check if a component has test files. Cached to avoid repeated filesystem operations for the same component. + Validate files (validate.*.yaml) count -- they exercise schema validation + in CI even though they are never compiled. Args: component: Component name to check Returns: - True if the component has test YAML files + True if the component has test or validate YAML files """ - return bool(get_component_test_files(component, all_variants=True)) + return bool( + get_component_test_files(component, all_variants=True, include_validate=True) + ) + + +def _component_change_is_validate_only(component: str, changed: list[str]) -> bool: + """Return True if every changed file for this component is a validate.*.yaml. + + Used to decide whether a directly-changed component can skip the compile + stage in CI. A component qualifies when: + - at least one file under ``tests/components//`` changed, AND + - no source file under ``esphome/components//`` changed, AND + - every changed test file is a ``validate.*.yaml`` or + ``validate-*.yaml`` (i.e. no regular ``test.*.yaml`` was touched). + """ + test_prefix = f"tests/components/{component}/" + src_prefix = f"esphome/components/{component}/" + test_changes: list[Path] = [] + for path in changed: + if path.startswith(src_prefix): + return False + if path.startswith(test_prefix): + test_changes.append(Path(path)) + if not test_changes: + return False + return all(is_validate_only_file(p) for p in test_changes) def _select_platform_by_preference( @@ -977,6 +1007,17 @@ def main() -> None: if component not in directly_changed_components ] + # Components whose only changes are validate.*.yaml files can skip the + # compile stage in CI -- their source and test fixtures didn't move, so + # rebuilding firmware adds no signal. Only directly-changed components + # qualify: a component pulled in transitively (because a dependency + # changed) still needs the compile to catch regressions. + validate_only_components = sorted( + component + for component in directly_changed_with_tests + if _component_change_is_validate_only(component, changed) + ) + # Detect components for memory impact analysis (merged config) memory_impact = detect_memory_impact_config(args.branch) @@ -1073,6 +1114,7 @@ def main() -> None: "cpp_unit_tests_run_all": cpp_run_all, "cpp_unit_tests_components": cpp_components, "component_test_batches": component_test_batches, + "validate_only_components": validate_only_components, "benchmarks": run_benchmarks, } diff --git a/script/helpers.py b/script/helpers.py index 7a6d7ecef68..cf82a89f932 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -117,7 +117,7 @@ def get_component_from_path(file_path: str) -> str | None: def get_component_test_files( - component: str, *, all_variants: bool = False + component: str, *, all_variants: bool = False, include_validate: bool = False ) -> list[Path]: """Get test files for a component. @@ -126,6 +126,10 @@ def get_component_test_files( all_variants: If True, returns all test files including variants (test-*.yaml). If False, returns only base test files (test.*.yaml). Default is False. + include_validate: If True, also returns config-only files (validate.*.yaml, + and validate-*.yaml when all_variants is True). These files + are validated with `esphome config` but never compiled. + Default is False. Returns: List of test file paths for the component, or empty list if none exist @@ -136,9 +140,27 @@ def get_component_test_files( if all_variants: # Match both test.*.yaml and test-*.yaml patterns - return list(tests_dir.glob("test[.-]*.yaml")) + files = list(tests_dir.glob("test[.-]*.yaml")) + if include_validate: + files.extend(tests_dir.glob("validate[.-]*.yaml")) + return files # Match only test.*.yaml (base tests) - return list(tests_dir.glob("test.*.yaml")) + files = list(tests_dir.glob("test.*.yaml")) + if include_validate: + files.extend(tests_dir.glob("validate.*.yaml")) + return files + + +def is_validate_only_file(test_file: Path) -> bool: + """Return True if the given path is a config-only validate file. + + Validate files follow the same grammar as test files but with a + ``validate`` prefix instead of ``test``: ``validate..yaml`` + or ``validate-..yaml``. They are exercised with + ``esphome config`` only and skipped during compile. + """ + name = test_file.name + return name.startswith("validate.") or name.startswith("validate-") @dataclass(frozen=True) diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index d95cdcbe815..0d10246bb48 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -44,14 +44,21 @@ ALL_PLATFORMS = "all" def has_test_files(component_name: str, tests_dir: Path) -> bool: """Check if a component has test files. + Validate files (validate.*.yaml) count -- a component with only config-only + test files still needs a CI runner for schema validation. + Args: component_name: Name of the component tests_dir: Path to tests/components directory (unused, kept for compatibility) Returns: - True if the component has test.*.yaml or test-*.yaml files + True if the component has test.*.yaml, test-*.yaml, or validate.*.yaml files """ - return bool(get_component_test_files(component_name, all_variants=True)) + return bool( + get_component_test_files( + component_name, all_variants=True, include_validate=True + ) + ) def create_intelligent_batches( diff --git a/script/test_build_components.py b/script/test_build_components.py index 10c5e5463fe..43b71004eb8 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -39,7 +39,11 @@ from script.analyze_component_buses import ( merge_compatible_bus_groups, uses_local_file_references, ) -from script.helpers import get_component_test_files, split_conflicting_groups +from script.helpers import ( + get_component_test_files, + is_validate_only_file, + split_conflicting_groups, +) from script.merge_component_configs import merge_component_configs @@ -83,7 +87,10 @@ def show_disk_space_if_ci(esphome_command: str) -> None: def find_component_tests( - components_dir: Path, component_pattern: str = "*", base_only: bool = False + components_dir: Path, + component_pattern: str = "*", + base_only: bool = False, + include_validate: bool = False, ) -> dict[str, list[Path]]: """Find all component test files. @@ -91,6 +98,8 @@ def find_component_tests( components_dir: Path to tests/components directory component_pattern: Glob pattern for component names base_only: If True, only find base test files (test.*.yaml), not variant files (test-*.yaml) + include_validate: If True, also include config-only files (validate.*.yaml). + These are run with `esphome config` only and never compiled. Returns: Dictionary mapping component name to list of test files @@ -102,7 +111,11 @@ def find_component_tests( continue # Get test files using helper function - test_files = get_component_test_files(comp_dir.name, all_variants=not base_only) + test_files = get_component_test_files( + comp_dir.name, + all_variants=not base_only, + include_validate=include_validate, + ) if test_files: component_tests[comp_dir.name] = test_files @@ -836,12 +849,25 @@ def run_grouped_component_tests( # With grouping: # - 1 build per group (regardless of how many components) # - Individual components still need all their platform builds + # - Validate files of grouped components still run individually + # (they're config-only and bypass the grouped compile, see + # run_individual_component_test), so each adds one more invocation. individual_test_file_count = sum( len(all_tests[comp]) for comp in individual_tests if comp in all_tests ) + grouped_component_set = {c for _, _, comps in groups_to_test for c in comps} + grouped_validate_file_count = sum( + 1 + for comp in grouped_component_set + for test_file in all_tests.get(comp, []) + if is_validate_only_file(test_file) + ) + total_grouped_components = sum(len(comps) for _, _, comps in groups_to_test) - total_builds_with_grouping = len(groups_to_test) + individual_test_file_count + total_builds_with_grouping = ( + len(groups_to_test) + individual_test_file_count + grouped_validate_file_count + ) builds_saved = total_test_files - total_builds_with_grouping print(f"\n{'=' * 80}") @@ -854,6 +880,10 @@ def run_grouped_component_tests( print( f" • {individual_test_file_count} individual builds ({len(individual_tests)} components)" ) + if grouped_validate_file_count: + print( + f" • {grouped_validate_file_count} validate-only invocations for grouped components" + ) if total_test_files > 0: reduction_pct = (builds_saved / total_test_files) * 100 print(f" • Saves {builds_saved} builds ({reduction_pct:.1f}% reduction)") @@ -937,8 +967,13 @@ def run_individual_component_test( tested_components: Set of already tested components test_results: List to append test results """ - # Skip if already tested in a group - if (component, platform_with_version) in tested_components: + # Validate files (validate.*.yaml) are config-only and never participate + # in compile-time bus grouping, so always run them individually even when + # the (component, platform) pair was covered by a group test. + if ( + not is_validate_only_file(test_file) + and (component, platform_with_version) in tested_components + ): return test_result = run_esphome_test( @@ -992,13 +1027,23 @@ def test_components( # Get platform base files platform_bases = get_platform_base_files(build_components_dir) + # Validate files (validate.*.yaml) are config-only -- they exercise + # schema/validation paths but are never compiled. Include them when running + # `config` or `clean`; exclude them under `compile` so they never reach a + # toolchain build. + include_validate = esphome_command != "compile" + # Find all component tests all_tests = {} for pattern in component_patterns: # Skip empty patterns (happens when components list is empty string) if not pattern: continue - all_tests.update(find_component_tests(tests_dir, pattern, base_only)) + all_tests.update( + find_component_tests( + tests_dir, pattern, base_only, include_validate=include_validate + ) + ) # If no components found, build a reference configuration for baseline comparison # Create a synthetic "empty" component test that will build just the base config diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index cc795bc5532..5e2dd670dc1 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -2215,3 +2215,230 @@ def test_should_run_benchmarks_with_branch() -> None: mock_changed.return_value = [] determine_jobs.should_run_benchmarks("release") mock_changed.assert_called_with("release") + + +# --------------------------------------------------------------------------- +# _component_change_is_validate_only +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("component", "changed", "expected"), + [ + # Only a base validate file changed. + ( + "foo", + ["tests/components/foo/validate.esp32-idf.yaml"], + True, + ), + # Only a validate variant changed. + ( + "foo", + ["tests/components/foo/validate-legacy.esp32-idf.yaml"], + True, + ), + # Multiple validate files (all validate). + ( + "foo", + [ + "tests/components/foo/validate.esp32-idf.yaml", + "tests/components/foo/validate-legacy.esp32-idf.yaml", + ], + True, + ), + # Mixed: validate + regular test must NOT be classified as validate-only. + ( + "foo", + [ + "tests/components/foo/validate.esp32-idf.yaml", + "tests/components/foo/test.esp32-idf.yaml", + ], + False, + ), + # Regular test only. + ( + "foo", + ["tests/components/foo/test.esp32-idf.yaml"], + False, + ), + # Source change disqualifies even if a validate file is also touched. + ( + "foo", + [ + "esphome/components/foo/foo.cpp", + "tests/components/foo/validate.esp32-idf.yaml", + ], + False, + ), + # No matching files at all. + ("foo", ["esphome/core/helpers.cpp"], False), + # Filenames merely starting with "validate" but not following the + # grammar must not match (defensive against accidental classification). + ( + "foo", + ["tests/components/foo/validatesomething.yaml"], + False, + ), + # An unrelated component's validate change doesn't affect this one. + ( + "foo", + ["tests/components/bar/validate.esp32-idf.yaml"], + False, + ), + # common.yaml change in the component dir disqualifies. + ( + "foo", + [ + "tests/components/foo/common.yaml", + "tests/components/foo/validate.esp32-idf.yaml", + ], + False, + ), + ], +) +def test_component_change_is_validate_only( + component: str, changed: list[str], expected: bool +) -> None: + """The validate-only classifier rejects anything beyond validate.* edits.""" + assert ( + determine_jobs._component_change_is_validate_only(component, changed) + is expected + ) + + +def test_main_emits_validate_only_components( + mock_determine_integration_tests: Mock, + mock_should_run_clang_tidy: Mock, + mock_should_run_clang_format: Mock, + mock_should_run_python_linters: Mock, + mock_should_run_import_time: Mock, + mock_should_run_device_builder: Mock, + mock_changed_files: Mock, + mock_determine_cpp_unit_tests: Mock, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Directly-changed components whose only edits are validate.*.yaml are + listed in `validate_only_components` so CI can skip their compile stage. + """ + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + mock_determine_integration_tests.return_value = (False, []) + mock_should_run_clang_tidy.return_value = False + mock_should_run_clang_format.return_value = False + mock_should_run_python_linters.return_value = False + mock_should_run_import_time.return_value = False + mock_should_run_device_builder.return_value = False + mock_determine_cpp_unit_tests.return_value = (False, []) + + # foo: only validate file changed (qualifies) + # bar: test file changed (does not qualify) + mock_changed_files.return_value = [ + "tests/components/foo/validate.esp32-idf.yaml", + "tests/components/bar/test.esp32-idf.yaml", + ] + + with ( + patch("sys.argv", ["determine-jobs.py"]), + patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), + patch.object( + determine_jobs, + "get_changed_components", + return_value=["foo", "bar"], + ), + patch.object( + determine_jobs, + "filter_component_and_test_files", + side_effect=lambda f: f.startswith("tests/components/"), + ), + patch.object( + determine_jobs, + "get_components_with_dependencies", + side_effect=lambda files, deps: ["foo", "bar"], + ), + patch.object(determine_jobs, "_component_has_tests", return_value=True), + patch.object( + determine_jobs, + "detect_memory_impact_config", + return_value={"should_run": "false"}, + ), + patch.object( + determine_jobs, + "create_intelligent_batches", + return_value=([["foo", "bar"]], {}), + ), + ): + determine_jobs.main() + + output = json.loads(capsys.readouterr().out) + assert output["validate_only_components"] == ["foo"] + + +def test_main_validate_only_excludes_transitive_components( + mock_determine_integration_tests: Mock, + mock_should_run_clang_tidy: Mock, + mock_should_run_clang_format: Mock, + mock_should_run_python_linters: Mock, + mock_should_run_import_time: Mock, + mock_should_run_device_builder: Mock, + mock_changed_files: Mock, + mock_determine_cpp_unit_tests: Mock, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A component pulled in only as a dependency must NOT be considered + validate-only, even if it has no source changes -- its dependency moved, + so the compile is still required. + """ + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + mock_determine_integration_tests.return_value = (False, []) + mock_should_run_clang_tidy.return_value = False + mock_should_run_clang_format.return_value = False + mock_should_run_python_linters.return_value = False + mock_should_run_import_time.return_value = False + mock_should_run_device_builder.return_value = False + mock_determine_cpp_unit_tests.return_value = (False, []) + + # Only foo's validate file changed directly. bar is a transitive dep. + mock_changed_files.return_value = [ + "tests/components/foo/validate.esp32-idf.yaml", + ] + + with ( + patch("sys.argv", ["determine-jobs.py"]), + patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), + patch.object( + determine_jobs, + "get_changed_components", + return_value=["foo", "bar"], # bar pulled in via dependencies + ), + patch.object( + determine_jobs, + "filter_component_and_test_files", + side_effect=lambda f: f.startswith("tests/components/"), + ), + patch.object( + determine_jobs, + "get_components_with_dependencies", + # deps=False -> directly_changed = [foo]; deps=True -> [foo, bar] + side_effect=lambda files, deps: ["foo", "bar"] if deps else ["foo"], + ), + patch.object(determine_jobs, "_component_has_tests", return_value=True), + patch.object( + determine_jobs, + "detect_memory_impact_config", + return_value={"should_run": "false"}, + ), + patch.object( + determine_jobs, + "create_intelligent_batches", + return_value=([["foo", "bar"]], {}), + ), + ): + determine_jobs.main() + + output = json.loads(capsys.readouterr().out) + # Only foo (directly changed, validate-only). bar is a transitive dep + # and still needs compile despite no source change of its own. + assert output["validate_only_components"] == ["foo"] diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index db0d2908f47..10f258aa830 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -1624,3 +1624,171 @@ def test_split_conflicting_groups_preserves_original_signature_for_first_bucket( platform, signature = next(iter(extra)) assert platform == "esp32" assert signature.startswith("i2c__conflict") + + +# --------------------------------------------------------------------------- +# get_component_test_files / is_validate_only_file +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_component_tests(tmp_path: Path) -> Path: + """Create a fake tests/components/ tree and return the repo root. + + Layout for component "demo": + test.esp32-idf.yaml + test.esp8266-ard.yaml + test-variant.esp32-idf.yaml + validate.esp32-idf.yaml + validate-legacy.esp32-idf.yaml + + Layout for component "validate_only": + validate.esp32-idf.yaml (only validate files) + + Layout for component "no_tests": + common.yaml (no test/validate files at all) + """ + tests_dir = tmp_path / "tests" / "components" + + demo = tests_dir / "demo" + demo.mkdir(parents=True) + (demo / "test.esp32-idf.yaml").write_text("") + (demo / "test.esp8266-ard.yaml").write_text("") + (demo / "test-variant.esp32-idf.yaml").write_text("") + (demo / "validate.esp32-idf.yaml").write_text("") + (demo / "validate-legacy.esp32-idf.yaml").write_text("") + + validate_only = tests_dir / "validate_only" + validate_only.mkdir(parents=True) + (validate_only / "validate.esp32-idf.yaml").write_text("") + + no_tests = tests_dir / "no_tests" + no_tests.mkdir(parents=True) + (no_tests / "common.yaml").write_text("") + + return tmp_path + + +def _names(paths: list[Path]) -> set[str]: + return {p.name for p in paths} + + +def test_get_component_test_files_default_excludes_validate( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """Default behaviour: only base test.*.yaml; no variants, no validate.""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + files = helpers.get_component_test_files("demo") + + assert _names(files) == {"test.esp32-idf.yaml", "test.esp8266-ard.yaml"} + + +def test_get_component_test_files_all_variants_excludes_validate( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """all_variants=True picks up test variants but still skips validate.""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + files = helpers.get_component_test_files("demo", all_variants=True) + + assert _names(files) == { + "test.esp32-idf.yaml", + "test.esp8266-ard.yaml", + "test-variant.esp32-idf.yaml", + } + + +def test_get_component_test_files_include_validate_base_only( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """include_validate=True with base-only adds validate.*.yaml only.""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + files = helpers.get_component_test_files("demo", include_validate=True) + + assert _names(files) == { + "test.esp32-idf.yaml", + "test.esp8266-ard.yaml", + "validate.esp32-idf.yaml", + } + + +def test_get_component_test_files_include_validate_all_variants( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """include_validate=True with all_variants adds validate variants too.""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + files = helpers.get_component_test_files( + "demo", all_variants=True, include_validate=True + ) + + assert _names(files) == { + "test.esp32-idf.yaml", + "test.esp8266-ard.yaml", + "test-variant.esp32-idf.yaml", + "validate.esp32-idf.yaml", + "validate-legacy.esp32-idf.yaml", + } + + +def test_get_component_test_files_validate_only_component( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """A component with only validate files is invisible without the flag.""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + assert helpers.get_component_test_files("validate_only") == [] + assert helpers.get_component_test_files("validate_only", all_variants=True) == [] + + files = helpers.get_component_test_files( + "validate_only", all_variants=True, include_validate=True + ) + assert _names(files) == {"validate.esp32-idf.yaml"} + + +def test_get_component_test_files_missing_component( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """Unknown components return an empty list, regardless of flags.""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + assert ( + helpers.get_component_test_files( + "does_not_exist", all_variants=True, include_validate=True + ) + == [] + ) + + +def test_get_component_test_files_component_without_tests( + fake_component_tests: Path, monkeypatch: MonkeyPatch +) -> None: + """A component with only common.yaml and no test/validate files returns [].""" + monkeypatch.setattr(helpers, "root_path", str(fake_component_tests)) + + assert ( + helpers.get_component_test_files( + "no_tests", all_variants=True, include_validate=True + ) + == [] + ) + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + ("validate.esp32-idf.yaml", True), + ("validate-legacy.esp32-idf.yaml", True), + ("validate.host.yaml", True), + ("test.esp32-idf.yaml", False), + ("test-variant.esp32-idf.yaml", False), + ("common.yaml", False), + # Defensive: a hypothetical name starting with "validate" but not + # following the grammar must not be classified as a validate file. + ("validatesomething.yaml", False), + ], +) +def test_is_validate_only_file(filename: str, expected: bool, tmp_path: Path) -> None: + assert helpers.is_validate_only_file(tmp_path / filename) is expected