mirror of
https://github.com/esphome/esphome.git
synced 2026-05-19 03:01:49 +08:00
Group component tests to reduce CI time (#11134)
This commit is contained in:
@@ -186,6 +186,11 @@ This document provides essential context for AI models interacting with this pro
|
||||
└── components/[component]/ # Component-specific tests
|
||||
```
|
||||
Run them using `script/test_build_components`. Use `-c <component>` to test specific components and `-t <target>` for specific platforms.
|
||||
* **Testing All Components Together:** To verify that all components can be tested together without ID conflicts or configuration issues, use:
|
||||
```bash
|
||||
./script/test_component_grouping.py -e config --all
|
||||
```
|
||||
This tests all components in a single build to catch conflicts that might not appear when testing components individually. Use `-e config` for fast configuration validation, or `-e compile` for full compilation testing.
|
||||
* **Debugging and Troubleshooting:**
|
||||
* **Debug Tools:**
|
||||
- `esphome config <file>.yaml` to validate configuration.
|
||||
|
||||
+66
-30
@@ -369,10 +369,11 @@ jobs:
|
||||
matrix:
|
||||
file: ${{ fromJson(needs.determine-jobs.outputs.changed-components) }}
|
||||
steps:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libsdl2-dev
|
||||
- name: Cache apt packages
|
||||
uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.5.3
|
||||
with:
|
||||
packages: libsdl2-dev
|
||||
version: 1.0
|
||||
|
||||
- name: Check out code from GitHub
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
@@ -381,17 +382,17 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON }}
|
||||
cache-key: ${{ needs.common.outputs.cache-key }}
|
||||
- name: test_build_components -e config -c ${{ matrix.file }}
|
||||
- name: Validate config for ${{ matrix.file }}
|
||||
run: |
|
||||
. venv/bin/activate
|
||||
./script/test_build_components -e config -c ${{ matrix.file }}
|
||||
- name: test_build_components -e compile -c ${{ matrix.file }}
|
||||
python3 script/test_build_components.py -e config -c ${{ matrix.file }}
|
||||
- name: Compile config for ${{ matrix.file }}
|
||||
run: |
|
||||
. venv/bin/activate
|
||||
./script/test_build_components -e compile -c ${{ matrix.file }}
|
||||
python3 script/test_build_components.py -e compile -c ${{ matrix.file }}
|
||||
|
||||
test-build-components-splitter:
|
||||
name: Split components for testing into 10 components per group
|
||||
name: Split components for intelligent grouping (40 weighted per batch)
|
||||
runs-on: ubuntu-24.04
|
||||
needs:
|
||||
- common
|
||||
@@ -402,14 +403,26 @@ jobs:
|
||||
steps:
|
||||
- name: Check out code from GitHub
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
- name: Split components into groups of 10
|
||||
- name: Restore Python
|
||||
uses: ./.github/actions/restore-python
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON }}
|
||||
cache-key: ${{ needs.common.outputs.cache-key }}
|
||||
- name: Split components intelligently based on bus configurations
|
||||
id: split
|
||||
run: |
|
||||
components=$(echo '${{ needs.determine-jobs.outputs.changed-components }}' | jq -c '.[]' | shuf | jq -s -c '[_nwise(10) | join(" ")]')
|
||||
echo "components=$components" >> $GITHUB_OUTPUT
|
||||
. venv/bin/activate
|
||||
|
||||
# Use intelligent splitter that groups components with same bus configs
|
||||
components='${{ needs.determine-jobs.outputs.changed-components }}'
|
||||
|
||||
echo "Splitting components intelligently..."
|
||||
output=$(python3 script/split_components_for_ci.py --components "$components" --batch-size 40 --output github)
|
||||
|
||||
echo "$output" >> $GITHUB_OUTPUT
|
||||
|
||||
test-build-components-split:
|
||||
name: Test split components
|
||||
name: Test components batch (${{ matrix.components }})
|
||||
runs-on: ubuntu-24.04
|
||||
needs:
|
||||
- common
|
||||
@@ -418,17 +431,23 @@ jobs:
|
||||
if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) >= 100
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 4
|
||||
max-parallel: 5
|
||||
matrix:
|
||||
components: ${{ fromJson(needs.test-build-components-splitter.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Show disk space
|
||||
run: |
|
||||
echo "Available disk space:"
|
||||
df -h
|
||||
|
||||
- name: List components
|
||||
run: echo ${{ matrix.components }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libsdl2-dev
|
||||
- name: Cache apt packages
|
||||
uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.5.3
|
||||
with:
|
||||
packages: libsdl2-dev
|
||||
version: 1.0
|
||||
|
||||
- name: Check out code from GitHub
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
||||
@@ -437,20 +456,37 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON }}
|
||||
cache-key: ${{ needs.common.outputs.cache-key }}
|
||||
- name: Validate config
|
||||
- name: Validate and compile components with intelligent grouping
|
||||
run: |
|
||||
. venv/bin/activate
|
||||
for component in ${{ matrix.components }}; do
|
||||
./script/test_build_components -e config -c $component
|
||||
done
|
||||
- name: Compile config
|
||||
run: |
|
||||
. venv/bin/activate
|
||||
mkdir build_cache
|
||||
export PLATFORMIO_BUILD_CACHE_DIR=$PWD/build_cache
|
||||
for component in ${{ matrix.components }}; do
|
||||
./script/test_build_components -e compile -c $component
|
||||
done
|
||||
# Use /mnt for build files (70GB available vs ~29GB on /)
|
||||
# Bind mount PlatformIO directory to /mnt (tools, packages, build cache all go there)
|
||||
sudo mkdir -p /mnt/platformio
|
||||
sudo chown $USER:$USER /mnt/platformio
|
||||
mkdir -p ~/.platformio
|
||||
sudo mount --bind /mnt/platformio ~/.platformio
|
||||
|
||||
# Bind mount test build directory to /mnt
|
||||
sudo mkdir -p /mnt/test_build_components_build
|
||||
sudo chown $USER:$USER /mnt/test_build_components_build
|
||||
mkdir -p tests/test_build_components/build
|
||||
sudo mount --bind /mnt/test_build_components_build tests/test_build_components/build
|
||||
|
||||
# Convert space-separated components to comma-separated for Python script
|
||||
components_csv=$(echo "${{ matrix.components }}" | tr ' ' ',')
|
||||
|
||||
echo "Testing components: $components_csv"
|
||||
echo ""
|
||||
|
||||
# Run config validation with grouping
|
||||
python3 script/test_build_components.py -e config -c "$components_csv" -f
|
||||
|
||||
echo ""
|
||||
echo "Config validation passed! Starting compilation..."
|
||||
echo ""
|
||||
|
||||
# Run compilation with grouping
|
||||
python3 script/test_build_components.py -e compile -c "$components_csv" -f
|
||||
|
||||
pre-commit-ci-lite:
|
||||
name: pre-commit.ci lite
|
||||
|
||||
@@ -1002,6 +1002,12 @@ def parse_args(argv):
|
||||
action="append",
|
||||
default=[],
|
||||
)
|
||||
options_parser.add_argument(
|
||||
"--testing-mode",
|
||||
help="Enable testing mode (disables validation checks for grouped component testing)",
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description=f"ESPHome {const.__version__}", parents=[options_parser]
|
||||
@@ -1260,6 +1266,7 @@ def run_esphome(argv):
|
||||
|
||||
args = parse_args(argv)
|
||||
CORE.dashboard = args.dashboard
|
||||
CORE.testing_mode = args.testing_mode
|
||||
|
||||
# Create address cache from command-line arguments
|
||||
CORE.address_cache = AddressCache.from_cli_args(
|
||||
|
||||
@@ -285,6 +285,10 @@ def consume_connection_slots(
|
||||
|
||||
def validate_connection_slots(max_connections: int) -> None:
|
||||
"""Validate that BLE connection slots don't exceed the configured maximum."""
|
||||
# Skip validation in testing mode to allow component grouping
|
||||
if CORE.testing_mode:
|
||||
return
|
||||
|
||||
ble_data = CORE.data.get(KEY_ESP32_BLE, {})
|
||||
used_slots = ble_data.get(KEY_USED_CONNECTION_SLOTS, [])
|
||||
num_used = len(used_slots)
|
||||
|
||||
@@ -347,7 +347,7 @@ def final_validate_device_schema(
|
||||
|
||||
def validate_pin(opt, device):
|
||||
def validator(value):
|
||||
if opt in device:
|
||||
if opt in device and not CORE.testing_mode:
|
||||
raise cv.Invalid(
|
||||
f"The uart {opt} is used both by {name} and {device[opt]}, "
|
||||
f"but can only be used by one. Please create a new uart bus for {name}."
|
||||
|
||||
@@ -529,6 +529,8 @@ class EsphomeCore:
|
||||
self.dashboard = False
|
||||
# True if command is run from vscode api
|
||||
self.vscode = False
|
||||
# True if running in testing mode (disables validation checks for grouped testing)
|
||||
self.testing_mode = False
|
||||
# The name of the node
|
||||
self.name: str | None = None
|
||||
# The friendly name of the node
|
||||
|
||||
@@ -246,12 +246,15 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy
|
||||
"\n to distinguish them"
|
||||
)
|
||||
|
||||
raise cv.Invalid(
|
||||
f"Duplicate {platform} entity with name '{entity_name}' found{device_prefix}. "
|
||||
f"{conflict_msg}. "
|
||||
"Each entity on a device must have a unique name within its platform."
|
||||
f"{sanitized_msg}"
|
||||
)
|
||||
# Skip duplicate entity name validation when testing_mode is enabled
|
||||
# This flag is used for grouped component testing
|
||||
if not CORE.testing_mode:
|
||||
raise cv.Invalid(
|
||||
f"Duplicate {platform} entity with name '{entity_name}' found{device_prefix}. "
|
||||
f"{conflict_msg}. "
|
||||
"Each entity on a device must have a unique name within its platform."
|
||||
f"{sanitized_msg}"
|
||||
)
|
||||
|
||||
# Store metadata about this entity
|
||||
entity_metadata: EntityMetadata = {
|
||||
|
||||
+2
-2
@@ -118,11 +118,11 @@ class PinRegistry(dict):
|
||||
parent_config = fconf.get_config_for_path(parent_path)
|
||||
final_val_fun(pin_config, parent_config)
|
||||
allow_others = pin_config.get(CONF_ALLOW_OTHER_USES, False)
|
||||
if count != 1 and not allow_others:
|
||||
if count != 1 and not allow_others and not CORE.testing_mode:
|
||||
raise cv.Invalid(
|
||||
f"Pin {pin_config[CONF_NUMBER]} is used in multiple places"
|
||||
)
|
||||
if count == 1 and allow_others:
|
||||
if count == 1 and allow_others and not CORE.testing_mode:
|
||||
raise cv.Invalid(
|
||||
f"Pin {pin_config[CONF_NUMBER]} incorrectly sets {CONF_ALLOW_OTHER_USES}: true"
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Any
|
||||
|
||||
from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE
|
||||
from esphome.core import CORE, EsphomeError
|
||||
@@ -42,6 +43,35 @@ def patch_structhash():
|
||||
cli.clean_build_dir = patched_clean_build_dir
|
||||
|
||||
|
||||
def patch_file_downloader():
|
||||
"""Patch PlatformIO's FileDownloader to retry on PackageException errors."""
|
||||
from platformio.package.download import FileDownloader
|
||||
from platformio.package.exception import PackageException
|
||||
|
||||
original_init = FileDownloader.__init__
|
||||
|
||||
def patched_init(self, *args: Any, **kwargs: Any) -> None:
|
||||
max_retries = 3
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return original_init(self, *args, **kwargs)
|
||||
except PackageException as e:
|
||||
if attempt < max_retries - 1:
|
||||
_LOGGER.warning(
|
||||
"Package download failed: %s. Retrying... (attempt %d/%d)",
|
||||
str(e),
|
||||
attempt + 1,
|
||||
max_retries,
|
||||
)
|
||||
else:
|
||||
# Final attempt - re-raise
|
||||
raise
|
||||
return None
|
||||
|
||||
FileDownloader.__init__ = patched_init
|
||||
|
||||
|
||||
IGNORE_LIB_WARNINGS = f"(?:{'|'.join(['Hash', 'Update'])})"
|
||||
FILTER_PLATFORMIO_LINES = [
|
||||
r"Verbose mode can be enabled via `-v, --verbose` option.*",
|
||||
@@ -99,6 +129,7 @@ def run_platformio_cli(*args, **kwargs) -> str | int:
|
||||
import platformio.__main__
|
||||
|
||||
patch_structhash()
|
||||
patch_file_downloader()
|
||||
return run_external_command(platformio.__main__.main, *cmd, **kwargs)
|
||||
|
||||
|
||||
|
||||
Executable
+523
File diff suppressed because it is too large
Load Diff
Executable
+379
@@ -0,0 +1,379 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Merge multiple component test configurations into a single test file.
|
||||
|
||||
This script combines multiple component test files that use the same common bus
|
||||
configurations into a single merged test file. This allows testing multiple
|
||||
compatible components together, reducing CI build time.
|
||||
|
||||
The merger handles:
|
||||
- Component-specific substitutions (prefixing to avoid conflicts)
|
||||
- Multiple instances of component configurations
|
||||
- Shared common bus packages (included only once)
|
||||
- Platform-specific configurations
|
||||
- Uses ESPHome's built-in merge_config for proper YAML merging
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
# Add esphome to path so we can import from it
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from esphome import yaml_util
|
||||
from esphome.config_helpers import merge_config
|
||||
from script.analyze_component_buses import PACKAGE_DEPENDENCIES, get_common_bus_packages
|
||||
|
||||
|
||||
def load_yaml_file(yaml_file: Path) -> dict:
|
||||
"""Load YAML file using ESPHome's YAML loader.
|
||||
|
||||
Args:
|
||||
yaml_file: Path to the YAML file
|
||||
|
||||
Returns:
|
||||
Parsed YAML as dictionary
|
||||
"""
|
||||
if not yaml_file.exists():
|
||||
raise FileNotFoundError(f"YAML file not found: {yaml_file}")
|
||||
|
||||
return yaml_util.load_yaml(yaml_file)
|
||||
|
||||
|
||||
def extract_packages_from_yaml(data: dict) -> dict[str, str]:
|
||||
"""Extract COMMON BUS package includes from parsed YAML.
|
||||
|
||||
Only extracts packages that are from test_build_components/common/,
|
||||
ignoring component-specific packages.
|
||||
|
||||
Args:
|
||||
data: Parsed YAML dictionary
|
||||
|
||||
Returns:
|
||||
Dictionary mapping package name to include path (as string representation)
|
||||
Only includes common bus packages (i2c, spi, uart, etc.)
|
||||
"""
|
||||
if "packages" not in data:
|
||||
return {}
|
||||
|
||||
packages_value = data["packages"]
|
||||
if not isinstance(packages_value, dict):
|
||||
# List format doesn't include common bus packages (those use dict format)
|
||||
return {}
|
||||
|
||||
# Get common bus package names (cached)
|
||||
common_bus_packages = get_common_bus_packages()
|
||||
packages = {}
|
||||
|
||||
# Dictionary format: packages: {name: value}
|
||||
for name, value in packages_value.items():
|
||||
# Only include common bus packages, ignore component-specific ones
|
||||
if name not in common_bus_packages:
|
||||
continue
|
||||
packages[name] = str(value)
|
||||
# Also track package dependencies (e.g., modbus includes uart)
|
||||
if name not in PACKAGE_DEPENDENCIES:
|
||||
continue
|
||||
for dep in PACKAGE_DEPENDENCIES[name]:
|
||||
if dep not in common_bus_packages:
|
||||
continue
|
||||
# Mark as included via dependency
|
||||
packages[f"_dep_{dep}"] = f"(included via {name})"
|
||||
|
||||
return packages
|
||||
|
||||
|
||||
def prefix_substitutions_in_dict(
|
||||
data: Any, prefix: str, exclude: set[str] | None = None
|
||||
) -> Any:
|
||||
"""Recursively prefix all substitution references in a data structure.
|
||||
|
||||
Args:
|
||||
data: YAML data structure (dict, list, or scalar)
|
||||
prefix: Prefix to add to substitution names
|
||||
exclude: Set of substitution names to exclude from prefixing
|
||||
|
||||
Returns:
|
||||
Data structure with prefixed substitution references
|
||||
"""
|
||||
if exclude is None:
|
||||
exclude = set()
|
||||
|
||||
def replace_sub(text: str) -> str:
|
||||
"""Replace substitution references in a string."""
|
||||
|
||||
def replace_match(match):
|
||||
sub_name = match.group(1)
|
||||
if sub_name in exclude:
|
||||
return match.group(0)
|
||||
# Always use braced format in output for consistency
|
||||
return f"${{{prefix}_{sub_name}}}"
|
||||
|
||||
# Match both ${substitution} and $substitution formats
|
||||
return re.sub(r"\$\{?(\w+)\}?", replace_match, text)
|
||||
|
||||
if isinstance(data, dict):
|
||||
result = {}
|
||||
for key, value in data.items():
|
||||
result[key] = prefix_substitutions_in_dict(value, prefix, exclude)
|
||||
return result
|
||||
if isinstance(data, list):
|
||||
return [prefix_substitutions_in_dict(item, prefix, exclude) for item in data]
|
||||
if isinstance(data, str):
|
||||
return replace_sub(data)
|
||||
return data
|
||||
|
||||
|
||||
def deduplicate_by_id(data: dict) -> dict:
|
||||
"""Deduplicate list items with the same ID.
|
||||
|
||||
Keeps only the first occurrence of each ID. If items with the same ID
|
||||
are identical, this silently deduplicates. If they differ, the first
|
||||
one is kept (ESPHome's validation will catch if this causes issues).
|
||||
|
||||
Args:
|
||||
data: Parsed config dictionary
|
||||
|
||||
Returns:
|
||||
Config with deduplicated lists
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
result = {}
|
||||
for key, value in data.items():
|
||||
if isinstance(value, list):
|
||||
# Check for items with 'id' field
|
||||
seen_ids = set()
|
||||
deduped_list = []
|
||||
|
||||
for item in value:
|
||||
if isinstance(item, dict) and "id" in item:
|
||||
item_id = item["id"]
|
||||
if item_id not in seen_ids:
|
||||
seen_ids.add(item_id)
|
||||
deduped_list.append(item)
|
||||
# else: skip duplicate ID (keep first occurrence)
|
||||
else:
|
||||
# No ID, just add it
|
||||
deduped_list.append(item)
|
||||
|
||||
result[key] = deduped_list
|
||||
elif isinstance(value, dict):
|
||||
# Recursively deduplicate nested dicts
|
||||
result[key] = deduplicate_by_id(value)
|
||||
else:
|
||||
result[key] = value
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def merge_component_configs(
|
||||
component_names: list[str],
|
||||
platform: str,
|
||||
tests_dir: Path,
|
||||
output_file: Path,
|
||||
) -> None:
|
||||
"""Merge multiple component test configs into a single file.
|
||||
|
||||
Args:
|
||||
component_names: List of component names to merge
|
||||
platform: Platform to merge for (e.g., "esp32-ard")
|
||||
tests_dir: Path to tests/components directory
|
||||
output_file: Path to output merged config file
|
||||
"""
|
||||
if not component_names:
|
||||
raise ValueError("No components specified")
|
||||
|
||||
# Track packages to ensure they're identical
|
||||
all_packages = None
|
||||
|
||||
# Start with empty config
|
||||
merged_config_data = {}
|
||||
|
||||
# Process each component
|
||||
for comp_name in component_names:
|
||||
comp_dir = tests_dir / comp_name
|
||||
test_file = comp_dir / f"test.{platform}.yaml"
|
||||
|
||||
if not test_file.exists():
|
||||
raise FileNotFoundError(f"Test file not found: {test_file}")
|
||||
|
||||
# Load the component's test file
|
||||
comp_data = load_yaml_file(test_file)
|
||||
|
||||
# Validate packages are compatible
|
||||
# Components with no packages (no_buses) can merge with any group
|
||||
comp_packages = extract_packages_from_yaml(comp_data)
|
||||
|
||||
if all_packages is None:
|
||||
# First component - set the baseline
|
||||
all_packages = comp_packages
|
||||
elif not comp_packages:
|
||||
# This component has no packages (no_buses) - it can merge with any group
|
||||
pass
|
||||
elif not all_packages:
|
||||
# Previous components had no packages, but this one does - adopt these packages
|
||||
all_packages = comp_packages
|
||||
elif comp_packages != all_packages:
|
||||
# Both have packages but they differ - this is an error
|
||||
raise ValueError(
|
||||
f"Component {comp_name} has different packages than previous components. "
|
||||
f"Expected: {all_packages}, Got: {comp_packages}. "
|
||||
f"All components must use the same common bus configs to be merged."
|
||||
)
|
||||
|
||||
# Handle $component_dir by replacing with absolute path
|
||||
# This allows components that use local file references to be grouped
|
||||
comp_abs_dir = str(comp_dir.absolute())
|
||||
|
||||
# Save top-level substitutions BEFORE expanding packages
|
||||
# In ESPHome, top-level substitutions override package substitutions
|
||||
top_level_subs = (
|
||||
comp_data["substitutions"].copy()
|
||||
if "substitutions" in comp_data and comp_data["substitutions"] is not None
|
||||
else {}
|
||||
)
|
||||
|
||||
# Expand packages - but we'll restore substitution priority after
|
||||
if "packages" in comp_data:
|
||||
packages_value = comp_data["packages"]
|
||||
|
||||
if isinstance(packages_value, dict):
|
||||
# Dict format - check each package
|
||||
common_bus_packages = get_common_bus_packages()
|
||||
for pkg_name, pkg_value in list(packages_value.items()):
|
||||
if pkg_name in common_bus_packages:
|
||||
continue
|
||||
if not isinstance(pkg_value, dict):
|
||||
continue
|
||||
# Component-specific package - expand its content into top level
|
||||
comp_data = merge_config(comp_data, pkg_value)
|
||||
elif isinstance(packages_value, list):
|
||||
# List format - expand all package includes
|
||||
for pkg_value in packages_value:
|
||||
if not isinstance(pkg_value, dict):
|
||||
continue
|
||||
comp_data = merge_config(comp_data, pkg_value)
|
||||
|
||||
# Remove all packages (common will be re-added at the end)
|
||||
del comp_data["packages"]
|
||||
|
||||
# Restore top-level substitution priority
|
||||
# Top-level substitutions override any from packages
|
||||
if "substitutions" not in comp_data or comp_data["substitutions"] is None:
|
||||
comp_data["substitutions"] = {}
|
||||
|
||||
# Merge: package subs as base, top-level subs override
|
||||
comp_data["substitutions"].update(top_level_subs)
|
||||
|
||||
# Now prefix the final merged substitutions
|
||||
comp_data["substitutions"] = {
|
||||
f"{comp_name}_{sub_name}": sub_value
|
||||
for sub_name, sub_value in comp_data["substitutions"].items()
|
||||
}
|
||||
|
||||
# Add component_dir substitution with absolute path for this component
|
||||
comp_data["substitutions"][f"{comp_name}_component_dir"] = comp_abs_dir
|
||||
|
||||
# Prefix substitution references throughout the config
|
||||
comp_data = prefix_substitutions_in_dict(comp_data, comp_name)
|
||||
|
||||
# Use ESPHome's merge_config to merge this component into the result
|
||||
# merge_config handles list merging with ID-based deduplication automatically
|
||||
merged_config_data = merge_config(merged_config_data, comp_data)
|
||||
|
||||
# Add packages back (only once, since they're identical)
|
||||
# IMPORTANT: Only re-add common bus packages (spi, i2c, uart, etc.)
|
||||
# Do NOT re-add component-specific packages as they contain unprefixed $component_dir refs
|
||||
if all_packages:
|
||||
first_comp_data = load_yaml_file(
|
||||
tests_dir / component_names[0] / f"test.{platform}.yaml"
|
||||
)
|
||||
if "packages" in first_comp_data and isinstance(
|
||||
first_comp_data["packages"], dict
|
||||
):
|
||||
# Filter to only include common bus packages
|
||||
# Only dict format can contain common bus packages
|
||||
common_bus_packages = get_common_bus_packages()
|
||||
filtered_packages = {
|
||||
name: value
|
||||
for name, value in first_comp_data["packages"].items()
|
||||
if name in common_bus_packages
|
||||
}
|
||||
if filtered_packages:
|
||||
merged_config_data["packages"] = filtered_packages
|
||||
|
||||
# Deduplicate items with same ID (keeps first occurrence)
|
||||
merged_config_data = deduplicate_by_id(merged_config_data)
|
||||
|
||||
# Remove esphome section since it will be provided by the wrapper file
|
||||
# The wrapper file includes this merged config via packages and provides
|
||||
# the proper esphome: section with name, platform, etc.
|
||||
if "esphome" in merged_config_data:
|
||||
del merged_config_data["esphome"]
|
||||
|
||||
# Write merged config
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
yaml_content = yaml_util.dump(merged_config_data)
|
||||
output_file.write_text(yaml_content)
|
||||
|
||||
print(f"Successfully merged {len(component_names)} components into {output_file}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Merge multiple component test configs into a single file"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--components",
|
||||
"-c",
|
||||
required=True,
|
||||
help="Comma-separated list of component names to merge",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
"-p",
|
||||
required=True,
|
||||
help="Platform to merge for (e.g., esp32-ard)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="Output file path for merged config",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tests-dir",
|
||||
type=Path,
|
||||
default=Path("tests/components"),
|
||||
help="Path to tests/components directory",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
component_names = [c.strip() for c in args.components.split(",")]
|
||||
|
||||
try:
|
||||
merge_component_configs(
|
||||
component_names=component_names,
|
||||
platform=args.platform,
|
||||
tests_dir=args.tests_dir,
|
||||
output_file=args.output,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error merging configs: {e}", file=sys.stderr)
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+268
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split components into batches with intelligent grouping.
|
||||
|
||||
This script analyzes components to identify which ones share common bus configurations
|
||||
and intelligently groups them into batches to maximize the efficiency of the
|
||||
component grouping system in CI.
|
||||
|
||||
Components with the same bus signature are placed in the same batch whenever possible,
|
||||
allowing the test_build_components.py script to merge them into single builds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# Add esphome to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from script.analyze_component_buses import (
|
||||
ISOLATED_COMPONENTS,
|
||||
NO_BUSES_SIGNATURE,
|
||||
analyze_all_components,
|
||||
create_grouping_signature,
|
||||
)
|
||||
|
||||
# Weighting for batch creation
|
||||
# Isolated components can't be grouped/merged, so they count as 10x
|
||||
# Groupable components can be merged into single builds, so they count as 1x
|
||||
ISOLATED_WEIGHT = 10
|
||||
GROUPABLE_WEIGHT = 1
|
||||
|
||||
|
||||
def has_test_files(component_name: str, tests_dir: Path) -> bool:
|
||||
"""Check if a component has test files.
|
||||
|
||||
Args:
|
||||
component_name: Name of the component
|
||||
tests_dir: Path to tests/components directory
|
||||
|
||||
Returns:
|
||||
True if the component has test.*.yaml files
|
||||
"""
|
||||
component_dir = tests_dir / component_name
|
||||
if not component_dir.exists() or not component_dir.is_dir():
|
||||
return False
|
||||
|
||||
# Check for test.*.yaml files
|
||||
return any(component_dir.glob("test.*.yaml"))
|
||||
|
||||
|
||||
def create_intelligent_batches(
|
||||
components: list[str],
|
||||
tests_dir: Path,
|
||||
batch_size: int = 40,
|
||||
) -> list[list[str]]:
|
||||
"""Create batches optimized for component grouping.
|
||||
|
||||
Args:
|
||||
components: List of component names to batch
|
||||
tests_dir: Path to tests/components directory
|
||||
batch_size: Target size for each batch
|
||||
|
||||
Returns:
|
||||
List of component batches (lists of component names)
|
||||
"""
|
||||
# Filter out components without test files
|
||||
# Platform components like 'climate' and 'climate_ir' don't have test files
|
||||
components_with_tests = [
|
||||
comp for comp in components if has_test_files(comp, tests_dir)
|
||||
]
|
||||
|
||||
# Log filtered components to stderr for debugging
|
||||
if len(components_with_tests) < len(components):
|
||||
filtered_out = set(components) - set(components_with_tests)
|
||||
print(
|
||||
f"Note: Filtered {len(filtered_out)} components without test files: "
|
||||
f"{', '.join(sorted(filtered_out))}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Analyze all components to get their bus signatures
|
||||
component_buses, non_groupable, _direct_bus_components = analyze_all_components(
|
||||
tests_dir
|
||||
)
|
||||
|
||||
# Group components by their bus signature ONLY (ignore platform)
|
||||
# All platforms will be tested by test_build_components.py for each batch
|
||||
# Key: signature, Value: list of components
|
||||
signature_groups: dict[str, list[str]] = defaultdict(list)
|
||||
|
||||
for component in components_with_tests:
|
||||
# Components that can't be grouped get unique signatures
|
||||
# This includes both manually curated ISOLATED_COMPONENTS and
|
||||
# automatically detected non_groupable components
|
||||
# These can share a batch/runner but won't be grouped/merged
|
||||
if component in ISOLATED_COMPONENTS or component in non_groupable:
|
||||
signature_groups[f"isolated_{component}"].append(component)
|
||||
continue
|
||||
|
||||
# Get signature from any platform (they should all have the same buses)
|
||||
# Components not in component_buses were filtered out by has_test_files check
|
||||
comp_platforms = component_buses[component]
|
||||
for platform, buses in comp_platforms.items():
|
||||
if buses:
|
||||
signature = create_grouping_signature({platform: buses}, platform)
|
||||
# Group by signature only - platform doesn't matter for batching
|
||||
signature_groups[signature].append(component)
|
||||
break # Only use first platform for grouping
|
||||
else:
|
||||
# No buses found for any platform - can be grouped together
|
||||
signature_groups[NO_BUSES_SIGNATURE].append(component)
|
||||
|
||||
# Create batches by keeping signature groups together
|
||||
# Components with the same signature stay in the same batches
|
||||
batches = []
|
||||
|
||||
# Sort signature groups to prioritize groupable components
|
||||
# 1. Put "isolated_*" signatures last (can't be grouped with others)
|
||||
# 2. Sort groupable signatures by size (largest first)
|
||||
# 3. "no_buses" components CAN be grouped together
|
||||
def sort_key(item):
|
||||
signature, components = item
|
||||
is_isolated = signature.startswith("isolated_")
|
||||
# Put "isolated_*" last (1), groupable first (0)
|
||||
# Within each category, sort by size (largest first)
|
||||
return (is_isolated, -len(components))
|
||||
|
||||
sorted_groups = sorted(signature_groups.items(), key=sort_key)
|
||||
|
||||
# Strategy: Create batches using weighted sizes
|
||||
# - Isolated components count as 10x (since they can't be grouped/merged)
|
||||
# - Groupable components count as 1x (can be merged into single builds)
|
||||
# - This distributes isolated components across more runners
|
||||
# - Ensures each runner has a good mix of groupable vs isolated components
|
||||
|
||||
current_batch = []
|
||||
current_weight = 0
|
||||
|
||||
for signature, group_components in sorted_groups:
|
||||
is_isolated = signature.startswith("isolated_")
|
||||
weight_per_component = ISOLATED_WEIGHT if is_isolated else GROUPABLE_WEIGHT
|
||||
|
||||
for component in group_components:
|
||||
# Check if adding this component would exceed the batch size
|
||||
if current_weight + weight_per_component > batch_size and current_batch:
|
||||
# Start a new batch
|
||||
batches.append(current_batch)
|
||||
current_batch = []
|
||||
current_weight = 0
|
||||
|
||||
# Add component to current batch
|
||||
current_batch.append(component)
|
||||
current_weight += weight_per_component
|
||||
|
||||
# Don't forget the last batch
|
||||
if current_batch:
|
||||
batches.append(current_batch)
|
||||
|
||||
return batches
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Split components into intelligent batches for CI testing"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--components",
|
||||
"-c",
|
||||
required=True,
|
||||
help="JSON array of component names",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
"-b",
|
||||
type=int,
|
||||
default=40,
|
||||
help="Target batch size (default: 40, weighted)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tests-dir",
|
||||
type=Path,
|
||||
default=Path("tests/components"),
|
||||
help="Path to tests/components directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
choices=["json", "github"],
|
||||
default="github",
|
||||
help="Output format (json or github for GitHub Actions)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Parse component list from JSON
|
||||
try:
|
||||
components = json.loads(args.components)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Error parsing components JSON: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not isinstance(components, list):
|
||||
print("Components must be a JSON array", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Create intelligent batches
|
||||
batches = create_intelligent_batches(
|
||||
components=components,
|
||||
tests_dir=args.tests_dir,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
|
||||
# Convert batches to space-separated strings for CI
|
||||
batch_strings = [" ".join(batch) for batch in batches]
|
||||
|
||||
if args.output == "json":
|
||||
# Output as JSON array
|
||||
print(json.dumps(batch_strings))
|
||||
else:
|
||||
# Output for GitHub Actions (set output)
|
||||
output_json = json.dumps(batch_strings)
|
||||
print(f"components={output_json}")
|
||||
|
||||
# Print summary to stderr so it shows in CI logs
|
||||
# Count actual components being batched
|
||||
actual_components = sum(len(batch.split()) for batch in batch_strings)
|
||||
|
||||
# Re-analyze to get isolated component counts for summary
|
||||
_, non_groupable, _ = analyze_all_components(args.tests_dir)
|
||||
|
||||
# Count isolated vs groupable components
|
||||
all_batched_components = [comp for batch in batches for comp in batch]
|
||||
isolated_count = sum(
|
||||
1
|
||||
for comp in all_batched_components
|
||||
if comp in ISOLATED_COMPONENTS or comp in non_groupable
|
||||
)
|
||||
groupable_count = actual_components - isolated_count
|
||||
|
||||
print("\n=== Intelligent Batch Summary ===", file=sys.stderr)
|
||||
print(f"Total components requested: {len(components)}", file=sys.stderr)
|
||||
print(f"Components with test files: {actual_components}", file=sys.stderr)
|
||||
print(f" - Groupable (weight=1): {groupable_count}", file=sys.stderr)
|
||||
print(f" - Isolated (weight=10): {isolated_count}", file=sys.stderr)
|
||||
if actual_components < len(components):
|
||||
print(
|
||||
f"Components skipped (no test files): {len(components) - actual_components}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(f"Number of batches: {len(batches)}", file=sys.stderr)
|
||||
print(f"Batch size target (weighted): {args.batch_size}", file=sys.stderr)
|
||||
if len(batches) > 0:
|
||||
print(
|
||||
f"Average components per batch: {actual_components / len(batches):.1f}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(file=sys.stderr)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,106 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
help() {
|
||||
echo "Usage: $0 [-e <config|compile|clean>] [-c <string>] [-t <string>]" 1>&2
|
||||
echo 1>&2
|
||||
echo " - e - Parameter for esphome command. Default compile. Common alternative is config." 1>&2
|
||||
echo " - c - Component folder name to test. Default *. E.g. '-c logger'." 1>&2
|
||||
echo " - t - Target name to test. Put '-t list' to display all possibilities. E.g. '-t esp32-s2-idf-51'." 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Parse parameter:
|
||||
# - `e` - Parameter for `esphome` command. Default `compile`. Common alternative is `config`.
|
||||
# - `c` - Component folder name to test. Default `*`.
|
||||
esphome_command="compile"
|
||||
target_component="*"
|
||||
while getopts e:c:t: flag
|
||||
do
|
||||
case $flag in
|
||||
e) esphome_command=${OPTARG};;
|
||||
c) target_component=${OPTARG};;
|
||||
t) requested_target_platform=${OPTARG};;
|
||||
\?) help;;
|
||||
esac
|
||||
done
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if ! [ -d "./tests/test_build_components/build" ]; then
|
||||
mkdir ./tests/test_build_components/build
|
||||
fi
|
||||
|
||||
start_esphome() {
|
||||
if [ -n "$requested_target_platform" ] && [ "$requested_target_platform" != "$target_platform_with_version" ]; then
|
||||
echo "Skipping $target_platform_with_version"
|
||||
return
|
||||
fi
|
||||
# create dynamic yaml file in `build` folder.
|
||||
# `./tests/test_build_components/build/[target_component].[test_name].[target_platform_with_version].yaml`
|
||||
component_test_file="./tests/test_build_components/build/$target_component.$test_name.$target_platform_with_version.yaml"
|
||||
|
||||
cp $target_platform_file $component_test_file
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
# macOS sed is...different
|
||||
sed -i '' "s!\$component_test_file!../../.$f!g" $component_test_file
|
||||
else
|
||||
sed -i "s!\$component_test_file!../../.$f!g" $component_test_file
|
||||
fi
|
||||
|
||||
# Start esphome process
|
||||
echo "> [$target_component] [$test_name] [$target_platform_with_version]"
|
||||
set -x
|
||||
# TODO: Validate escape of Command line substitution value
|
||||
python3 -m esphome -s component_name $target_component -s component_dir ../../components/$target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file
|
||||
{ set +x; } 2>/dev/null
|
||||
}
|
||||
|
||||
# Find all test yaml files.
|
||||
# - `./tests/components/[target_component]/[test_name].[target_platform].yaml`
|
||||
# - `./tests/components/[target_component]/[test_name].all.yaml`
|
||||
for f in ./tests/components/$target_component/*.*.yaml; do
|
||||
[ -f "$f" ] || continue
|
||||
IFS='/' read -r -a folder_name <<< "$f"
|
||||
target_component="${folder_name[3]}"
|
||||
|
||||
IFS='.' read -r -a file_name <<< "${folder_name[4]}"
|
||||
test_name="${file_name[0]}"
|
||||
target_platform="${file_name[1]}"
|
||||
file_name_parts=${#file_name[@]}
|
||||
|
||||
if [ "$target_platform" = "all" ] || [ $file_name_parts = 2 ]; then
|
||||
# Test has *not* defined a specific target platform. Need to run tests for all possible target platforms.
|
||||
|
||||
for target_platform_file in ./tests/test_build_components/build_components_base.*.yaml; do
|
||||
IFS='/' read -r -a folder_name <<< "$target_platform_file"
|
||||
IFS='.' read -r -a file_name <<< "${folder_name[3]}"
|
||||
target_platform="${file_name[1]}"
|
||||
|
||||
start_esphome
|
||||
done
|
||||
|
||||
else
|
||||
# Test has defined a specific target platform.
|
||||
|
||||
# Validate we have a base test yaml for selected platform.
|
||||
# The target_platform is sourced from the following location.
|
||||
# 1. `./tests/test_build_components/build_components_base.[target_platform].yaml`
|
||||
# 2. `./tests/test_build_components/build_components_base.[target_platform]-ard.yaml`
|
||||
target_platform_file="./tests/test_build_components/build_components_base.$target_platform.yaml"
|
||||
if ! [ -f "$target_platform_file" ]; then
|
||||
echo "No base test file [./tests/test_build_components/build_components_base.$target_platform.yaml] for component test [$f] found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for target_platform_file in ./tests/test_build_components/build_components_base.$target_platform*.yaml; do
|
||||
# trim off "./tests/test_build_components/build_components_base." prefix
|
||||
target_platform_with_version=${target_platform_file:52}
|
||||
# ...now remove suffix starting with "." leaving just the test target hardware and software platform (possibly with version)
|
||||
# For example: "esp32-s3-idf-50"
|
||||
target_platform_with_version=${target_platform_with_version%.*}
|
||||
start_esphome
|
||||
done
|
||||
fi
|
||||
done
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
test_build_components.py
|
||||
Executable
+931
File diff suppressed because it is too large
Load Diff
Executable
+227
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test component grouping by finding and testing groups of components.
|
||||
|
||||
This script analyzes components, finds groups that can be tested together,
|
||||
and runs test builds for those groups.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Add esphome to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from script.analyze_component_buses import (
|
||||
analyze_all_components,
|
||||
group_components_by_signature,
|
||||
)
|
||||
|
||||
|
||||
def test_component_group(
|
||||
components: list[str],
|
||||
platform: str,
|
||||
esphome_command: str = "compile",
|
||||
dry_run: bool = False,
|
||||
) -> bool:
|
||||
"""Test a group of components together.
|
||||
|
||||
Args:
|
||||
components: List of component names to test together
|
||||
platform: Platform to test on (e.g., "esp32-idf")
|
||||
esphome_command: ESPHome command to run (config/compile/clean)
|
||||
dry_run: If True, only print the command without running it
|
||||
|
||||
Returns:
|
||||
True if test passed, False otherwise
|
||||
"""
|
||||
components_str = ",".join(components)
|
||||
cmd = [
|
||||
"./script/test_build_components",
|
||||
"-c",
|
||||
components_str,
|
||||
"-t",
|
||||
platform,
|
||||
"-e",
|
||||
esphome_command,
|
||||
]
|
||||
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"Testing {len(components)} components on {platform}:")
|
||||
for comp in components:
|
||||
print(f" - {comp}")
|
||||
print(f"{'=' * 80}")
|
||||
print(f"Command: {' '.join(cmd)}\n")
|
||||
|
||||
if dry_run:
|
||||
print("[DRY RUN] Skipping actual test")
|
||||
return True
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, check=False)
|
||||
return result.returncode == 0
|
||||
except Exception as e:
|
||||
print(f"Error running test: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Test component grouping by finding and testing groups"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
"-p",
|
||||
default="esp32-idf",
|
||||
help="Platform to test (default: esp32-idf)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e",
|
||||
"--esphome-command",
|
||||
default="compile",
|
||||
choices=["config", "compile", "clean"],
|
||||
help="ESPHome command to run (default: compile)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="Test all components (sets --min-size=1, --max-size=10000, --max-groups=10000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-size",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Minimum group size to test (default: 3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-size",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Maximum group size to test (default: 10)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-groups",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Maximum number of groups to test (default: 5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--signature",
|
||||
"-s",
|
||||
help="Only test groups with this bus signature (e.g., 'spi', 'i2c', 'uart')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print commands without running them",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# If --all is specified, test all components without grouping
|
||||
if args.all:
|
||||
# Get all components from tests/components directory
|
||||
components_dir = Path("tests/components")
|
||||
all_components = sorted(
|
||||
[d.name for d in components_dir.iterdir() if d.is_dir()]
|
||||
)
|
||||
|
||||
if not all_components:
|
||||
print(f"\nNo components found in {components_dir}")
|
||||
return
|
||||
|
||||
print(f"\nTesting all {len(all_components)} components together")
|
||||
|
||||
success = test_component_group(
|
||||
all_components, args.platform, args.esphome_command, args.dry_run
|
||||
)
|
||||
|
||||
# Print summary
|
||||
print(f"\n{'=' * 80}")
|
||||
print("TEST SUMMARY")
|
||||
print(f"{'=' * 80}")
|
||||
status = "✅ PASS" if success else "❌ FAIL"
|
||||
print(f"{status} All components: {len(all_components)} components")
|
||||
|
||||
if not args.dry_run and not success:
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
print("Analyzing all components...")
|
||||
components, non_groupable, _ = analyze_all_components(Path("tests/components"))
|
||||
|
||||
print(f"Found {len(components)} components, {len(non_groupable)} non-groupable")
|
||||
|
||||
# Group components by signature for the platform
|
||||
groups = group_components_by_signature(components, args.platform)
|
||||
|
||||
# Filter and sort groups
|
||||
filtered_groups = []
|
||||
for signature, comp_list in groups.items():
|
||||
# Filter by signature if specified
|
||||
if args.signature and signature != args.signature:
|
||||
continue
|
||||
|
||||
# Remove non-groupable components
|
||||
comp_list = [c for c in comp_list if c not in non_groupable]
|
||||
|
||||
# Filter by minimum size
|
||||
if len(comp_list) < args.min_size:
|
||||
continue
|
||||
|
||||
# If group is larger than max_size, we'll take a subset later
|
||||
filtered_groups.append((signature, comp_list))
|
||||
|
||||
# Sort by group size (largest first)
|
||||
filtered_groups.sort(key=lambda x: len(x[1]), reverse=True)
|
||||
|
||||
# Limit number of groups
|
||||
filtered_groups = filtered_groups[: args.max_groups]
|
||||
|
||||
if not filtered_groups:
|
||||
print("\nNo groups found matching criteria:")
|
||||
print(f" - Platform: {args.platform}")
|
||||
print(f" - Size: {args.min_size}-{args.max_size}")
|
||||
if args.signature:
|
||||
print(f" - Signature: {args.signature}")
|
||||
return
|
||||
|
||||
print(f"\nFound {len(filtered_groups)} groups to test:")
|
||||
for signature, comp_list in filtered_groups:
|
||||
print(f" [{signature}]: {len(comp_list)} components")
|
||||
|
||||
# Test each group
|
||||
results = []
|
||||
for signature, comp_list in filtered_groups:
|
||||
# Limit to max_size if group is larger
|
||||
if len(comp_list) > args.max_size:
|
||||
comp_list = comp_list[: args.max_size]
|
||||
|
||||
success = test_component_group(
|
||||
comp_list, args.platform, args.esphome_command, args.dry_run
|
||||
)
|
||||
results.append((signature, comp_list, success))
|
||||
|
||||
if not args.dry_run and not success:
|
||||
print(f"\n❌ FAILED: {signature} group")
|
||||
break
|
||||
|
||||
# Print summary
|
||||
print(f"\n{'=' * 80}")
|
||||
print("TEST SUMMARY")
|
||||
print(f"{'=' * 80}")
|
||||
for signature, comp_list, success in results:
|
||||
status = "✅ PASS" if success else "❌ FAIL"
|
||||
print(f"{status} [{signature}]: {len(comp_list)} components")
|
||||
|
||||
# Exit with error if any tests failed
|
||||
if not args.dry_run and any(not success for _, _, success in results):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,11 +1,4 @@
|
||||
uart:
|
||||
- id: uart_a01nyub
|
||||
tx_pin: ${tx_pin}
|
||||
rx_pin: ${rx_pin}
|
||||
baud_rate: 9600
|
||||
|
||||
sensor:
|
||||
- platform: a01nyub
|
||||
id: a01nyub_sensor
|
||||
name: a01nyub Distance
|
||||
uart_id: uart_a01nyub
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/esp32-c3-idf.yaml
|
||||
|
||||
substitutions:
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
substitutions:
|
||||
tx_pin: GPIO17
|
||||
rx_pin: GPIO16
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/esp32-idf.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
substitutions:
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
substitutions:
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
uart:
|
||||
- id: uart_a02yyuw
|
||||
tx_pin: ${tx_pin}
|
||||
rx_pin: ${rx_pin}
|
||||
baud_rate: 9600
|
||||
|
||||
sensor:
|
||||
- platform: a02yyuw
|
||||
id: a02yyuw_sensor
|
||||
name: a02yyuw Distance
|
||||
uart_id: uart_a02yyuw
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/esp32-c3-idf.yaml
|
||||
|
||||
substitutions:
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
substitutions:
|
||||
tx_pin: GPIO17
|
||||
rx_pin: GPIO16
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/esp32-idf.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
substitutions:
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
substitutions:
|
||||
tx_pin: GPIO4
|
||||
rx_pin: GPIO5
|
||||
packages:
|
||||
uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
substitutions:
|
||||
step_pin: GPIO22
|
||||
dir_pin: GPIO23
|
||||
dir_pin: GPIO4
|
||||
sleep_pin: GPIO25
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
substitutions:
|
||||
step_pin: GPIO1
|
||||
dir_pin: GPIO2
|
||||
sleep_pin: GPIO5
|
||||
sleep_pin: GPIO0
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
substitutions:
|
||||
gate_pin: GPIO18
|
||||
zero_cross_pin: GPIO19
|
||||
gate_pin: GPIO4
|
||||
zero_cross_pin: GPIO5
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
substitutions:
|
||||
gate_pin: GPIO5
|
||||
zero_cross_pin: GPIO4
|
||||
gate_pin: GPIO0
|
||||
zero_cross_pin: GPIO2
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
sensor:
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
attenuation: 2.5db
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
@@ -1,7 +1,11 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: P23
|
||||
attenuation: !remove
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
pin: 4
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: GPIO1
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
attenuation: 2.5db
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: A0
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
attenuation: 2.5db
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
pin: GPIO50
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: GPIO16
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
attenuation: 2.5db
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
pin: 1
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: GPIO1
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
attenuation: 2.5db
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
pin: 1
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: GPIO1
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
attenuation: 2.5db
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: VCC
|
||||
attenuation: !remove
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
pin: PA0
|
||||
attenuation: !remove
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: A5
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
packages:
|
||||
base: !include common.yaml
|
||||
|
||||
sensor:
|
||||
- id: !extend my_sensor
|
||||
- id: my_sensor
|
||||
platform: adc
|
||||
pin: VCC
|
||||
attenuation: !remove
|
||||
name: ADC Test sensor
|
||||
update_interval: "1:01"
|
||||
unit_of_measurement: "°C"
|
||||
icon: "mdi:water-percent"
|
||||
accuracy_decimals: 5
|
||||
setup_priority: -100
|
||||
force_update: true
|
||||
|
||||
@@ -1,9 +1,3 @@
|
||||
spi:
|
||||
- id: spi_adc128s102
|
||||
clk_pin: ${clk_pin}
|
||||
mosi_pin: ${mosi_pin}
|
||||
miso_pin: ${miso_pin}
|
||||
|
||||
adc128s102:
|
||||
cs_pin: ${cs_pin}
|
||||
id: adc128s102_adc
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
substitutions:
|
||||
clk_pin: GPIO6
|
||||
mosi_pin: GPIO7
|
||||
miso_pin: GPIO5
|
||||
cs_pin: GPIO2
|
||||
|
||||
packages:
|
||||
spi: !include ../../test_build_components/common/spi/esp32-c3-idf.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
substitutions:
|
||||
clk_pin: GPIO16
|
||||
mosi_pin: GPIO17
|
||||
miso_pin: GPIO15
|
||||
cs_pin: GPIO12
|
||||
|
||||
packages:
|
||||
spi: !include ../../test_build_components/common/spi/esp32-idf.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
substitutions:
|
||||
clk_pin: GPIO14
|
||||
mosi_pin: GPIO13
|
||||
miso_pin: GPIO12
|
||||
clk_pin: GPIO0
|
||||
mosi_pin: GPIO2
|
||||
miso_pin: GPIO16
|
||||
cs_pin: GPIO15
|
||||
|
||||
packages:
|
||||
spi: !include ../../test_build_components/common/spi/esp8266-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -4,4 +4,7 @@ substitutions:
|
||||
miso_pin: GPIO4
|
||||
cs_pin: GPIO5
|
||||
|
||||
packages:
|
||||
spi: !include ../../test_build_components/common/spi/rp2040-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
i2c:
|
||||
- id: i2c_ade7880
|
||||
scl: ${scl_pin}
|
||||
sda: ${sda_pin}
|
||||
|
||||
sensor:
|
||||
- platform: ade7880
|
||||
i2c_id: i2c_ade7880
|
||||
i2c_id: i2c_bus
|
||||
irq0_pin: ${irq0_pin}
|
||||
irq1_pin: ${irq1_pin}
|
||||
reset_pin: ${reset_pin}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
substitutions:
|
||||
scl_pin: GPIO5
|
||||
sda_pin: GPIO4
|
||||
irq0_pin: GPIO6
|
||||
irq1_pin: GPIO7
|
||||
reset_pin: GPIO10
|
||||
reset_pin: GPIO9
|
||||
|
||||
packages:
|
||||
i2c: !include ../../test_build_components/common/i2c/esp32-c3-idf.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
substitutions:
|
||||
scl_pin: GPIO5
|
||||
sda_pin: GPIO4
|
||||
irq0_pin: GPIO13
|
||||
irq1_pin: GPIO15
|
||||
reset_pin: GPIO16
|
||||
reset_pin: GPIO12
|
||||
|
||||
packages:
|
||||
i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
substitutions:
|
||||
scl_pin: GPIO5
|
||||
sda_pin: GPIO4
|
||||
irq0_pin: GPIO13
|
||||
irq1_pin: GPIO15
|
||||
reset_pin: GPIO16
|
||||
|
||||
packages:
|
||||
i2c: !include ../../test_build_components/common/i2c/esp8266-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
substitutions:
|
||||
scl_pin: GPIO5
|
||||
sda_pin: GPIO4
|
||||
irq0_pin: GPIO13
|
||||
irq1_pin: GPIO15
|
||||
reset_pin: GPIO16
|
||||
|
||||
packages:
|
||||
i2c: !include ../../test_build_components/common/i2c/rp2040-ard.yaml
|
||||
|
||||
<<: !include common.yaml
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user