diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57053c36457..55ee136daa4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,8 +199,7 @@ jobs: - common outputs: integration-tests: ${{ steps.determine.outputs.integration-tests }} - integration-tests-run-all: ${{ steps.determine.outputs.integration-tests-run-all }} - integration-test-files: ${{ steps.determine.outputs.integration-test-files }} + integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} python-linters: ${{ steps.determine.outputs.python-linters }} @@ -243,8 +242,7 @@ jobs: # Extract individual fields echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT - echo "integration-tests-run-all=$(echo "$output" | jq -r '.integration_tests_run_all')" >> $GITHUB_OUTPUT - echo "integration-test-files=$(echo "$output" | jq -c '.integration_test_files')" >> $GITHUB_OUTPUT + echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT @@ -267,12 +265,16 @@ jobs: key: components-graph-${{ hashFiles('esphome/components/**/*.py') }} integration-tests: - name: Run integration tests + name: Run integration tests (${{ matrix.bucket.name }}) runs-on: ubuntu-latest needs: - common - determine-jobs if: needs.determine-jobs.outputs.integration-tests == 'true' + strategy: + fail-fast: false + matrix: + bucket: ${{ fromJson(needs.determine-jobs.outputs.integration-test-buckets) }} steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -298,20 +300,9 @@ jobs: - name: Register matcher run: echo "::add-matcher::.github/workflows/matchers/pytest.json" - name: Run integration tests - env: - INTEGRATION_TEST_FILES: ${{ needs.determine-jobs.outputs.integration-test-files }} - INTEGRATION_TESTS_RUN_ALL: ${{ needs.determine-jobs.outputs.integration-tests-run-all }} run: | . venv/bin/activate - if [[ "$INTEGRATION_TESTS_RUN_ALL" == "true" ]]; then - echo "Running all integration tests" - pytest -vv --no-cov --tb=native -n auto tests/integration/ - else - # Parse JSON array into bash array to avoid shell expansion issues - mapfile -t test_files < <(echo "$INTEGRATION_TEST_FILES" | jq -r '.[]') - echo "Running ${#test_files[@]} specific integration tests" - pytest -vv --no-cov --tb=native -n auto "${test_files[@]}" - fi + pytest -vv --no-cov --tb=native -n auto ${{ matrix.bucket.tests }} cpp-unit-tests: name: Run C++ unit tests diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 6fd7ab297c8..e7ad061ae12 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -6,8 +6,7 @@ what files have changed. It outputs JSON with the following structure: { "integration_tests": true/false, - "integration_tests_run_all": true/false, - "integration_test_files": ["tests/integration/test_foo.py", ...], + "integration_test_buckets": [{"name": "1/3", "tests": "tests/integration/test_foo.py ..."}, ...], "clang_tidy": true/false, "clang_format": true/false, "python_linters": true/false, @@ -81,6 +80,25 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65 # Isolated components count as 10x, groupable components count as 1x COMPONENT_TEST_BATCH_SIZE = 40 +# Integration test bucketing: when more than the threshold tests are scheduled, +# fan out across this many parallel jobs. Below the threshold, a single job runs. +INTEGRATION_TESTS_SPLIT_THRESHOLD = 10 +INTEGRATION_TESTS_SPLIT_BUCKETS = 3 + + +def _split_list(items: list[str], n: int) -> list[list[str]]: + """Split a list into n roughly-equal contiguous parts (matches script/clang-tidy).""" + k, m = divmod(len(items), n) + return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] + + +def _all_integration_test_files() -> list[str]: + """Return all integration test file paths, sorted, relative to repo root.""" + return sorted( + str(p.relative_to(root_path)) + for p in (Path(root_path) / "tests" / "integration").glob("test_*.py") + ) + class Platform(StrEnum): """Platform identifiers for memory impact analysis.""" @@ -813,6 +831,36 @@ def main() -> None: args.branch ) run_integration = integration_run_all or bool(integration_test_files) + + # When run_all is set, expand to the full glob here so determine-jobs.py + # remains the single source of truth for which tests run. The workflow + # never re-globs the filesystem. + if integration_run_all: + integration_test_files = _all_integration_test_files() + else: + integration_test_files = sorted(integration_test_files) + + # Pre-bucket the test list so the CI matrix can consume it directly. + # Below threshold => 1 bucket; above threshold => INTEGRATION_TESTS_SPLIT_BUCKETS. + integration_test_buckets: list[dict[str, str]] + if not run_integration: + integration_test_buckets = [] + elif len(integration_test_files) > INTEGRATION_TESTS_SPLIT_THRESHOLD: + parts = [ + part + for part in _split_list( + integration_test_files, INTEGRATION_TESTS_SPLIT_BUCKETS + ) + if part + ] + integration_test_buckets = [ + {"name": f"{i + 1}/{len(parts)}", "tests": " ".join(part)} + for i, part in enumerate(parts) + ] + else: + integration_test_buckets = [ + {"name": "1/1", "tests": " ".join(integration_test_files)} + ] run_clang_tidy = should_run_clang_tidy(args.branch) run_clang_format = should_run_clang_format(args.branch) run_python_linters = should_run_python_linters(args.branch) @@ -944,8 +992,7 @@ def main() -> None: output: dict[str, Any] = { "integration_tests": run_integration, - "integration_tests_run_all": integration_run_all, - "integration_test_files": integration_test_files, + "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, "clang_format": run_clang_format, diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 44c110b6892..af37cc61821 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -122,10 +122,19 @@ def test_main_all_tests_should_run( "esphome/helpers.py", ] + # Stable, deterministic stand-in for the tests/integration/ glob so the + # bucket assertions don't drift with the real test count. + fake_test_files = [f"tests/integration/test_{i:03d}.py" for i in range(15)] + # Run main function with mocked argv with ( patch("sys.argv", ["determine-jobs.py"]), patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), + patch.object( + determine_jobs, + "_all_integration_test_files", + return_value=fake_test_files, + ), patch.object( determine_jobs, "get_changed_components", @@ -161,8 +170,21 @@ def test_main_all_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is True - assert output["integration_tests_run_all"] is True - assert output["integration_test_files"] == [] + # run_all=True expands to the full glob and pre-buckets into 3 parts + assert isinstance(output["integration_test_buckets"], list) + assert len(output["integration_test_buckets"]) == 3 + assert [b["name"] for b in output["integration_test_buckets"]] == [ + "1/3", + "2/3", + "3/3", + ] + bucket_files = [ + f + for b in output["integration_test_buckets"] + for f in b["tests"].split(" ") + if f + ] + assert bucket_files == fake_test_files assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True @@ -247,8 +269,7 @@ def test_main_no_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is False - assert output["integration_tests_run_all"] is False - assert output["integration_test_files"] == [] + assert output["integration_test_buckets"] == [] assert output["clang_tidy"] is False assert output["clang_tidy_mode"] == "disabled" assert output["clang_format"] is False @@ -332,8 +353,7 @@ def test_main_with_branch_argument( output = json.loads(captured.out) assert output["integration_tests"] is False - assert output["integration_tests_run_all"] is False - assert output["integration_test_files"] == [] + assert output["integration_test_buckets"] == [] assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is False