diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06c6c0fec17..819dac926e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,7 +423,10 @@ jobs: - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@3194d9a39c4d46684cb44bf7207fc56626aad8fd # v4.15.1 with: - run: ${{ steps.build.outputs.binary }} + run: | + . venv/bin/activate + ${{ steps.build.outputs.binary }} + pytest tests/benchmarks/python/ --codspeed --no-cov mode: simulation clang-tidy-single: diff --git a/requirements_test.txt b/requirements_test.txt index 568d79d6764..218bc0083c9 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -13,5 +13,10 @@ pytest-xdist==3.8.0 asyncmock==0.4.2 hypothesis==6.92.1 +# CodSpeed benchmarks under tests/benchmarks/python/ +# (skipped via pytest.importorskip when missing -- only required for the +# benchmarks job in .github/workflows/ci.yml) +pytest-codspeed==5.0.1 + # Used by the import-time regression check (.github/workflows/ci.yml → import-time job) importtime-waterfall==1.0.0 diff --git a/tests/benchmarks/python/__init__.py b/tests/benchmarks/python/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/benchmarks/python/conftest.py b/tests/benchmarks/python/conftest.py new file mode 100644 index 00000000000..9b0f1a3d2bb --- /dev/null +++ b/tests/benchmarks/python/conftest.py @@ -0,0 +1,22 @@ +"""Shared fixtures for the Python benchmark suite.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest + +from esphome.core import CORE + + +@pytest.fixture(autouse=True) +def reset_core_state() -> Generator[None]: + """Reset CORE before and after every benchmark. + + Per-iteration setups inside benchmarks reset CORE for the loop body; + this fixture handles the test-level boundary so stale state from + fixture priming doesn't leak across benchmarks. + """ + CORE.reset() + yield + CORE.reset() diff --git a/tests/benchmarks/python/fixtures/bluetooth_proxy_device.yaml b/tests/benchmarks/python/fixtures/bluetooth_proxy_device.yaml new file mode 100644 index 00000000000..dfa5a487b85 --- /dev/null +++ b/tests/benchmarks/python/fixtures/bluetooth_proxy_device.yaml @@ -0,0 +1,62 @@ +substitutions: + devicename: bluetooth_proxy_device + friendly_name: bluetooth_proxy_device + +esphome: + name: $devicename + friendly_name: $friendly_name + +esp32: + board: esp32-poe-iso + framework: + type: esp-idf + advanced: + sram1_as_iram: true + minimum_chip_revision: "3.0" + +esp32_ble_tracker: + scan_parameters: + active: false + +bluetooth_proxy: + active: true + +ethernet: + type: LAN8720 + mdc_pin: GPIO23 + mdio_pin: GPIO18 + clk_mode: GPIO17_OUT + phy_addr: 0 + power_pin: GPIO12 + +debug: +logger: +api: +ota: + platform: esphome + +button: + - platform: restart + name: Restart + +time: + - platform: homeassistant + id: homeassistant_time + - platform: sntp + id: sntp_time + +sensor: + - platform: uptime + name: Ethernet Uptime + - platform: template + name: Free Memory + lambda: return heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + unit_of_measurement: B + state_class: measurement + - platform: debug + free: + name: Heap Free + fragmentation: + name: Heap Fragmentation + min_free: + name: Heap Min Free diff --git a/tests/benchmarks/python/test_compiled_config_bench.py b/tests/benchmarks/python/test_compiled_config_bench.py new file mode 100644 index 00000000000..5c8892f8d00 --- /dev/null +++ b/tests/benchmarks/python/test_compiled_config_bench.py @@ -0,0 +1,116 @@ +"""CodSpeed benchmarks for the validated-config cache fast path. + +PR #16381 added a cache that lets ``esphome upload`` / ``esphome logs`` +skip re-running the full config-validation pipeline. These benchmarks +compare the cached path (``load_compiled_config``) against the slow +path (``read_config``) on the same input. + +The fixture YAML is a modest bluetooth-proxy device. The two paths +end up close on a config this small -- the win grows with config +complexity (external components, large package trees, deeply nested +schemas), where the slow path can be orders of magnitude slower than +the cache load. + +Skipped when ``pytest-codspeed`` isn't installed so the regular +unit-test suite keeps working unchanged. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +import shutil +from typing import Any + +import pytest + +from esphome.compiled_config import compiled_config_path, load_compiled_config +from esphome.config import read_config +from esphome.core import CORE +from esphome.storage_json import ext_storage_path +from esphome.writer import update_storage_json + +pytest.importorskip("pytest_codspeed") + +HERE = Path(__file__).parent +FIXTURE_YAML = HERE / "fixtures" / "bluetooth_proxy_device.yaml" + + +def _stage_yaml(tmp_path: Path) -> Path: + """Copy fixture YAML into a fresh tmp dir. + + Each benchmark gets its own copy so the cache files (under + ``.esphome/storage/`` next to the YAML) don't bleed between cases. + """ + target = tmp_path / FIXTURE_YAML.name + shutil.copy2(FIXTURE_YAML, target) + return target + + +def _prime_cache(yaml_path: Path) -> None: + """Run full validation once and persist the cache + sidecar. + + Mirrors ``esphome compile``: ``read_config`` populates ``CORE.config``, + then ``update_storage_json`` writes both the StorageJSON sidecar and + the ``.validated.yaml`` compiled-config cache. + """ + CORE.config_path = yaml_path + config = read_config({}, skip_external_update=True) + assert config is not None, f"fixture YAML failed to validate: {yaml_path}" + CORE.config = config + update_storage_json() + + +@pytest.fixture +def staged_yaml(tmp_path: Path) -> Path: + """YAML copied into tmp_path; no cache files written yet.""" + return _stage_yaml(tmp_path) + + +@pytest.fixture +def primed_yaml(staged_yaml: Path) -> Path: + """YAML plus a fresh cache + sidecar on disk.""" + _prime_cache(staged_yaml) + assert compiled_config_path(staged_yaml.name).is_file() + assert ext_storage_path(staged_yaml.name).is_file() + return staged_yaml + + +def _resetting_setup( + yaml_path: Path, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Callable[[], tuple[tuple[Any, ...], dict[str, Any]]]: + """Build a per-iteration setup that resets CORE and re-pins config_path.""" + + def setup() -> tuple[tuple[Any, ...], dict[str, Any]]: + CORE.reset() + CORE.config_path = yaml_path + return args, kwargs + + return setup + + +def test_load_compiled_config_cached(primed_yaml: Path, benchmark) -> None: + """Fast path: deserialize the cached, already-validated config.""" + benchmark.pedantic( + load_compiled_config, + setup=_resetting_setup(primed_yaml, (primed_yaml,), {}), + rounds=5, + iterations=1, + ) + + +def test_read_config_uncached(primed_yaml: Path, benchmark) -> None: + """Slow path: full validation pipeline (yaml load + schema + components). + + Uses the same primed fixture as the cached path -- ``read_config`` + ignores the cache file on disk, so the two benchmarks measure the + same input from two different code paths. + """ + benchmark.pedantic( + read_config, + setup=_resetting_setup(primed_yaml, ({},), {"skip_external_update": True}), + rounds=3, + iterations=1, + )