Add integration tests for host (#8912)

.coveragerc

@@ -1,2 +1,4 @@
 [run]
-omit = esphome/components/*
+omit = 
+    esphome/components/*
+    tests/integration/*

.github/workflows/ci.yml

@@ -214,12 +214,12 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: |
           ./venv/Scripts/activate
-          pytest -vv --cov-report=xml --tb=native tests
+          pytest -vv --cov-report=xml --tb=native -n auto tests
       - name: Run pytest
         if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
         run: |
           . venv/bin/activate
-          pytest -vv --cov-report=xml --tb=native tests
+          pytest -vv --cov-report=xml --tb=native -n auto tests
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5.4.3
         with:

requirements_test.txt

@@ -9,5 +9,6 @@ pytest==8.3.5
 pytest-cov==6.1.1
 pytest-mock==3.14.0
 pytest-asyncio==0.26.0
+pytest-xdist==3.6.1
 asyncmock==0.4.2
 hypothesis==6.92.1

script/integration_test

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "${script_dir}/.."
+
+set -x
+
+pytest -vvs --no-cov --tb=native -n 0 tests/integration/

tests/integration/README.md

@@ -0,0 +1,80 @@
+# ESPHome Integration Tests
+
+This directory contains end-to-end integration tests for ESPHome, focusing on testing the complete flow from YAML configuration to running devices with API connections.
+
+## Structure
+
+- `conftest.py` - Common fixtures and utilities
+- `const.py` - Constants used throughout the integration tests
+- `types.py` - Type definitions for fixtures and functions
+- `fixtures/` - YAML configuration files for tests
+- `test_*.py` - Individual test files
+
+## How it works
+
+### Automatic YAML Loading
+
+The `yaml_config` fixture automatically loads YAML configurations based on the test name:
+- It looks for a file named after the test function (e.g., `test_host_mode_basic` → `fixtures/host_mode_basic.yaml`)
+- The fixture file must exist or the test will fail with a clear error message
+- The fixture automatically injects a dynamic port number into the API configuration
+
+### Key Fixtures
+
+- `run_compiled` - Combines write, compile, and run operations into a single context manager
+- `api_client_connected` - Creates an API client that automatically connects using ReconnectLogic
+- `reserved_tcp_port` - Reserves a TCP port by holding the socket open until ESPHome needs it
+- `unused_tcp_port` - Provides the reserved port number for each test
+
+### Writing Tests
+
+The simplest way to write a test is to use the `run_compiled` and `api_client_connected` fixtures:
+
+```python
+@pytest.mark.asyncio
+async def test_my_feature(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    # Write, compile and run the ESPHome device, then connect to API
+    async with run_compiled(yaml_config), api_client_connected() as client:
+        # Test your feature using the connected client
+        device_info = await client.device_info()
+        assert device_info is not None
+```
+
+### Creating YAML Fixtures
+
+Create a YAML file in the `fixtures/` directory with the same name as your test function (without the `test_` prefix):
+
+```yaml
+# fixtures/my_feature.yaml
+esphome:
+  name: my-test-device
+host:
+api:  # Port will be automatically injected
+logger:
+# Add your components here
+```
+
+## Running Tests
+
+```bash
+# Run all integration tests
+script/integration_test
+
+# Run a specific test
+pytest -vv tests/integration/test_host_mode_basic.py
+
+# Debug compilation errors or see ESPHome output
+pytest -s tests/integration/test_host_mode_basic.py
+```
+
+## Implementation Details
+
+- Tests automatically wait for the API port to be available before connecting
+- Process cleanup is handled automatically, with graceful shutdown using SIGINT
+- Each test gets its own temporary directory and unique port
+- Port allocation minimizes race conditions by holding the socket until just before ESPHome starts
+- Output from ESPHome processes is displayed for debugging

tests/integration/__init__.py

@@ -0,0 +1,3 @@
+"""ESPHome integration tests."""
+
+from __future__ import annotations

tests/integration/const.py

@@ -0,0 +1,14 @@
+"""Constants for integration tests."""
+
+# Network constants
+DEFAULT_API_PORT = 6053
+LOCALHOST = "localhost"
+
+# Timeout constants
+API_CONNECTION_TIMEOUT = 30.0  # seconds
+PORT_WAIT_TIMEOUT = 30.0  # seconds
+PORT_POLL_INTERVAL = 0.1  # seconds
+
+# Process shutdown timeouts
+SIGINT_TIMEOUT = 5.0  # seconds
+SIGTERM_TIMEOUT = 2.0  # seconds

tests/integration/fixtures/host_mode_basic.yaml

@@ -0,0 +1,5 @@
+esphome:
+  name: host-test
+host:
+api:
+logger:

tests/integration/fixtures/host_mode_noise_encryption.yaml

@@ -0,0 +1,7 @@
+esphome:
+  name: host-noise-test
+host:
+api:
+  encryption:
+    key: N4Yle5YirwZhPiHHsdZLdOA73ndj/84veVaLhTvxCuU=
+logger:

tests/integration/fixtures/host_mode_noise_encryption_wrong_key.yaml

@@ -0,0 +1,7 @@
+esphome:
+  name: host-noise-test
+host:
+api:
+  encryption:
+    key: N4Yle5YirwZhPiHHsdZLdOA73ndj/84veVaLhTvxCuU=
+logger:

tests/integration/fixtures/host_mode_reconnect.yaml

@@ -0,0 +1,5 @@
+esphome:
+  name: host-reconnect-test
+host:
+api:
+logger:

tests/integration/fixtures/host_mode_with_sensor.yaml

@@ -0,0 +1,12 @@
+esphome:
+  name: host-sensor-test
+host:
+api:
+logger:
+sensor:
+  - platform: template
+    name: Test Sensor
+    id: test_sensor
+    unit_of_measurement: °C
+    lambda: return 42.0;
+    update_interval: 0.1s

tests/integration/test_host_mode_basic.py

@@ -0,0 +1,22 @@
+"""Basic integration test for Host mode."""
+
+from __future__ import annotations
+
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_host_mode_basic(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test basic Host mode functionality with API connection."""
+    # Write, compile and run the ESPHome device, then connect to API
+    async with run_compiled(yaml_config), api_client_connected() as client:
+        # Verify we can get device info
+        device_info = await client.device_info()
+        assert device_info is not None
+        assert device_info.name == "host-test"

tests/integration/test_host_mode_noise_encryption.py

@@ -0,0 +1,53 @@
+"""Integration test for Host mode with noise encryption."""
+
+from __future__ import annotations
+
+from aioesphomeapi import InvalidEncryptionKeyAPIError
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+# The API key for noise encryption
+NOISE_KEY = "N4Yle5YirwZhPiHHsdZLdOA73ndj/84veVaLhTvxCuU="
+
+
+@pytest.mark.asyncio
+async def test_host_mode_noise_encryption(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test Host mode with noise encryption enabled."""
+    # Write, compile and run the ESPHome device, then connect to API
+    # The API client should handle noise encryption automatically
+    async with (
+        run_compiled(yaml_config),
+        api_client_connected(noise_psk=NOISE_KEY) as client,
+    ):
+        # If we can get device info, the encryption is working
+        device_info = await client.device_info()
+        assert device_info is not None
+        assert device_info.name == "host-noise-test"
+
+        # List entities to ensure the encrypted connection is fully functional
+        entities = await client.list_entities_services()
+        assert entities is not None
+
+
+@pytest.mark.asyncio
+async def test_host_mode_noise_encryption_wrong_key(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test that connection fails with wrong encryption key."""
+    # Write, compile and run the ESPHome device
+    async with run_compiled(yaml_config):
+        # Try to connect with wrong key - should fail with InvalidEncryptionKeyAPIError
+        with pytest.raises(InvalidEncryptionKeyAPIError):
+            async with api_client_connected(
+                noise_psk="wrong_key_that_should_not_work",
+                timeout=5,  # Shorter timeout for expected failure
+            ) as client:
+                # This should not be reached
+                await client.device_info()

tests/integration/test_host_mode_reconnect.py

@@ -0,0 +1,28 @@
+"""Integration test for Host mode reconnection."""
+
+from __future__ import annotations
+
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_host_mode_reconnect(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test reconnecting to a Host mode device."""
+    # Write, compile and run the ESPHome device
+    async with run_compiled(yaml_config):
+        # First connection
+        async with api_client_connected() as client:
+            device_info = await client.device_info()
+            assert device_info is not None
+
+        # Reconnect with a new client
+        async with api_client_connected() as client2:
+            device_info2 = await client2.device_info()
+            assert device_info2 is not None
+            assert device_info2.name == device_info.name

tests/integration/test_host_mode_sensor.py

@@ -0,0 +1,49 @@
+"""Integration test for Host mode with sensor."""
+
+from __future__ import annotations
+
+import asyncio
+
+from aioesphomeapi import EntityState
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_host_mode_with_sensor(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test Host mode with a sensor component."""
+    # Write, compile and run the ESPHome device, then connect to API
+    async with run_compiled(yaml_config), api_client_connected() as client:
+        # Subscribe to state changes
+        states: dict[int, EntityState] = {}
+        sensor_future: asyncio.Future[EntityState] = asyncio.Future()
+
+        def on_state(state: EntityState) -> None:
+            states[state.key] = state
+            # If this is our sensor with value 42.0, resolve the future
+            if (
+                hasattr(state, "state")
+                and state.state == 42.0
+                and not sensor_future.done()
+            ):
+                sensor_future.set_result(state)
+
+        client.subscribe_states(on_state)
+
+        # Wait for sensor with specific value (42.0) with timeout
+        try:
+            test_sensor_state = await asyncio.wait_for(sensor_future, timeout=5.0)
+        except asyncio.TimeoutError:
+            pytest.fail(
+                f"Sensor with value 42.0 not received within 5 seconds. "
+                f"Received states: {list(states.values())}"
+            )
+
+        # Verify the sensor state
+        assert test_sensor_state.state == 42.0
+        assert len(states) > 0, "No states received"

tests/integration/types.py

@@ -0,0 +1,46 @@
+"""Type definitions for integration tests."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from contextlib import AbstractAsyncContextManager
+from pathlib import Path
+from typing import Protocol
+
+from aioesphomeapi import APIClient
+
+ConfigWriter = Callable[[str, str | None], Awaitable[Path]]
+CompileFunction = Callable[[Path], Awaitable[None]]
+RunFunction = Callable[[Path], Awaitable[asyncio.subprocess.Process]]
+RunCompiledFunction = Callable[
+    [str, str | None], AbstractAsyncContextManager[asyncio.subprocess.Process]
+]
+WaitFunction = Callable[[APIClient, float], Awaitable[bool]]
+
+
+class APIClientFactory(Protocol):
+    """Protocol for API client factory."""
+
+    def __call__(  # noqa: E704
+        self,
+        address: str = "localhost",
+        port: int | None = None,
+        password: str = "",
+        noise_psk: str | None = None,
+        client_info: str = "integration-test",
+    ) -> AbstractAsyncContextManager[APIClient]: ...
+
+
+class APIClientConnectedFactory(Protocol):
+    """Protocol for connected API client factory."""
+
+    def __call__(  # noqa: E704
+        self,
+        address: str = "localhost",
+        port: int | None = None,
+        password: str = "",
+        noise_psk: str | None = None,
+        client_info: str = "integration-test",
+        timeout: float = 30,
+    ) -> AbstractAsyncContextManager[APIClient]: ...