[wifi] Filter scan results to only store matching networks (#13409)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

esphome/components/wifi/wifi_component.cpp

@@ -39,6 +39,10 @@
 #include "esphome/components/esp32_improv/esp32_improv_component.h"
 #endif
 
+#ifdef USE_IMPROV_SERIAL
+#include "esphome/components/improv_serial/improv_serial_component.h"
+#endif
+
 namespace esphome::wifi {
 
 static const char *const TAG = "wifi";
@@ -365,6 +369,75 @@ bool WiFiComponent::ssid_was_seen_in_scan_(const std::string &ssid) const {
   return false;
 }
 
+bool WiFiComponent::needs_full_scan_results_() const {
+  // Components that require full scan results (for example, scan result listeners)
+  // are expected to call request_wifi_scan_results(), which sets keep_scan_results_.
+  if (this->keep_scan_results_) {
+    return true;
+  }
+
+#ifdef USE_CAPTIVE_PORTAL
+  // Captive portal needs full results when active (showing network list to user)
+  if (captive_portal::global_captive_portal != nullptr && captive_portal::global_captive_portal->is_active()) {
+    return true;
+  }
+#endif
+
+#ifdef USE_IMPROV_SERIAL
+  // Improv serial needs results during provisioning (before connected)
+  if (improv_serial::global_improv_serial_component != nullptr && !this->is_connected()) {
+    return true;
+  }
+#endif
+
+#ifdef USE_IMPROV
+  // BLE improv also needs results during provisioning
+  if (esp32_improv::global_improv_component != nullptr && esp32_improv::global_improv_component->is_active()) {
+    return true;
+  }
+#endif
+
+  return false;
+}
+
+bool WiFiComponent::matches_configured_network_(const char *ssid, const uint8_t *bssid) const {
+  // Hidden networks in scan results have empty SSIDs - skip them
+  if (ssid[0] == '\0') {
+    return false;
+  }
+  for (const auto &sta : this->sta_) {
+    // Skip hidden network configs (they don't appear in normal scans)
+    if (sta.get_hidden()) {
+      continue;
+    }
+    // For BSSID-only configs (empty SSID), match by BSSID
+    if (sta.get_ssid().empty()) {
+      if (sta.has_bssid() && std::memcmp(sta.get_bssid().data(), bssid, 6) == 0) {
+        return true;
+      }
+      continue;
+    }
+    // Match by SSID
+    if (sta.get_ssid() == ssid) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void WiFiComponent::log_discarded_scan_result_(const char *ssid, const uint8_t *bssid, int8_t rssi, uint8_t channel) {
+#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
+  // Skip logging during roaming scans to avoid log buffer overflow
+  // (roaming scans typically find many networks but only care about same-SSID APs)
+  if (this->roaming_state_ == RoamingState::SCANNING) {
+    return;
+  }
+  char bssid_s[MAC_ADDRESS_PRETTY_BUFFER_SIZE];
+  format_mac_addr_upper(bssid, bssid_s);
+  ESP_LOGV(TAG, "- " LOG_SECRET("'%s'") " " LOG_SECRET("(%s)") " %ddB Ch:%u", ssid, bssid_s, rssi, channel);
+#endif
+}
+
 int8_t WiFiComponent::find_next_hidden_sta_(int8_t start_index) {
   // Find next SSID to try in RETRY_HIDDEN phase.
   //
@@ -656,8 +729,12 @@ void WiFiComponent::loop() {
         ESP_LOGI(TAG, "Starting fallback AP");
         this->setup_ap_config_();
 #ifdef USE_CAPTIVE_PORTAL
-        if (captive_portal::global_captive_portal != nullptr)
+        if (captive_portal::global_captive_portal != nullptr) {
+          // Reset so we force one full scan after captive portal starts
+          // (previous scans were filtered because captive portal wasn't active yet)
+          this->has_completed_scan_after_captive_portal_start_ = false;
           captive_portal::global_captive_portal->start();
+        }
 #endif
       }
     }
@@ -1195,7 +1272,7 @@ template<typename VectorType> static void insertion_sort_scan_results(VectorType
 // has overhead from UART transmission, so combining INFO+DEBUG into one line halves
 // the blocking time. Do NOT split this into separate ESP_LOGI/ESP_LOGD calls.
 __attribute__((noinline)) static void log_scan_result(const WiFiScanResult &res) {
-  char bssid_s[18];
+  char bssid_s[MAC_ADDRESS_PRETTY_BUFFER_SIZE];
   auto bssid = res.get_bssid();
   format_mac_addr_upper(bssid.data(), bssid_s);
 
@@ -1211,18 +1288,6 @@ __attribute__((noinline)) static void log_scan_result(const WiFiScanResult &res)
 #endif
 }
 
-#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
-// Helper function to log non-matching scan results at verbose level
-__attribute__((noinline)) static void log_scan_result_non_matching(const WiFiScanResult &res) {
-  char bssid_s[18];
-  auto bssid = res.get_bssid();
-  format_mac_addr_upper(bssid.data(), bssid_s);
-
-  ESP_LOGV(TAG, "- " LOG_SECRET("'%s'") " " LOG_SECRET("(%s) ") "%s", res.get_ssid().c_str(), bssid_s,
-           LOG_STR_ARG(get_signal_bars(res.get_rssi())));
-}
-#endif
-
 void WiFiComponent::check_scanning_finished() {
   if (!this->scan_done_) {
     if (millis() - this->action_started_ > WIFI_SCAN_TIMEOUT_MS) {
@@ -1232,6 +1297,8 @@ void WiFiComponent::check_scanning_finished() {
     return;
   }
   this->scan_done_ = false;
+  this->has_completed_scan_after_captive_portal_start_ =
+      true;  // Track that we've done a scan since captive portal started
   this->retry_hidden_mode_ = RetryHiddenMode::SCAN_BASED;
 
   if (this->scan_result_.empty()) {
@@ -1259,21 +1326,12 @@ void WiFiComponent::check_scanning_finished() {
   // Sort scan results using insertion sort for better memory efficiency
   insertion_sort_scan_results(this->scan_result_);
 
-  size_t non_matching_count = 0;
+  // Log matching networks (non-matching already logged at VERBOSE in scan callback)
   for (auto &res : this->scan_result_) {
     if (res.get_matches()) {
       log_scan_result(res);
-    } else {
-#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
-      log_scan_result_non_matching(res);
-#else
-      non_matching_count++;
-#endif
     }
   }
-  if (non_matching_count > 0) {
-    ESP_LOGD(TAG, "- %zu non-matching (VERBOSE to show)", non_matching_count);
-  }
 
   // SYNCHRONIZATION POINT: Establish link between scan_result_[0] and selected_sta_index_
   // After sorting, scan_result_[0] contains the best network. Now find which sta_[i] config
@@ -1532,7 +1590,10 @@ WiFiRetryPhase WiFiComponent::determine_next_phase_() {
       if (this->went_through_explicit_hidden_phase_()) {
         return WiFiRetryPhase::EXPLICIT_HIDDEN;
       }
-      // Skip scanning when captive portal/improv is active to avoid disrupting AP.
+      // Skip scanning when captive portal/improv is active to avoid disrupting AP,
+      // BUT only if we've already completed at least one scan AFTER the portal started.
+      // When captive portal first starts, scan results may be filtered/stale, so we need
+      // to do one full scan to populate available networks for the captive portal UI.
       //
       // WHY SCANNING DISRUPTS AP MODE:
       // WiFi scanning requires the radio to leave the AP's channel and hop through
@@ -1549,7 +1610,16 @@ WiFiRetryPhase WiFiComponent::determine_next_phase_() {
       //
       // This allows users to configure WiFi via captive portal while the device keeps
       // attempting to connect to all configured networks in sequence.
-      if (this->is_captive_portal_active_() || this->is_esp32_improv_active_()) {
+      // Captive portal needs scan results to show available networks.
+      // If captive portal is active, only skip scanning if we've done a scan after it started.
+      // If only improv is active (no captive portal), skip scanning since improv doesn't need results.
+      if (this->is_captive_portal_active_()) {
+        if (this->has_completed_scan_after_captive_portal_start_) {
+          return WiFiRetryPhase::RETRY_HIDDEN;
+        }
+        // Need to scan for captive portal
+      } else if (this->is_esp32_improv_active_()) {
+        // Improv doesn't need scan results
         return WiFiRetryPhase::RETRY_HIDDEN;
       }
       return WiFiRetryPhase::SCAN_CONNECTING;
@@ -2096,7 +2166,7 @@ void WiFiComponent::clear_roaming_state_() {
 
 void WiFiComponent::release_scan_results_() {
   if (!this->keep_scan_results_) {
-#ifdef USE_RP2040
+#if defined(USE_RP2040) || defined(USE_ESP32)
     // std::vector - use swap trick since shrink_to_fit is non-binding
     decltype(this->scan_result_)().swap(this->scan_result_);
 #else

esphome/components/wifi/wifi_component.h

@@ -161,9 +161,12 @@ struct EAPAuth {
 
 using bssid_t = std::array<uint8_t, 6>;
 
-// Use std::vector for RP2040 since scan count is unknown (callback-based)
+/// Initial reserve size for filtered scan results (typical: 1-3 matching networks per SSID)
-// Use FixedVector for other platforms where count is queried first
+static constexpr size_t WIFI_SCAN_RESULT_FILTERED_RESERVE = 8;
-#ifdef USE_RP2040
+
+// Use std::vector for RP2040 (callback-based) and ESP32 (destructive scan API)
+// Use FixedVector for ESP8266 and LibreTiny where two-pass exact allocation is possible
+#if defined(USE_RP2040) || defined(USE_ESP32)
 template<typename T> using wifi_scan_vector_t = std::vector<T>;
 #else
 template<typename T> using wifi_scan_vector_t = FixedVector<T>;
@@ -539,6 +542,13 @@ class WiFiComponent : public Component {
   /// Check if an SSID was seen in the most recent scan results
   /// Used to skip hidden mode for SSIDs we know are visible
   bool ssid_was_seen_in_scan_(const std::string &ssid) const;
+  /// Check if full scan results are needed (captive portal active, improv, listeners)
+  bool needs_full_scan_results_() const;
+  /// Check if network matches any configured network (for scan result filtering)
+  /// Matches by SSID when configured, or by BSSID for BSSID-only configs
+  bool matches_configured_network_(const char *ssid, const uint8_t *bssid) const;
+  /// Log a discarded scan result at VERBOSE level (skipped during roaming scans to avoid log overflow)
+  void log_discarded_scan_result_(const char *ssid, const uint8_t *bssid, int8_t rssi, uint8_t channel);
   /// Find next SSID that wasn't in scan results (might be hidden)
   /// Returns index of next potentially hidden SSID, or -1 if none found
   /// @param start_index Start searching from index after this (-1 to start from beginning)
@@ -710,6 +720,8 @@ class WiFiComponent : public Component {
   bool enable_on_boot_{true};
   bool got_ipv4_address_{false};
   bool keep_scan_results_{false};
+  bool has_completed_scan_after_captive_portal_start_{
+      false};  // Tracks if we've completed a scan after captive portal started
   RetryHiddenMode retry_hidden_mode_{RetryHiddenMode::BLIND_RETRY};
   bool skip_cooldown_next_cycle_{false};
   bool post_connect_roaming_{true};  // Enabled by default

esphome/components/wifi/wifi_component_esp8266.cpp

@@ -760,20 +760,35 @@ void WiFiComponent::wifi_scan_done_callback_(void *arg, STATUS status) {
     return;
   }
 
-  // Count the number of results first
   auto *head = reinterpret_cast<bss_info *>(arg);
+  bool needs_full = this->needs_full_scan_results_();
+
+  // First pass: count matching networks (linked list is non-destructive)
+  size_t total = 0;
   size_t count = 0;
   for (bss_info *it = head; it != nullptr; it = STAILQ_NEXT(it, next)) {
-    count++;
+    total++;
+    const char *ssid_cstr = reinterpret_cast<const char *>(it->ssid);
+    if (needs_full || this->matches_configured_network_(ssid_cstr, it->bssid)) {
+      count++;
+    }
   }
 
-  this->scan_result_.init(count);
+  this->scan_result_.init(count);  // Exact allocation
+
+  // Second pass: store matching networks
   for (bss_info *it = head; it != nullptr; it = STAILQ_NEXT(it, next)) {
-    this->scan_result_.emplace_back(
+    const char *ssid_cstr = reinterpret_cast<const char *>(it->ssid);
-        bssid_t{it->bssid[0], it->bssid[1], it->bssid[2], it->bssid[3], it->bssid[4], it->bssid[5]},
+    if (needs_full || this->matches_configured_network_(ssid_cstr, it->bssid)) {
-        std::string(reinterpret_cast<char *>(it->ssid), it->ssid_len), it->channel, it->rssi, it->authmode != AUTH_OPEN,
+      this->scan_result_.emplace_back(
-        it->is_hidden != 0);
+          bssid_t{it->bssid[0], it->bssid[1], it->bssid[2], it->bssid[3], it->bssid[4], it->bssid[5]},
+          std::string(ssid_cstr, it->ssid_len), it->channel, it->rssi, it->authmode != AUTH_OPEN, it->is_hidden != 0);
+    } else {
+      this->log_discarded_scan_result_(ssid_cstr, it->bssid, it->rssi, it->channel);
+    }
   }
+  ESP_LOGV(TAG, "Scan complete: %zu found, %zu stored%s", total, this->scan_result_.size(),
+           needs_full ? "" : " (filtered)");
   this->scan_done_ = true;
 #ifdef USE_WIFI_SCAN_RESULTS_LISTENERS
   for (auto *listener : global_wifi_component->scan_results_listeners_) {

esphome/components/wifi/wifi_component_esp_idf.cpp

@@ -828,11 +828,21 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) {
     }
 
     uint16_t number = it.number;
-    scan_result_.init(number);
+    bool needs_full = this->needs_full_scan_results_();
+
+    // Smart reserve: full capacity if needed, small reserve otherwise
+    if (needs_full) {
+      this->scan_result_.reserve(number);
+    } else {
+      this->scan_result_.reserve(WIFI_SCAN_RESULT_FILTERED_RESERVE);
+    }
+
 #ifdef USE_ESP32_HOSTED
     // getting records one at a time fails on P4 with hosted esp32 WiFi coprocessor
     // Presumably an upstream bug, work-around by getting all records at once
-    auto records = std::make_unique<wifi_ap_record_t[]>(number);
+    // Use stack buffer (3904 bytes / ~80 bytes per record = ~48 records) with heap fallback
+    static constexpr size_t SCAN_RECORD_STACK_COUNT = 3904 / sizeof(wifi_ap_record_t);
+    SmallBufferWithHeapFallback<SCAN_RECORD_STACK_COUNT, wifi_ap_record_t> records(number);
     err = esp_wifi_scan_get_ap_records(&number, records.get());
     if (err != ESP_OK) {
       esp_wifi_clear_ap_list();
@@ -840,7 +850,7 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) {
       return;
     }
     for (uint16_t i = 0; i < number; i++) {
-      wifi_ap_record_t &record = records[i];
+      wifi_ap_record_t &record = records.get()[i];
 #else
     // Process one record at a time to avoid large buffer allocation
     for (uint16_t i = 0; i < number; i++) {
@@ -852,12 +862,23 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) {
         break;
       }
 #endif  // USE_ESP32_HOSTED
-      bssid_t bssid;
+
-      std::copy(record.bssid, record.bssid + 6, bssid.begin());
+      // Check C string first - avoid std::string construction for non-matching networks
-      std::string ssid(reinterpret_cast<const char *>(record.ssid));
+      const char *ssid_cstr = reinterpret_cast<const char *>(record.ssid);
-      scan_result_.emplace_back(bssid, ssid, record.primary, record.rssi, record.authmode != WIFI_AUTH_OPEN,
+
-                                ssid.empty());
+      // Only construct std::string and store if needed
+      if (needs_full || this->matches_configured_network_(ssid_cstr, record.bssid)) {
+        bssid_t bssid;
+        std::copy(record.bssid, record.bssid + 6, bssid.begin());
+        std::string ssid(ssid_cstr);
+        this->scan_result_.emplace_back(bssid, std::move(ssid), record.primary, record.rssi,
+                                        record.authmode != WIFI_AUTH_OPEN, ssid_cstr[0] == '\0');
+      } else {
+        this->log_discarded_scan_result_(ssid_cstr, record.bssid, record.rssi, record.primary);
+      }
     }
+    ESP_LOGV(TAG, "Scan complete: %u found, %zu stored%s", number, this->scan_result_.size(),
+             needs_full ? "" : " (filtered)");
 #ifdef USE_WIFI_SCAN_RESULTS_LISTENERS
     for (auto *listener : this->scan_results_listeners_) {
       listener->on_wifi_scan_results(this->scan_result_);

esphome/components/wifi/wifi_component_libretiny.cpp

@@ -670,18 +670,39 @@ void WiFiComponent::wifi_scan_done_callback_() {
   if (num < 0)
     return;
 
-  this->scan_result_.init(static_cast<unsigned int>(num));
+  bool needs_full = this->needs_full_scan_results_();
-  for (int i = 0; i < num; i++) {
-    String ssid = WiFi.SSID(i);
-    wifi_auth_mode_t authmode = WiFi.encryptionType(i);
-    int32_t rssi = WiFi.RSSI(i);
-    uint8_t *bssid = WiFi.BSSID(i);
-    int32_t channel = WiFi.channel(i);
 
-    this->scan_result_.emplace_back(bssid_t{bssid[0], bssid[1], bssid[2], bssid[3], bssid[4], bssid[5]},
+  // Access scan results directly via WiFi.scan struct to avoid Arduino String allocations
-                                    std::string(ssid.c_str()), channel, rssi, authmode != WIFI_AUTH_OPEN,
+  // WiFi.scan is public in LibreTiny for WiFiEvents & WiFiScan static handlers
-                                    ssid.length() == 0);
+  auto *scan = WiFi.scan;
+
+  // First pass: count matching networks
+  size_t count = 0;
+  for (int i = 0; i < num; i++) {
+    const char *ssid_cstr = scan->ap[i].ssid;
+    if (needs_full || this->matches_configured_network_(ssid_cstr, scan->ap[i].bssid.addr)) {
+      count++;
+    }
   }
+
+  this->scan_result_.init(count);  // Exact allocation
+
+  // Second pass: store matching networks
+  for (int i = 0; i < num; i++) {
+    const char *ssid_cstr = scan->ap[i].ssid;
+    if (needs_full || this->matches_configured_network_(ssid_cstr, scan->ap[i].bssid.addr)) {
+      auto &ap = scan->ap[i];
+      this->scan_result_.emplace_back(bssid_t{ap.bssid.addr[0], ap.bssid.addr[1], ap.bssid.addr[2], ap.bssid.addr[3],
+                                              ap.bssid.addr[4], ap.bssid.addr[5]},
+                                      std::string(ssid_cstr), ap.channel, ap.rssi, ap.auth != WIFI_AUTH_OPEN,
+                                      ssid_cstr[0] == '\0');
+    } else {
+      auto &ap = scan->ap[i];
+      this->log_discarded_scan_result_(ssid_cstr, ap.bssid.addr, ap.rssi, ap.channel);
+    }
+  }
+  ESP_LOGV(TAG, "Scan complete: %d found, %zu stored%s", num, this->scan_result_.size(),
+           needs_full ? "" : " (filtered)");
   WiFi.scanDelete();
 #ifdef USE_WIFI_SCAN_RESULTS_LISTENERS
   for (auto *listener : this->scan_results_listeners_) {

esphome/components/wifi/wifi_component_pico_w.cpp

@@ -21,6 +21,7 @@ static const char *const TAG = "wifi_pico_w";
 // Track previous state for detecting changes
 static bool s_sta_was_connected = false;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 static bool s_sta_had_ip = false;         // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+static size_t s_scan_result_count = 0;    // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 bool WiFiComponent::wifi_mode_(optional<bool> sta, optional<bool> ap) {
   if (sta.has_value()) {
@@ -137,10 +138,20 @@ int WiFiComponent::s_wifi_scan_result(void *env, const cyw43_ev_scan_result_t *r
 }
 
 void WiFiComponent::wifi_scan_result(void *env, const cyw43_ev_scan_result_t *result) {
+  s_scan_result_count++;
+  const char *ssid_cstr = reinterpret_cast<const char *>(result->ssid);
+
+  // Skip networks that don't match any configured network (unless full results needed)
+  if (!this->needs_full_scan_results_() && !this->matches_configured_network_(ssid_cstr, result->bssid)) {
+    this->log_discarded_scan_result_(ssid_cstr, result->bssid, result->rssi, result->channel);
+    return;
+  }
+
   bssid_t bssid;
   std::copy(result->bssid, result->bssid + 6, bssid.begin());
-  std::string ssid(reinterpret_cast<const char *>(result->ssid));
+  std::string ssid(ssid_cstr);
-  WiFiScanResult res(bssid, ssid, result->channel, result->rssi, result->auth_mode != CYW43_AUTH_OPEN, ssid.empty());
+  WiFiScanResult res(bssid, std::move(ssid), result->channel, result->rssi, result->auth_mode != CYW43_AUTH_OPEN,
+                     ssid_cstr[0] == '\0');
   if (std::find(this->scan_result_.begin(), this->scan_result_.end(), res) == this->scan_result_.end()) {
     this->scan_result_.push_back(res);
   }
@@ -149,6 +160,7 @@ void WiFiComponent::wifi_scan_result(void *env, const cyw43_ev_scan_result_t *re
 bool WiFiComponent::wifi_scan_start_(bool passive) {
   this->scan_result_.clear();
   this->scan_done_ = false;
+  s_scan_result_count = 0;
   cyw43_wifi_scan_options_t scan_options = {0};
   scan_options.scan_type = passive ? 1 : 0;
   int err = cyw43_wifi_scan(&cyw43_state, &scan_options, nullptr, &s_wifi_scan_result);
@@ -244,7 +256,9 @@ void WiFiComponent::wifi_loop_() {
   // Handle scan completion
   if (this->state_ == WIFI_COMPONENT_STATE_STA_SCANNING && !cyw43_wifi_scan_active(&cyw43_state)) {
     this->scan_done_ = true;
-    ESP_LOGV(TAG, "Scan done");
+    bool needs_full = this->needs_full_scan_results_();
+    ESP_LOGV(TAG, "Scan complete: %zu found, %zu stored%s", s_scan_result_count, this->scan_result_.size(),
+             needs_full ? "" : " (filtered)");
 #ifdef USE_WIFI_SCAN_RESULTS_LISTENERS
     for (auto *listener : this->scan_results_listeners_) {
       listener->on_wifi_scan_results(this->scan_result_);