diff --git a/esphome/components/improv_serial/improv_serial_component.cpp b/esphome/components/improv_serial/improv_serial_component.cpp
index 003328d535e..9b8cf75f3a8 100644
--- a/esphome/components/improv_serial/improv_serial_component.cpp
+++ b/esphome/components/improv_serial/improv_serial_component.cpp
@@ -17,6 +17,7 @@ void ImprovSerialComponent::setup() {
   global_improv_serial_component = this;
 #ifdef USE_ESP32
   this->uart_num_ = logger::global_logger->get_uart_num();
+  this->uart_selection_ = logger::global_logger->get_uart();
 #elif defined(USE_ARDUINO)
   this->hw_serial_ = logger::global_logger->get_hw_serial();
 #endif
@@ -29,7 +30,8 @@ void ImprovSerialComponent::setup() {
 }
 
 void ImprovSerialComponent::loop() {
-  if (this->last_read_byte_ && (millis() - this->last_read_byte_ > IMPROV_SERIAL_TIMEOUT)) {
+  const uint32_t now = App.get_loop_component_start_time();
+  if (this->last_read_byte_ && (now - this->last_read_byte_ > IMPROV_SERIAL_TIMEOUT)) {
     this->last_read_byte_ = 0;
     this->rx_buffer_.clear();
     ESP_LOGV(TAG, "Timeout");
@@ -38,7 +40,7 @@ void ImprovSerialComponent::loop() {
   auto byte = this->read_byte_();
   while (byte.has_value()) {
     if (this->parse_improv_serial_byte_(byte.value())) {
-      this->last_read_byte_ = millis();
+      this->last_read_byte_ = now;
     } else {
       this->last_read_byte_ = 0;
       this->rx_buffer_.clear();
@@ -62,55 +64,6 @@ void ImprovSerialComponent::loop() {
 
 void ImprovSerialComponent::dump_config() { ESP_LOGCONFIG(TAG, "Improv Serial:"); }
 
-optional<uint8_t> ImprovSerialComponent::read_byte_() {
-  optional<uint8_t> byte;
-  uint8_t data = 0;
-#ifdef USE_ESP32
-  switch (logger::global_logger->get_uart()) {
-    case logger::UART_SELECTION_UART0:
-    case logger::UART_SELECTION_UART1:
-#if !defined(USE_ESP32_VARIANT_ESP32C3) && !defined(USE_ESP32_VARIANT_ESP32C6) && \
-    !defined(USE_ESP32_VARIANT_ESP32C61) && !defined(USE_ESP32_VARIANT_ESP32S2) && !defined(USE_ESP32_VARIANT_ESP32S3)
-    case logger::UART_SELECTION_UART2:
-#endif  // !USE_ESP32_VARIANT_ESP32C3 && !USE_ESP32_VARIANT_ESP32C6 && !USE_ESP32_VARIANT_ESP32C61 &&
-        // !USE_ESP32_VARIANT_ESP32S2 && !USE_ESP32_VARIANT_ESP32S3
-      if (this->uart_num_ >= 0) {
-        size_t available;
-        uart_get_buffered_data_len(this->uart_num_, &available);
-        if (available) {
-          uart_read_bytes(this->uart_num_, &data, 1, 0);
-          byte = data;
-        }
-      }
-      break;
-#if defined(USE_LOGGER_USB_CDC) && defined(CONFIG_ESP_CONSOLE_USB_CDC)
-    case logger::UART_SELECTION_USB_CDC:
-      if (esp_usb_console_available_for_read()) {
-        esp_usb_console_read_buf((char *) &data, 1);
-        byte = data;
-      }
-      break;
-#endif  // USE_LOGGER_USB_CDC
-#ifdef USE_LOGGER_USB_SERIAL_JTAG
-    case logger::UART_SELECTION_USB_SERIAL_JTAG: {
-      if (usb_serial_jtag_read_bytes((char *) &data, 1, 0)) {
-        byte = data;
-      }
-      break;
-    }
-#endif  // USE_LOGGER_USB_SERIAL_JTAG
-    default:
-      break;
-  }
-#elif defined(USE_ARDUINO)
-  if (this->hw_serial_->available()) {
-    this->hw_serial_->readBytes(&data, 1);
-    byte = data;
-  }
-#endif
-  return byte;
-}
-
 void ImprovSerialComponent::write_data_(const uint8_t *data, const size_t size) {
   // First, set length field
   this->tx_header_[TX_LENGTH_IDX] = this->tx_header_[TX_TYPE_IDX] == TYPE_RPC_RESPONSE ? size : 1;
@@ -134,7 +87,7 @@ void ImprovSerialComponent::write_data_(const uint8_t *data, const size_t size)
   this->tx_header_[TX_CHECKSUM_IDX] = checksum;
 
 #ifdef USE_ESP32
-  switch (logger::global_logger->get_uart()) {
+  switch (this->uart_selection_) {
     case logger::UART_SELECTION_UART0:
     case logger::UART_SELECTION_UART1:
 #if !defined(USE_ESP32_VARIANT_ESP32C3) && !defined(USE_ESP32_VARIANT_ESP32C6) && \
diff --git a/esphome/components/improv_serial/improv_serial_component.h b/esphome/components/improv_serial/improv_serial_component.h
index dd8f5e47191..4b09c662aab 100644
--- a/esphome/components/improv_serial/improv_serial_component.h
+++ b/esphome/components/improv_serial/improv_serial_component.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "esphome/components/improv_base/improv_base.h"
+#include "esphome/components/logger/logger.h"
 #include "esphome/components/wifi/wifi_component.h"
 #include "esphome/core/component.h"
 #include "esphome/core/defines.h"
@@ -66,7 +67,53 @@ class ImprovSerialComponent : public Component, public improv_base::ImprovBase {
   std::vector<uint8_t> build_rpc_settings_response_(improv::Command command);
   std::vector<uint8_t> build_version_info_();
 
-  optional<uint8_t> read_byte_();
+  ESPHOME_ALWAYS_INLINE optional<uint8_t> read_byte_() {
+    optional<uint8_t> byte;
+    uint8_t data = 0;
+#ifdef USE_ESP32
+    switch (this->uart_selection_) {
+      case logger::UART_SELECTION_UART0:
+      case logger::UART_SELECTION_UART1:
+#if !defined(USE_ESP32_VARIANT_ESP32C3) && !defined(USE_ESP32_VARIANT_ESP32C6) && \
+    !defined(USE_ESP32_VARIANT_ESP32C61) && !defined(USE_ESP32_VARIANT_ESP32S2) && !defined(USE_ESP32_VARIANT_ESP32S3)
+      case logger::UART_SELECTION_UART2:
+#endif
+        if (this->uart_num_ >= 0) {
+          size_t available;
+          uart_get_buffered_data_len(this->uart_num_, &available);
+          if (available) {
+            uart_read_bytes(this->uart_num_, &data, 1, 0);
+            byte = data;
+          }
+        }
+        break;
+#if defined(USE_LOGGER_USB_CDC) && defined(CONFIG_ESP_CONSOLE_USB_CDC)
+      case logger::UART_SELECTION_USB_CDC:
+        if (esp_usb_console_available_for_read()) {
+          esp_usb_console_read_buf((char *) &data, 1);
+          byte = data;
+        }
+        break;
+#endif
+#ifdef USE_LOGGER_USB_SERIAL_JTAG
+      case logger::UART_SELECTION_USB_SERIAL_JTAG: {
+        if (usb_serial_jtag_read_bytes((char *) &data, 1, 0)) {
+          byte = data;
+        }
+        break;
+      }
+#endif
+      default:
+        break;
+    }
+#elif defined(USE_ARDUINO)
+    if (this->hw_serial_->available()) {
+      this->hw_serial_->readBytes(&data, 1);
+      byte = data;
+    }
+#endif
+    return byte;
+  }
   void write_data_(const uint8_t *data = nullptr, size_t size = 0);
 
   uint8_t tx_header_[TX_BUFFER_SIZE] = {
@@ -86,6 +133,7 @@ class ImprovSerialComponent : public Component, public improv_base::ImprovBase {
 
 #ifdef USE_ESP32
   uart_port_t uart_num_;
+  logger::UARTSelection uart_selection_{logger::UART_SELECTION_UART0};
 #elif defined(USE_ARDUINO)
   Stream *hw_serial_{nullptr};
 #endif