diff --git a/esphome/components/i2s_audio/__init__.py b/esphome/components/i2s_audio/__init__.py
index 65b09b93f6..977a239497 100644
--- a/esphome/components/i2s_audio/__init__.py
+++ b/esphome/components/i2s_audio/__init__.py
@@ -1,3 +1,5 @@
+from dataclasses import dataclass, field
+
 from esphome import pins
 import esphome.codegen as cg
 from esphome.components.esp32 import (
@@ -26,6 +28,9 @@ CODEOWNERS = ["@jesserockz"]
 DEPENDENCIES = ["esp32"]
 MULTI_CONF = True
 
+CONF_PDM = "pdm"
+CONF_ADC_TYPE = "adc_type"
+
 CONF_I2S_DOUT_PIN = "i2s_dout_pin"
 CONF_I2S_DIN_PIN = "i2s_din_pin"
 CONF_I2S_MCLK_PIN = "i2s_mclk_pin"
@@ -254,7 +259,65 @@ CONFIG_SCHEMA = cv.All(
 )
 
 
+@dataclass
+class I2SAudioData:
+    """I2S audio component state stored in CORE.data."""
+
+    port_map: dict[str, int] = field(default_factory=dict)
+
+
+def _get_data() -> I2SAudioData:
+    if CONF_I2S_AUDIO not in CORE.data:
+        CORE.data[CONF_I2S_AUDIO] = I2SAudioData()
+    return CORE.data[CONF_I2S_AUDIO]
+
+
+def _assign_ports() -> None:
+    """Assign I2S port numbers, prioritizing instances with microphone children.
+
+    Microphones (especially PDM) require port 0 on most ESP32 variants.
+    This runs once and stores the mapping in CORE.data.
+    """
+    data = _get_data()
+    if data.port_map:
+        return
+
+    full_config = fv.full_config.get()
+    i2s_configs = full_config[CONF_I2S_AUDIO]
+
+    # Find i2s_audio instances with microphones that require port 0
+    # (PDM and internal ADC only work on I2S port 0)
+    port0_parent_id = None
+    for mic_config in full_config.get("microphone", []):
+        if CONF_I2S_AUDIO_ID not in mic_config:
+            continue
+        if mic_config.get(CONF_PDM) or mic_config.get(CONF_ADC_TYPE) == "internal":
+            if port0_parent_id is not None:
+                raise cv.Invalid(
+                    "Only one PDM/ADC microphone is supported (requires I2S port 0)"
+                )
+            port0_parent_id = str(mic_config[CONF_I2S_AUDIO_ID])
+
+    # Assign ports: port 0 parent first (if any), rest get sequential
+    next_port = 0
+    if port0_parent_id is not None:
+        data.port_map[port0_parent_id] = next_port
+        next_port += 1
+    for config in i2s_configs:
+        config_id = str(config[CONF_ID])
+        if config_id != port0_parent_id:
+            data.port_map[config_id] = next_port
+            next_port += 1
+
+
 def _final_validate(_):
+    from esphome.components.esp32 import idf_version
+
+    if use_legacy() and idf_version() >= cv.Version(6, 0, 0):
+        raise cv.Invalid(
+            "The legacy I2S driver is not available in ESP-IDF 6.0+. "
+            "Set 'use_legacy: false' in i2s_audio configuration."
+        )
     i2s_audio_configs = fv.full_config.get()[CONF_I2S_AUDIO]
     variant = get_esp32_variant()
     if variant not in I2S_PORTS:
@@ -263,6 +326,7 @@ def _final_validate(_):
         raise cv.Invalid(
             f"Only {I2S_PORTS[variant]} I2S audio ports are supported on {variant}"
         )
+    _assign_ports()
 
 
 def use_legacy():
@@ -276,6 +340,12 @@ async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
 
+    # Assign I2S port from _final_validate computed mapping
+    data = _get_data()
+    if (port := data.port_map.get(str(config[CONF_ID]))) is None:
+        raise ValueError(f"No I2S port assigned for {config[CONF_ID]}")
+    cg.add(var.set_port(port))
+
     # Re-enable ESP-IDF's I2S driver (excluded by default to save compile time)
     include_builtin_idf_component("esp_driver_i2s")
 
diff --git a/esphome/components/i2s_audio/i2s_audio.cpp b/esphome/components/i2s_audio/i2s_audio.cpp
deleted file mode 100644
index 43064498cc..0000000000
--- a/esphome/components/i2s_audio/i2s_audio.cpp
+++ /dev/null
@@ -1,27 +0,0 @@
-#include "i2s_audio.h"
-
-#ifdef USE_ESP32
-
-#include "esphome/core/log.h"
-
-namespace esphome {
-namespace i2s_audio {
-
-static const char *const TAG = "i2s_audio";
-
-void I2SAudioComponent::setup() {
-  static i2s_port_t next_port_num = I2S_NUM_0;
-  if (next_port_num >= SOC_I2S_NUM) {
-    ESP_LOGE(TAG, "Too many components");
-    this->mark_failed();
-    return;
-  }
-
-  this->port_ = next_port_num;
-  next_port_num = (i2s_port_t) (next_port_num + 1);
-}
-
-}  // namespace i2s_audio
-}  // namespace esphome
-
-#endif  // USE_ESP32
diff --git a/esphome/components/i2s_audio/i2s_audio.h b/esphome/components/i2s_audio/i2s_audio.h
index cfccf7e01f..f26ffddd46 100644
--- a/esphome/components/i2s_audio/i2s_audio.h
+++ b/esphome/components/i2s_audio/i2s_audio.h
@@ -5,6 +5,7 @@
 #include "esphome/core/component.h"
 #include "esphome/core/defines.h"
 #include "esphome/core/helpers.h"
+#include <esp_idf_version.h>
 #ifdef USE_I2S_LEGACY
 #include <driver/i2s.h>
 #else
@@ -56,8 +57,6 @@ class I2SAudioOut : public I2SAudioBase {};
 
 class I2SAudioComponent : public Component {
  public:
-  void setup() override;
-
 #ifdef USE_I2S_LEGACY
   i2s_pin_config_t get_pin_config() const {
     return {
@@ -86,13 +85,17 @@ class I2SAudioComponent : public Component {
   void set_mclk_pin(int pin) { this->mclk_pin_ = pin; }
   void set_bclk_pin(int pin) { this->bclk_pin_ = pin; }
   void set_lrclk_pin(int pin) { this->lrclk_pin_ = pin; }
+  void set_port(int port) { this->port_ = port; }
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0)
+  int get_port() const { return this->port_; }
+#else
+  i2s_port_t get_port() const { return static_cast<i2s_port_t>(this->port_); }
+#endif
 
   void lock() { this->lock_.lock(); }
   bool try_lock() { return this->lock_.try_lock(); }
   void unlock() { this->lock_.unlock(); }
 
-  i2s_port_t get_port() const { return this->port_; }
-
  protected:
   Mutex lock_;
 
@@ -106,7 +109,7 @@ class I2SAudioComponent : public Component {
   int bclk_pin_{I2S_GPIO_UNUSED};
 #endif
   int lrclk_pin_;
-  i2s_port_t port_{};
+  int port_{};
 };
 
 }  // namespace i2s_audio
diff --git a/esphome/components/i2s_audio/microphone/__init__.py b/esphome/components/i2s_audio/microphone/__init__.py
index bf583b9f81..aa15625cd7 100644
--- a/esphome/components/i2s_audio/microphone/__init__.py
+++ b/esphome/components/i2s_audio/microphone/__init__.py
@@ -13,9 +13,11 @@ from esphome.const import (
 )
 
 from .. import (
+    CONF_ADC_TYPE,
     CONF_I2S_DIN_PIN,
     CONF_LEFT,
     CONF_MONO,
+    CONF_PDM,
     CONF_RIGHT,
     I2SAudioIn,
     i2s_audio_component_schema,
@@ -29,9 +31,7 @@ CODEOWNERS = ["@jesserockz"]
 DEPENDENCIES = ["i2s_audio"]
 
 CONF_ADC_PIN = "adc_pin"
-CONF_ADC_TYPE = "adc_type"
 CONF_CORRECT_DC_OFFSET = "correct_dc_offset"
-CONF_PDM = "pdm"
 
 I2SAudioMicrophone = i2s_audio_ns.class_(
     "I2SAudioMicrophone", I2SAudioIn, microphone.Microphone, cg.Component
diff --git a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
index eb4506071e..a17cb0d84a 100644
--- a/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
+++ b/esphome/components/i2s_audio/microphone/i2s_audio_microphone.cpp
@@ -37,27 +37,6 @@ enum MicrophoneEventGroupBits : uint32_t {
 };
 
 void I2SAudioMicrophone::setup() {
-#ifdef USE_I2S_LEGACY
-#if SOC_I2S_SUPPORTS_ADC
-  if (this->adc_) {
-    if (this->parent_->get_port() != I2S_NUM_0) {
-      ESP_LOGE(TAG, "Internal ADC only works on I2S0");
-      this->mark_failed();
-      return;
-    }
-  } else
-#endif
-#endif
-  {
-    if (this->pdm_) {
-      if (this->parent_->get_port() != I2S_NUM_0) {
-        ESP_LOGE(TAG, "PDM only works on I2S0");
-        this->mark_failed();
-        return;
-      }
-    }
-  }
-
   this->active_listeners_semaphore_ = xSemaphoreCreateCounting(MAX_LISTENERS, MAX_LISTENERS);
   if (this->active_listeners_semaphore_ == nullptr) {
     ESP_LOGE(TAG, "Creating semaphore failed");