diff --git a/arch/xtensa/include/esp32s3/irq.h b/arch/xtensa/include/esp32s3/irq.h index 5419d49aaf4..25a248f422e 100644 --- a/arch/xtensa/include/esp32s3/irq.h +++ b/arch/xtensa/include/esp32s3/irq.h @@ -37,6 +37,16 @@ #define ESP32S3_INT_PRIO_DEF 1 +/* CPU interrupt flags: + * These flags can be used to specify which interrupt qualities the + * code calling esp32s3_setup_irq needs. + */ + +#define ESP32S3_CPUINT_FLAG_LEVEL (1 << 0) /* Level-triggered interrupt */ +#define ESP32S3_CPUINT_FLAG_EDGE (1 << 1) /* Edge-triggered interrupt */ +#define ESP32S3_CPUINT_FLAG_SHARED (1 << 2) /* Interrupt can be shared between ISRs */ +#define ESP32S3_CPUINT_FLAG_IRAM (1 << 3) /* ISR can be called if cache is disabled */ + /* Interrupt Matrix * * The Interrupt Matrix embedded in the ESP32-S3 independently allocates @@ -386,17 +396,13 @@ * 26 can be mapped to peripheral interrupts: * * Level triggered peripherals (21 total): - * 0-5, 8-9, 12-13, 17-18 - Priority 1 - * 19-21 - Priority 2 - * 23, 27 - Priority 3 - * 24-25 - Priority 4 - * 26, 31 - Priority 5 - * Edge triggered peripherals (4 total): - * 10 - Priority 1 - * 22 - Priority 3 - * 28, 30 - Priority 4 + * 0-5, 8-10, 12-13, 17-18 - Priority 1 + * 19-21 - Priority 2 + * 22-23, 27 - Priority 3 + * 24-25, 28, 30 - Priority 4 + * 26, 31 - Priority 5 * NMI (1 total): - * 14 - NMI + * 14 - NMI * * CPU peripheral interrupts can be a assigned to a CPU interrupt using the * PRO_*_MAP_REG or APP_*_MAP_REG. There are a pair of these registers for diff --git a/arch/xtensa/src/esp32s3/Kconfig b/arch/xtensa/src/esp32s3/Kconfig index 2f35146255c..0104cca61ec 100644 --- a/arch/xtensa/src/esp32s3/Kconfig +++ b/arch/xtensa/src/esp32s3/Kconfig @@ -734,6 +734,19 @@ menuconfig ESP32S3_WIFI_BT_COEXIST depends on ESP32S3_WIFI && ESP32S3_BLE select ESP32S3_WIFI_STA_DISCONNECT_PM +menu "Interrupt Configuration" + +config ESP32S3_IRAM_ISR_DEBUG + bool "Enable debugging of the IRAM-enabled interrupts" + default n + ---help--- + This option enables keeping track of the IRAM-enabled interrupts by + registering its execution when non-IRAM interrupts are disabled. It + keeps track of the IRQ executed and register how many times since + boot it was executed. + +endmenu # Interrupt Configuration + menu "SPI RAM Configuration" depends on ESP32S3_SPIRAM @@ -1605,6 +1618,17 @@ config ESP32S3_STORAGE_MTD_DEBUG If this option is enabled, Storage MTD driver read and write functions will output input parameters and return values (if applicable). +config ESP32S3_SPIFLASH_OP_TASK_STACKSIZE + int "The SPI flash operation task stack size" + default 768 + depends on SMP + ---help--- + When SMP is enabled, it is needed to create two tasks (one for each + core) to be able to run IRAM-enabled interrupts. These tasks ensures + that the core that isn't performing the SPI flash operation is able + to disable non-IRAM interrupts and wait for the SPI flash operation + to be finished. + endif # ESP32S3_SPIFLASH endmenu # SPI Flash configuration diff --git a/arch/xtensa/src/esp32s3/esp32s3_irq.c b/arch/xtensa/src/esp32s3/esp32s3_irq.c index 2afc13cad0c..5e3dca56a31 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_irq.c +++ b/arch/xtensa/src/esp32s3/esp32s3_irq.c @@ -33,8 +33,10 @@ #include #include #include +#include #include #include +#include #include "xtensa.h" @@ -161,6 +163,24 @@ static volatile uint8_t g_irqmap[NR_IRQS]; static uint32_t g_intenable[CONFIG_SMP_NCPUS]; +/* g_non_iram_int_mask[] is a bitmask of the interrupts that should be + * disabled during a SPI flash operation. Non-IRAM interrupts should always + * be disabled, but interrupts place on IRAM are able to run during a SPI + * flash operation. + */ + +static uint32_t g_non_iram_int_mask[CONFIG_SMP_NCPUS]; + +/* g_non_iram_int_disabled[] keeps track of the interrupts disabled during + * a SPI flash operation. + */ + +static uint32_t g_non_iram_int_disabled[CONFIG_SMP_NCPUS]; + +/* Per-CPU flag to indicate that non-IRAM interrupts were disabled */ + +static bool g_non_iram_int_disabled_flag[CONFIG_SMP_NCPUS]; + /* Bitsets for free, unallocated CPU interrupts available to peripheral * devices. */ @@ -184,6 +204,14 @@ static const uint32_t g_priority[5] = ESP32S3_INTPRI5_MASK }; +#ifdef CONFIG_ESP32S3_IRAM_ISR_DEBUG +/* The g_iram_count keeps track of how many times such an IRQ ran when the + * non-IRAM interrupts were disabled. + */ + +static uint64_t g_iram_count[NR_IRQS]; +#endif + /**************************************************************************** * Private Functions ****************************************************************************/ @@ -406,6 +434,33 @@ static void esp32s3_free_cpuint(int cpuint) *freeints |= bitmask; } +#ifdef CONFIG_ESP32S3_IRAM_ISR_DEBUG + +/**************************************************************************** + * Name: esp32s3_iram_interrupt_record + * + * Description: + * This function keeps track of the IRQs that ran when non-IRAM interrupts + * are disabled and enables debugging of the IRAM-enabled interrupts. + * + * Input Parameters: + * irq - The IRQ associated with a CPU interrupt + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void esp32s3_irq_iram_interrupt_record(int irq) +{ + irqstate_t flags = enter_critical_section(); + + g_iram_count[irq]++; + + leave_critical_section(flags); +} +#endif + /**************************************************************************** * Public Functions ****************************************************************************/ @@ -418,6 +473,15 @@ void up_irqinitialize(void) { int i; + /* All CPU ints are non-IRAM interrupts at the beginning and should be + * disabled during a SPI flash operation + */ + + for (i = 0; i < CONFIG_SMP_NCPUS; i++) + { + g_non_iram_int_mask[i] = UINT32_MAX; + } + for (i = 0; i < NR_IRQS; i++) { g_irqmap[i] = IRQ_UNMAPPED; @@ -588,6 +652,21 @@ void up_enable_irq(int irq) int cpu = IRQ_GETCPU(g_irqmap[irq]); + /* Check if the registered ISR for this IRQ is intended to be run from + * IRAM. If so, check if its interrupt handler is located in IRAM. + */ + + bool isr_in_iram = !((g_non_iram_int_mask[cpu] & (1 << cpuint)) > 0); + + xcpt_t handler = g_irqvector[irq].handler; + + if (isr_in_iram && handler && !esp32s3_ptr_iram(handler)) + { + irqerr("Interrupt handler isn't in IRAM (%08" PRIx32 ")", + (intptr_t)handler); + PANIC(); + } + DEBUGASSERT(cpu >= 0 && cpu < CONFIG_SMP_NCPUS); /* For peripheral interrupts, attach the interrupt to the peripheral; @@ -716,7 +795,7 @@ int esp32s3_cpuint_initialize(void) * ESP32S3_CPUINT_PROFILING 11 Not yet defined * ESP32S3_CPUINT_TIMER1 15 XTENSA_IRQ_TIMER1 1 * ESP32S3_CPUINT_TIMER2 16 XTENSA_IRQ_TIMER2 2 - * ESP32S2_CPUINT_SOFTWARE1 29 XTENSA_IRQ_SWINT 4 + * ESP32S3_CPUINT_SOFTWARE1 29 XTENSA_IRQ_SWINT 4 */ intmap[ESP32S3_CPUINT_TIMER0] = CPUINT_ASSIGN(XTENSA_IRQ_TIMER0); @@ -732,14 +811,16 @@ int esp32s3_cpuint_initialize(void) * * Description: * This function sets up the IRQ. It allocates a CPU interrupt of the given - * priority and type and attaches it to the given peripheral. + * priority and associated flags and attaches it to the given peripheral. * * Input Parameters: * cpu - The CPU to receive the interrupt 0=PRO CPU 1=APP CPU * periphid - The peripheral number from irq.h to be assigned to * a CPU interrupt. * priority - Interrupt's priority (1 - 5). - * type - Interrupt's type (level or edge). + * flags - An ORred mask of the ESP32S3_CPUINT_FLAG_* defines. These + * restrict the choice of interrupts that this routine can + * choose from. * * Returned Value: * The allocated CPU interrupt on success, a negated errno value on @@ -747,7 +828,7 @@ int esp32s3_cpuint_initialize(void) * ****************************************************************************/ -int esp32s3_setup_irq(int cpu, int periphid, int priority, int type) +int esp32s3_setup_irq(int cpu, int periphid, int priority, int flags) { irqstate_t irqstate; uintptr_t regaddr; @@ -755,19 +836,26 @@ int esp32s3_setup_irq(int cpu, int periphid, int priority, int type) int irq; int cpuint; + if ((flags & ESP32S3_CPUINT_EDGE) != 0) + { + irqerr("Only level-enabled interrupts are available"); + return -EINVAL; + } + irqstate = enter_critical_section(); /* Setting up an IRQ includes the following steps: * 1. Allocate a CPU interrupt. - * 2. Attach that CPU interrupt to the peripheral. - * 3. Map the CPU interrupt to the IRQ to ease searching later. + * 2. Check if its ISR is intended to run from IRAM. + * 3. Attach that CPU interrupt to the peripheral. + * 4. Map the CPU interrupt to the IRQ to ease searching later. */ - cpuint = esp32s3_alloc_cpuint(cpu, priority, type); + cpuint = esp32s3_alloc_cpuint(cpu, priority, ESP32S3_CPUINT_LEVEL); if (cpuint < 0) { - irqerr("Unable to allocate CPU interrupt for priority=%d and type=%d", - priority, type); + irqerr("Unable to allocate CPU interrupt for priority=%d and flags=%d", + priority, flags); leave_critical_section(irqstate); return cpuint; @@ -785,6 +873,15 @@ int esp32s3_setup_irq(int cpu, int periphid, int priority, int type) intmap[cpuint] = CPUINT_ASSIGN(periphid + XTENSA_IRQ_FIRSTPERIPH); g_irqmap[irq] = IRQ_MKMAP(cpu, cpuint); + if ((flags & ESP32S3_CPUINT_FLAG_IRAM) != 0) + { + esp32s3_irq_set_iram_isr(irq); + } + else + { + esp32s3_irq_unset_iram_isr(irq); + } + putreg32(cpuint, regaddr); leave_critical_section(irqstate); @@ -851,8 +948,8 @@ void esp32s3_teardown_irq(int cpu, int periphid, int cpuint) * This function returns the IRQ associated with a CPU interrupt * * Input Parameters: - * cpu - The CPU to receive the interrupt 0=PRO CPU 1=APP CPU - * cpuint - The CPU interrupt associated to the IRQ + * cpu - The CPU core of the IRQ being queried + * cpuint - The CPU interrupt associated to the IRQ * * Returned Value: * The IRQ associated with such CPU interrupt or CPUINT_UNASSIGNED if @@ -880,6 +977,29 @@ int esp32s3_getirq(int cpu, int cpuint) return CPUINT_GETIRQ(intmap[cpuint]); } +/**************************************************************************** + * Name: esp32s3_getcpuint_from_irq + * + * Description: + * This function returns the CPU interrupt associated with an IRQ + * + * Input Parameters: + * irq - The IRQ associated with a CPU interrupt + * cpu - Pointer to store the CPU core of the CPU interrupt + * + * Returned Value: + * The CPU interrupt associated with such IRQ or IRQ_UNMAPPED if + * CPU interrupt is not mapped to an IRQ. + * + ****************************************************************************/ + +int esp32s3_getcpuint_from_irq(int irq, int *cpu) +{ + (*cpu) = (int)IRQ_GETCPU(g_irqmap[irq]); + + return IRQ_GETCPUINT(g_irqmap[irq]); +} + /**************************************************************************** * Name: xtensa_int_decode * @@ -903,18 +1023,16 @@ uint32_t *xtensa_int_decode(uint32_t cpuints, uint32_t *regs) uint8_t *intmap; uint32_t mask; int bit; -#ifdef CONFIG_SMP int cpu; -#endif #ifdef CONFIG_ARCH_LEDS_CPU_ACTIVITY board_autoled_on(LED_CPU); #endif - -#ifdef CONFIG_SMP /* Select PRO or APP CPU interrupt mapping table */ cpu = up_cpu_index(); + +#ifdef CONFIG_SMP if (cpu != 0) { intmap = g_cpu1_intmap; @@ -945,6 +1063,17 @@ uint32_t *xtensa_int_decode(uint32_t cpuints, uint32_t *regs) DEBUGASSERT(CPUINT_GETEN(intmap[bit])); DEBUGASSERT(irq != CPUINT_UNASSIGNED); +#ifdef CONFIG_ESP32S3_IRAM_ISR_DEBUG + /* Check if non-IRAM interrupts are disabled */ + + if (esp32s3_irq_noniram_status(cpu) == 0) + { + /* Sum-up the IRAM-enabled counter associated with the IRQ */ + + esp32s3_irq_iram_interrupt_record(irq); + } +#endif + /* Clear software or edge-triggered interrupt */ xtensa_intclear(mask); @@ -965,6 +1094,185 @@ uint32_t *xtensa_int_decode(uint32_t cpuints, uint32_t *regs) } } + UNUSED(cpu); + return regs; } +/**************************************************************************** + * Name: esp32s3_irq_noniram_disable + * + * Description: + * Disable interrupts that aren't specifically marked as running from IRAM + * + * Input Parameters: + * None + * + * Input Parameters: + * None + * + ****************************************************************************/ + +void esp32s3_irq_noniram_disable(void) +{ + irqstate_t irqstate; + int cpu; + uint32_t oldint; + uint32_t non_iram_ints; + + irqstate = enter_critical_section(); + cpu = up_cpu_index(); + non_iram_ints = g_non_iram_int_mask[cpu]; + + ASSERT(!g_non_iram_int_disabled_flag[cpu]); + + g_non_iram_int_disabled_flag[cpu] = true; + oldint = g_intenable[cpu]; + + xtensa_disable_cpuint(&g_intenable[cpu], non_iram_ints); + + g_non_iram_int_disabled[cpu] = oldint & non_iram_ints; + + leave_critical_section(irqstate); +} + +/**************************************************************************** + * Name: esp32s3_irq_noniram_enable + * + * Description: + * Re-enable interrupts disabled by esp32s3_irq_noniram_disable + * + * Input Parameters: + * None + * + * Input Parameters: + * None + * + ****************************************************************************/ + +void esp32s3_irq_noniram_enable(void) +{ + irqstate_t irqstate; + int cpu; + uint32_t non_iram_ints; + + irqstate = enter_critical_section(); + cpu = up_cpu_index(); + non_iram_ints = g_non_iram_int_disabled[cpu]; + + ASSERT(g_non_iram_int_disabled_flag[cpu]); + + g_non_iram_int_disabled_flag[cpu] = false; + + xtensa_enable_cpuint(&g_intenable[cpu], non_iram_ints); + + leave_critical_section(irqstate); +} + +/**************************************************************************** + * Name: esp32s3_irq_noniram_status + * + * Description: + * Get the current status of non-IRAM interrupts on a specific CPU core + * + * Input Parameters: + * cpu - The CPU to check the non-IRAM interrupts state + * + * Returned Value: + * true if non-IRAM interrupts are enabled, false otherwise. + * + ****************************************************************************/ + +bool esp32s3_irq_noniram_status(int cpu) +{ + DEBUGASSERT(cpu >= 0 && cpu < CONFIG_SMP_NCPUS); + + return !g_non_iram_int_disabled_flag[cpu]; +} + +/**************************************************************************** + * Name: esp32s3_irq_set_iram_isr + * + * Description: + * Set the ISR associated to an IRQ as a IRAM-enabled ISR. + * + * Input Parameters: + * irq - The associated IRQ to set + * + * Returned Value: + * OK on success; A negated errno value on failure. + * + ****************************************************************************/ + +int esp32s3_irq_set_iram_isr(int irq) +{ + int cpu; + int cpuint = esp32s3_getcpuint_from_irq(irq, &cpu); + + if (cpuint == IRQ_UNMAPPED) + { + return -EINVAL; + } + + g_non_iram_int_mask[cpu] &= ~(1 << cpuint); + + return OK; +} + +/**************************************************************************** + * Name: esp32s3_irq_unset_iram_isr + * + * Description: + * Set the ISR associated to an IRQ as a non-IRAM ISR. + * + * Input Parameters: + * irq - The associated IRQ to set + * + * Returned Value: + * OK on success; A negated errno value on failure. + * + ****************************************************************************/ + +int esp32s3_irq_unset_iram_isr(int irq) +{ + int cpu; + int cpuint = esp32s3_getcpuint_from_irq(irq, &cpu); + + if (cpuint == IRQ_UNMAPPED) + { + return -EINVAL; + } + + g_non_iram_int_mask[cpu] |= (1 << cpuint); + + return OK; +} + +#ifdef CONFIG_ESP32S3_IRAM_ISR_DEBUG + +/**************************************************************************** + * Name: esp32s3_get_iram_interrupt_records + * + * Description: + * This function copies the vector that keeps track of the IRQs that ran + * when non-IRAM interrupts were disabled. + * + * Input Parameters: + * + * irq_count - A previously allocated pointer to store the counter of the + * interrupts that ran when non-IRAM interrupts were disabled. + * + * Returned Value: + * None + * + ****************************************************************************/ + +void esp32s3_get_iram_interrupt_records(uint64_t *irq_count) +{ + irqstate_t flags = enter_critical_section(); + + memcpy(irq_count, &g_iram_count, sizeof(uint64_t) * NR_IRQS); + + leave_critical_section(flags); +} +#endif diff --git a/arch/xtensa/src/esp32s3/esp32s3_irq.h b/arch/xtensa/src/esp32s3/esp32s3_irq.h index 6119cf73486..9a36b6813c6 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_irq.h +++ b/arch/xtensa/src/esp32s3/esp32s3_irq.h @@ -46,8 +46,8 @@ extern "C" /* CPU interrupt types. */ -#define ESP32S3_CPUINT_LEVEL 0 -#define ESP32S3_CPUINT_EDGE 1 +#define ESP32S3_CPUINT_LEVEL ESP32S3_CPUINT_FLAG_LEVEL +#define ESP32S3_CPUINT_EDGE ESP32S3_CPUINT_FLAG_EDGE /**************************************************************************** * Public Functions Prototypes @@ -82,7 +82,9 @@ int esp32s3_cpuint_initialize(void); * periphid - The peripheral number from irq.h to be assigned to * a CPU interrupt. * priority - Interrupt's priority (1 - 5). - * type - Interrupt's type (level or edge). + * flags - An ORred mask of the ESP32S3_CPUINT_FLAG_* defines. These + * restrict the choice of interrupts that this routine can + * choose from. * * Returned Value: * The allocated CPU interrupt on success, a negated errno value on @@ -90,7 +92,7 @@ int esp32s3_cpuint_initialize(void); * ****************************************************************************/ -int esp32s3_setup_irq(int cpu, int periphid, int priority, int type); +int esp32s3_setup_irq(int cpu, int periphid, int priority, int flags); /**************************************************************************** * Name: esp32s3_teardown_irq @@ -121,8 +123,8 @@ void esp32s3_teardown_irq(int cpu, int periphid, int cpuint); * This function returns the IRQ associated with a CPU interrupt * * Input Parameters: - * cpu - The CPU to receive the interrupt 0=PRO CPU 1=APP CPU - * cpuint - The CPU interrupt associated to the IRQ + * cpu - The CPU core of the IRQ being queried + * cpuint - The CPU interrupt associated to the IRQ * * Returned Value: * The IRQ associated with such CPU interrupt or CPUINT_UNASSIGNED if @@ -132,6 +134,126 @@ void esp32s3_teardown_irq(int cpu, int periphid, int cpuint); int esp32s3_getirq(int cpu, int cpuint); +/**************************************************************************** + * Name: esp32s3_getcpuint_from_irq + * + * Description: + * This function returns the CPU interrupt associated with an IRQ + * + * Input Parameters: + * irq - The IRQ associated with a CPU interrupt + * cpu - Pointer to store the CPU core of the CPU interrupt + * + * Returned Value: + * The CPU interrupt associated with such IRQ or IRQ_UNMAPPED if + * CPU interrupt is not mapped to an IRQ. + * + ****************************************************************************/ + +int esp32s3_getcpuint_from_irq(int irq, int *cpu); + +/**************************************************************************** + * Name: esp32s3_irq_noniram_disable + * + * Description: + * Disable interrupts that aren't specifically marked as running from IRAM + * + * Input Parameters: + * None + * + * Input Parameters: + * None + * + ****************************************************************************/ + +void esp32s3_irq_noniram_disable(void); + +/**************************************************************************** + * Name: esp32s3_irq_noniram_enable + * + * Description: + * Re-enable interrupts disabled by esp32s3_irq_noniram_disable + * + * Input Parameters: + * None + * + * Input Parameters: + * None + * + ****************************************************************************/ + +void esp32s3_irq_noniram_enable(void); + +/**************************************************************************** + * Name: esp32s3_irq_noniram_status + * + * Description: + * Get the current status of non-IRAM interrupts on a specific CPU core + * + * Input Parameters: + * cpu - The CPU to check the non-IRAM interrupts state + * + * Returned Value: + * true if non-IRAM interrupts are enabled, false otherwise. + * + ****************************************************************************/ + +bool esp32s3_irq_noniram_status(int cpu); + +/**************************************************************************** + * Name: esp32s3_irq_set_iram_isr + * + * Description: + * Set the ISR associated to an IRQ as a IRAM-enabled ISR. + * + * Input Parameters: + * irq - The associated IRQ to set + * + * Returned Value: + * OK on success; A negated errno value on failure. + * + ****************************************************************************/ + +int esp32s3_irq_set_iram_isr(int irq); + +/**************************************************************************** + * Name: esp32s3_irq_unset_iram_isr + * + * Description: + * Set the ISR associated to an IRQ as a non-IRAM ISR. + * + * Input Parameters: + * irq - The associated IRQ to set + * + * Returned Value: + * OK on success; A negated errno value on failure. + * + ****************************************************************************/ + +int esp32s3_irq_unset_iram_isr(int irq); + +#ifdef CONFIG_ESP32S3_IRAM_ISR_DEBUG + +/**************************************************************************** + * Name: esp32s3_get_iram_interrupt_records + * + * Description: + * This function copies the vector that keeps track of the IRQs that ran + * when non-IRAM interrupts were disabled. + * + * Input Parameters: + * + * irq_count - A previously allocated pointer to store the counter of the + * interrupts that ran when non-IRAM interrupts were disabled. + * + * Returned Value: + * None + * + ****************************************************************************/ + +void esp32s3_get_iram_interrupt_records(uint64_t *irq_count); +#endif + #undef EXTERN #if defined(__cplusplus) } diff --git a/arch/xtensa/src/esp32s3/esp32s3_spiflash.c b/arch/xtensa/src/esp32s3/esp32s3_spiflash.c index bf4e468a316..837acedb9c8 100644 --- a/arch/xtensa/src/esp32s3/esp32s3_spiflash.c +++ b/arch/xtensa/src/esp32s3/esp32s3_spiflash.c @@ -35,6 +35,9 @@ #include #include +#include + +#include "sched/sched.h" #include "xtensa.h" #include "xtensa_attr.h" @@ -155,7 +158,7 @@ buffer, size, \ NULL, 0, \ 0, \ - true) + true) # define READ_DATA_FROM_FLASH(addr, buffer, size) \ esp32s3_spi_trans(READ_CMD(addr), 8, \ @@ -163,7 +166,7 @@ NULL, 0, \ buffer, size, \ READ_DUMMY(addr), \ - false) + false) #endif /* CONFIG_ESP32S3_SPI_FLASH_DONT_USE_ROM_CODE */ @@ -196,22 +199,17 @@ struct spiflash_map_req_s uint32_t page_cnt; }; -struct spiflash_cachestate_s -{ - uint32_t value; - irqstate_t flags; - int cpu; -#ifdef CONFIG_SMP - int other; -#endif -}; - /**************************************************************************** * Private Functions Declaration ****************************************************************************/ static void spiflash_start(void); static void spiflash_end(void); +static void spi_flash_disable_cache(uint32_t cpuid); +static void spi_flash_restore_cache(uint32_t cpuid); +#ifdef CONFIG_SMP +static int spiflash_init_spi_flash_op_block_task(int cpu); +#endif /**************************************************************************** * Public Functions Declaration @@ -237,60 +235,130 @@ static struct spiflash_guard_funcs g_spi_flash_guard_funcs = .yield = NULL, }; -static struct spiflash_cachestate_s g_state; +static uint32_t s_flash_op_cache_state[CONFIG_SMP_NCPUS]; + +static spinlock_t g_flash_op_lock; +static rmutex_t g_flash_op_mutex; +static volatile bool g_flash_op_can_start = false; +static volatile bool g_flash_op_complete = false; +static volatile bool g_sched_suspended[CONFIG_SMP_NCPUS]; +#ifdef CONFIG_SMP +static sem_t g_disable_non_iram_isr_on_core[CONFIG_SMP_NCPUS]; +#endif /**************************************************************************** - * Name: spiflash_opstart + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: spiflash_start * * Description: - * Prepare for an SPIFLASH operation. + * Prepare for an SPI flash operation. * ****************************************************************************/ -static IRAM_ATTR void spiflash_start(void) +static void spiflash_start(void) { - g_state.flags = enter_critical_section(); - g_state.cpu = up_cpu_index(); + struct tcb_s *tcb = this_task(); + int cpu = up_cpu_index(); + int saved_priority = tcb->sched_priority; #ifdef CONFIG_SMP - g_state.other = g_state.cpu ? 0 : 1; + int other_cpu = cpu ? 0 : 1; #endif - DEBUGASSERT(g_state.cpu == 0 || g_state.cpu == 1); + nxrmutex_lock(&g_flash_op_mutex); + + DEBUGASSERT(cpu == 0 || cpu == 1); + + nxsched_set_priority(tcb, SCHED_PRIORITY_MAX); + #ifdef CONFIG_SMP - DEBUGASSERT(g_state.other == 0 || g_state.other == 1); - DEBUGASSERT(g_state.other != g_state.cpu); - up_cpu_pause(g_state.other); + + /* Temporary raise schedule priority */ + + DEBUGASSERT(other_cpu == 0 || other_cpu == 1); + DEBUGASSERT(other_cpu != cpu); + if (OSINIT_OS_READY()) + { + g_flash_op_can_start = false; + + cpu = up_cpu_index(); + other_cpu = cpu ? 0 : 1; + + nxsem_post(&g_disable_non_iram_isr_on_core[other_cpu]); + + while (!g_flash_op_can_start) + { + /* Busy loop and wait for spi_flash_op_block_task to disable cache + * on the other CPU + */ + } + } #endif - g_state.value = cache_suspend_icache() << 16; - g_state.value |= cache_suspend_dcache(); + g_sched_suspended[cpu] = true; + + sched_lock(); + + nxsched_set_priority(tcb, saved_priority); + + esp32s3_irq_noniram_disable(); + + spi_flash_disable_cache(cpu); +#ifdef CONFIG_SMP + spi_flash_disable_cache(other_cpu); +#endif } /**************************************************************************** - * Name: spiflash_opdone + * Name: spiflash_end * * Description: - * Undo all the steps of opstart. + * Undo all the steps of spiflash_start. * ****************************************************************************/ -static IRAM_ATTR void spiflash_end(void) +static void spiflash_end(void) { - DEBUGASSERT(g_state.cpu == 0 || g_state.cpu == 1); - + struct tcb_s *tcb = this_task(); + const int cpu = up_cpu_index(); #ifdef CONFIG_SMP - DEBUGASSERT(g_state.other == 0 || g_state.other == 1); - DEBUGASSERT(g_state.other != g_state.cpu); + const int other_cpu = cpu ? 0 : 1; #endif - cache_resume_icache(g_state.value >> 16); - cache_resume_dcache(g_state.value & 0xffff); + DEBUGASSERT(cpu == 0 || cpu == 1); #ifdef CONFIG_SMP - up_cpu_resume(g_state.other); + DEBUGASSERT(other_cpu == 0 || other_cpu == 1); + DEBUGASSERT(other_cpu != cpu); #endif - leave_critical_section(g_state.flags); + spi_flash_restore_cache(cpu); +#ifdef CONFIG_SMP + spi_flash_restore_cache(other_cpu); +#endif + + /* Signal to spi_flash_op_block_task that flash operation is complete */ + + g_flash_op_complete = true; + + esp32s3_irq_noniram_enable(); + + sched_unlock(); + + g_sched_suspended[cpu] = false; + +#ifdef CONFIG_SMP + while (g_sched_suspended[other_cpu]) + { + /* Busy loop and wait for spi_flash_op_block_task to properly finish + * and resume scheduler + */ + } +#endif + + nxrmutex_unlock(&g_flash_op_mutex); } /**************************************************************************** @@ -317,16 +385,16 @@ static IRAM_ATTR void spiflash_end(void) ****************************************************************************/ #ifdef CONFIG_ESP32S3_SPI_FLASH_DONT_USE_ROM_CODE -static IRAM_ATTR void esp32s3_spi_trans(uint32_t command, - uint32_t command_bits, - uint32_t address, - uint32_t address_bits, - uint32_t *tx_buffer, - uint32_t tx_bytes, - uint32_t *rx_buffer, - uint32_t rx_bytes, - uint32_t dummy_bits, - bool is_program) +static void esp32s3_spi_trans(uint32_t command, + uint32_t command_bits, + uint32_t address, + uint32_t address_bits, + uint32_t *tx_buffer, + uint32_t tx_bytes, + uint32_t *rx_buffer, + uint32_t rx_bytes, + uint32_t dummy_bits, + bool is_program) { uint32_t regval; uint32_t cmd_reg; @@ -475,7 +543,7 @@ static IRAM_ATTR void esp32s3_spi_trans(uint32_t command, * ****************************************************************************/ -static IRAM_ATTR void wait_flash_idle(void) +static void wait_flash_idle(void) { uint32_t status; @@ -501,7 +569,7 @@ static IRAM_ATTR void wait_flash_idle(void) * ****************************************************************************/ -static IRAM_ATTR void enable_flash_write(void) +static void enable_flash_write(void) { uint32_t status; @@ -528,7 +596,7 @@ static IRAM_ATTR void enable_flash_write(void) * ****************************************************************************/ -static IRAM_ATTR void disable_flash_write(void) +static void disable_flash_write(void) { uint32_t status; @@ -563,7 +631,7 @@ static IRAM_ATTR void disable_flash_write(void) * ****************************************************************************/ -static IRAM_ATTR int esp32s3_mmap(struct spiflash_map_req_s *req) +static int esp32s3_mmap(struct spiflash_map_req_s *req) { int ret; int i; @@ -629,7 +697,7 @@ static IRAM_ATTR int esp32s3_mmap(struct spiflash_map_req_s *req) * ****************************************************************************/ -static IRAM_ATTR void esp32s3_ummap(const struct spiflash_map_req_s *req) +static void esp32s3_ummap(const struct spiflash_map_req_s *req) { int i; @@ -701,7 +769,7 @@ int spi_flash_read_encrypted(uint32_t addr, void *buffer, uint32_t size) ****************************************************************************/ #ifdef CONFIG_ESP32S3_SPI_FLASH_DONT_USE_ROM_CODE -IRAM_ATTR int spi_flash_erase_sector(uint32_t sector) +int spi_flash_erase_sector(uint32_t sector) { int ret = OK; uint32_t addr = sector * FLASH_SECTOR_SIZE; @@ -738,7 +806,7 @@ IRAM_ATTR int spi_flash_erase_sector(uint32_t sector) * ****************************************************************************/ -IRAM_ATTR int spi_flash_erase_range(uint32_t start_address, uint32_t size) +int spi_flash_erase_range(uint32_t start_address, uint32_t size) { int ret = OK; uint32_t addr = start_address; @@ -778,9 +846,7 @@ IRAM_ATTR int spi_flash_erase_range(uint32_t start_address, uint32_t size) * ****************************************************************************/ -IRAM_ATTR int spi_flash_write(uint32_t dest_addr, - const void *buffer, - uint32_t size) +int spi_flash_write(uint32_t dest_addr, const void *buffer, uint32_t size) { int ret = OK; const uint8_t *tx_buf = (const uint8_t *)buffer; @@ -830,7 +896,7 @@ IRAM_ATTR int spi_flash_write(uint32_t dest_addr, * ****************************************************************************/ -IRAM_ATTR int spi_flash_read(uint32_t src_addr, void *dest, uint32_t size) +int spi_flash_read(uint32_t src_addr, void *dest, uint32_t size) { int ret = OK; uint8_t *rx_buf = (uint8_t *)dest; @@ -858,6 +924,177 @@ IRAM_ATTR int spi_flash_read(uint32_t src_addr, void *dest, uint32_t size) } #endif /* CONFIG_ESP32S3_SPI_FLASH_DONT_USE_ROM_CODE */ +/**************************************************************************** + * Name: spi_flash_disable_cache + * + * Description: + * Disable the I/D cache of a CPU core and save its status value on + * s_flash_op_cache_state. + * + * Input Parameters: + * cpuid - The CPU core whose cache will be disabled. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +static void spi_flash_disable_cache(uint32_t cpuid) +{ + s_flash_op_cache_state[cpuid] = cache_suspend_icache() << 16; + s_flash_op_cache_state[cpuid] |= cache_suspend_dcache(); +} + +/**************************************************************************** + * Name: spi_flash_restore_cache + * + * Description: + * Restore the I/D cache of a CPU core using the saved status value on + * s_flash_op_cache_state. + * + * Input Parameters: + * cpuid - The CPU core whose cache will be restored. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +static void spi_flash_restore_cache(uint32_t cpuid) +{ + cache_resume_dcache(s_flash_op_cache_state[cpuid] & 0xffff); + cache_resume_icache(s_flash_op_cache_state[cpuid] >> 16); +} + +#ifdef CONFIG_SMP + +/**************************************************************************** + * Name: spi_flash_op_block_task + * + * Description: + * Disable the non-IRAM interrupts on the other core (the one that isn't + * handling the SPI flash operation) and notify that the SPI flash + * operation can start. Wait on a busy loop until it's finished and then + * reenable the non-IRAM interrups. + * + * Input Parameters: + * argc - Not used. + * argv - Not used. + * + * Returned Value: + * Zero (OK) is returned on success. A negated errno value is returned to + * indicate the nature of any failure. + * + ****************************************************************************/ + +static int spi_flash_op_block_task(int argc, char *argv[]) +{ + struct tcb_s *tcb = this_task(); + int cpu = up_cpu_index(); + + for (; ; ) + { + DEBUGASSERT((1 << cpu) & tcb->affinity); + /* Wait for a SPI flash operation to take place and this (the other + * core) being asked to disable its non-IRAM interrupts. + */ + + nxsem_wait(&g_disable_non_iram_isr_on_core[cpu]); + + sched_lock(); + + esp32s3_irq_noniram_disable(); + + /* g_flash_op_complete flag is cleared on *this* CPU, otherwise the + * other CPU may reset the flag back to false before this task has a + * chance to check it (if it's preempted by an ISR taking non-trivial + * amount of time). + */ + + g_flash_op_complete = false; + g_flash_op_can_start = true; + while (!g_flash_op_complete) + { + /* Busy loop here and wait for the other CPU to finish the SPI + * flash operation. + */ + } + + /* Flash operation is complete, re-enable cache */ + + spi_flash_restore_cache(cpu); + + /* Restore interrupts that aren't located in IRAM */ + + esp32s3_irq_noniram_enable(); + + sched_unlock(); + } + + return OK; +} + +/**************************************************************************** + * Name: spiflash_init_spi_flash_op_block_task + * + * Description: + * Starts a kernel thread that waits for a semaphore indicating that a SPI + * flash operation is going to take place in the other CPU. It disables + * non-IRAM interrupts, indicates to the other core that the SPI flash + * operation can start and waits for it to be finished in a busy loop. + * + * Input Parameters: + * cpu - The CPU core that will run the created task to wait on a busy + * loop while the SPI flash operation finishes + * + * Returned Value: + * 0 (OK) on success; A negated errno value on failure. + * + ****************************************************************************/ + +int spiflash_init_spi_flash_op_block_task(int cpu) +{ + int pid; + int ret = OK; + char *argv[2]; + char arg1[32]; + cpu_set_t cpuset; + + snprintf(arg1, sizeof(arg1), "%p", &cpu); + argv[0] = arg1; + argv[1] = NULL; + + pid = kthread_create("spiflash_op", + SCHED_PRIORITY_MAX, + CONFIG_ESP32S3_SPIFLASH_OP_TASK_STACKSIZE, + spi_flash_op_block_task, + argv); + if (pid > 0) + { + if (cpu < CONFIG_SMP_NCPUS) + { + CPU_ZERO(&cpuset); + CPU_SET(cpu, &cpuset); + ret = nxsched_set_affinity(pid, sizeof(cpuset), &cpuset); + if (ret < 0) + { + return ret; + } + } + } + else + { + return -EPERM; + } + + return ret; +} +#endif /* CONFIG_SMP */ + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + /**************************************************************************** * Name: esp32s3_spiflash_init * @@ -871,7 +1108,33 @@ IRAM_ATTR int spi_flash_read(uint32_t src_addr, void *dest, uint32_t size) int esp32s3_spiflash_init(void) { + int cpu; + int ret = OK; + + /* Initializes SPI flash operation lock */ + + nxrmutex_init(&g_flash_op_mutex); + +#ifdef CONFIG_SMP + sched_lock(); + + for (cpu = 0; cpu < CONFIG_SMP_NCPUS; cpu++) + { + nxsem_init(&g_disable_non_iram_isr_on_core[cpu], 0, 0); + + ret = spiflash_init_spi_flash_op_block_task(cpu); + if (ret != OK) + { + return ret; + } + } + + sched_unlock(); +#else + UNUSED(cpu); +#endif + spi_flash_guard_set(&g_spi_flash_guard_funcs); - return OK; + return ret; } diff --git a/arch/xtensa/src/esp32s3/hardware/esp32s3_soc.h b/arch/xtensa/src/esp32s3/hardware/esp32s3_soc.h index a74513e4bf0..160cd27b88d 100644 --- a/arch/xtensa/src/esp32s3/hardware/esp32s3_soc.h +++ b/arch/xtensa/src/esp32s3/hardware/esp32s3_soc.h @@ -521,6 +521,19 @@ static inline bool IRAM_ATTR esp32s3_ptr_extram(const void *p) (intptr_t)p < SOC_EXTRAM_DATA_HIGH); } +/**************************************************************************** + * Name: esp32s3_ptr_iram + * + * Description: + * Check if the pointer is in IRAM + * + ****************************************************************************/ + +static inline bool IRAM_ATTR esp32s3_ptr_iram(const void *p) +{ + return ((intptr_t)p >= SOC_IRAM_LOW && (intptr_t)p < SOC_IRAM_HIGH); +} + /**************************************************************************** * Name: esp32s3_ptr_exec * diff --git a/boards/xtensa/esp32s3/common/scripts/legacy_sections.ld b/boards/xtensa/esp32s3/common/scripts/legacy_sections.ld index 8745dc15952..41d56c8129b 100644 --- a/boards/xtensa/esp32s3/common/scripts/legacy_sections.ld +++ b/boards/xtensa/esp32s3/common/scripts/legacy_sections.ld @@ -76,24 +76,30 @@ SECTIONS *(.iram1 .iram1.*) + *libarch.a:esp32s3_cpuindex.*(.literal .text .literal.* .text.*) + *libarch.a:esp32s3_irq.*(.literal .text .literal.* .text.*) *libarch.a:esp32s3_spiflash.*(.literal .text .literal.* .text.*) + *libarch.a:xtensa_assert.*(.literal .text .literal.* .text.*) + *libarch.a:xtensa_cpuint.*(.literal .text .literal.* .text.*) *libarch.a:xtensa_cpupause.*(.literal .text .literal.* .text.*) + *libarch.a:xtensa_irqdispatch.*(.literal .text .literal.* .text.*) + *libarch.a:xtensa_modifyreg32.*(.literal .text .literal.* .text.*) *libarch.a:xtensa_testset.*(.literal .text .literal.* .text.*) - *libarch.a:xtensa_modifyreg32.*(.literal.modifyreg32 .text.modifyreg32) + *libdrivers.a:syslog_flush.*(.literal .text .literal.* .text.*) + + *libsched.a:assert.*(.literal .text .literal.* .text.*) *libsched.a:irq_csection.*(.literal .text .literal.* .text.*) *libsched.a:irq_dispatch.*(.literal .text .literal.* .text.*) + *libsched.a:irq_spinlock.*(.literal .text .literal.* .text.*) *libsched.a:sched_note.*(.literal .text .literal.* .text.*) *libsched.a:sched_suspendscheduler.*(.literal .text .literal.* .text.*) *libsched.a:sched_thistask.*(.literal .text .literal.* .text.*) *libsched.a:spinlock.*(.literal .text .literal.* .text.*) #ifdef CONFIG_ESP32S3_SPEED_UP_ISR - *libarch.a:xtensa_irqdispatch.*(.literal.xtensa_irq_dispatch .text.xtensa_irq_dispatch) *libarch.a:xtensa_switchcontext.*(.literal.up_switch_context .text.up_switch_context) - *libarch.a:xtensa_modifyreg32.*(.literal.modifyreg32 .text.modifyreg32) - *libarch.a:esp32s3_irq.*(.literal.xtensa_int_decode .text.xtensa_int_decode) *libarch.a:esp32s3_timerisr.*(.literal.systimer_isr .text.systimer_isr) *libarch.a:esp32s3_idle.*(.literal.up_idle .text.up_idle) *libarch.a:esp32s3_dma.*(.literal.esp32s3_dma_load .text.esp32s3_dma_load \ @@ -195,6 +201,7 @@ SECTIONS *(.dram1 .dram1.*) *libphy.a:(.rodata .rodata.*) + *libarch.a:xtensa_context.*(.rodata .rodata.*) _edata = ABSOLUTE(.); . = ALIGN(4);