irq: add [enter|leave]_critical_section_nonirq

Configuring NuttX and compile:
$ ./tools/configure.sh -l qemu-armv8a:nsh_smp
$ make
Running with qemu
$ qemu-system-aarch64 -cpu cortex-a53 -smp 4 -nographic \
   -machine virt,virtualization=on,gic-version=3 \
   -net none -chardev stdio,id=con,mux=on -serial chardev:con \
   -mon chardev=con,mode=readline -kernel ./nuttx

reason:
In some non-irq scenarios, we can simplify
the implementation of critical sections to improve performance.

Signed-off-by: hujun5 <hujun5@xiaomi.com>

include/nuttx/irq.h

@@ -258,8 +258,10 @@ int irqchain_detach(int irq, xcpt_t isr, FAR void *arg);
  ****************************************************************************/
 
 #ifdef CONFIG_IRQCOUNT
+irqstate_t enter_critical_section_nonirq(void) noinstrument_function;
 irqstate_t enter_critical_section(void) noinstrument_function;
 #else
+#  define enter_critical_section_nonirq() up_irq_save()
 #  define enter_critical_section() up_irq_save()
 #endif
 
@@ -288,8 +290,10 @@ irqstate_t enter_critical_section(void) noinstrument_function;
  ****************************************************************************/
 
 #ifdef CONFIG_IRQCOUNT
+void leave_critical_section_nonirq(irqstate_t flags) noinstrument_function;
 void leave_critical_section(irqstate_t flags) noinstrument_function;
 #else
+#  define leave_critical_section_nonirq(f) up_irq_restore(f)
 #  define leave_critical_section(f) up_irq_restore(f)
 #endif
 

sched/irq/irq_csection.c

@@ -180,24 +180,13 @@ static inline_function bool irq_waitlock(int cpu)
 #ifdef CONFIG_SMP
 irqstate_t enter_critical_section(void)
 {
-  FAR struct tcb_s *rtcb;
-  irqstate_t ret;
   int cpu;
 
-  /* Disable interrupts.
+  /* Verify that the system has sufficiently initialized so that the task
-   *
+   * lists are valid.
-   * NOTE 1: Ideally this should disable interrupts on all CPUs, but most
-   * architectures only support disabling interrupts on the local CPU.
-   * NOTE 2: Interrupts may already be disabled, but we call up_irq_save()
-   * unconditionally because we need to return valid interrupt status in any
-   * event.
-   * NOTE 3: We disable local interrupts BEFORE taking the spinlock in order
-   * to prevent possible waits on the spinlock from interrupt handling on
-   * the local CPU.
    */
 
-try_again:
+  DEBUGASSERT(g_nx_initstate >= OSINIT_TASKLISTS);
-  ret = up_irq_save();
 
   /* If called from an interrupt handler, then just take the spinlock.
    * If we are already in a critical section, this will lock the CPU
@@ -312,9 +301,43 @@ try_again_in_irq:
           DEBUGASSERT(spin_is_locked(&g_cpu_irqlock) &&
                       (g_cpu_irqset & (1 << cpu)) != 0);
         }
+
+      return 0;
     }
   else
     {
+      return enter_critical_section_nonirq();
+    }
+}
+
+inline_function irqstate_t enter_critical_section_nonirq(void)
+{
+  FAR struct tcb_s *rtcb;
+  irqstate_t ret;
+  int cpu;
+
+  /* Disable interrupts.
+   *
+   * NOTE 1: Ideally this should disable interrupts on all CPUs, but most
+   * architectures only support disabling interrupts on the local CPU.
+   * NOTE 2: Interrupts may already be disabled, but we call up_irq_save()
+   * unconditionally because we need to return valid interrupt status in any
+   * event.
+   * NOTE 3: We disable local interrupts BEFORE taking the spinlock in order
+   * to prevent possible waits on the spinlock from interrupt handling on
+   * the local CPU.
+   */
+
+try_again:
+  ret = up_irq_save();
+
+  /* Verify that the system has sufficiently initialized so that the task
+   * lists are valid.
+   */
+
+  DEBUGASSERT(g_nx_initstate >= OSINIT_TASKLISTS);
+  DEBUGASSERT(!up_interrupt_context());
+
   /* Normal tasking environment.
    *
    * Get the TCB of the currently executing task on this CPU (avoid
@@ -387,7 +410,6 @@ try_again_in_irq:
       sched_note_csection(rtcb, true);
 #endif
     }
-    }
 
   /* Return interrupt status */
 
@@ -398,17 +420,41 @@ try_again_in_irq:
 
 irqstate_t enter_critical_section(void)
 {
+  /* Verify that the system has sufficiently initialized so that the task
+   * lists are valid.
+   */
+
+  DEBUGASSERT(g_nx_initstate >= OSINIT_TASKLISTS);
+
+  /* Check if we were called from an interrupt handler */
+
+  if (!up_interrupt_context())
+    {
+      return enter_critical_section_nonirq();
+    }
+
+  /* Return interrupt status */
+
+  return 0;
+}
+
+inline_function irqstate_t enter_critical_section_nonirq(void)
+{
+  FAR struct tcb_s *rtcb;
   irqstate_t ret;
 
   /* Disable interrupts */
 
   ret = up_irq_save();
 
-  /* Check if we were called from an interrupt handler */
+  /* Verify that the system has sufficiently initialized so that the task
+   * lists are valid.
+   */
 
-  if (!up_interrupt_context())
+  DEBUGASSERT(g_nx_initstate >= OSINIT_TASKLISTS);
-    {
+  DEBUGASSERT(!up_interrupt_context());
-      FAR struct tcb_s *rtcb = this_task();
+
+  rtcb = this_task();
   DEBUGASSERT(rtcb != NULL);
 
   /* Have we just entered the critical section?  Or is this a nested
@@ -418,8 +464,6 @@ irqstate_t enter_critical_section(void)
   DEBUGASSERT(rtcb->irqcount >= 0 && rtcb->irqcount < INT16_MAX);
   if (++rtcb->irqcount == 1)
     {
-          /* Note that we have entered the critical section */
-
 #ifdef CONFIG_SCHED_CRITMONITOR
       nxsched_critmon_csection(rtcb, true);
 #endif
@@ -427,7 +471,6 @@ irqstate_t enter_critical_section(void)
       sched_note_csection(rtcb, true);
 #endif
     }
-    }
 
   /* Return interrupt status */
 
@@ -447,6 +490,7 @@ irqstate_t enter_critical_section(void)
 #ifdef CONFIG_SMP
 void leave_critical_section(irqstate_t flags)
 {
+  FAR struct tcb_s *rtcb;
   int cpu;
 
   /* If called from an interrupt handler, then just release the
@@ -479,7 +523,7 @@ void leave_critical_section(irqstate_t flags)
           DEBUGASSERT(spin_is_locked(&g_cpu_irqlock) &&
                       g_cpu_nestcount[cpu] == 1);
 
-          FAR struct tcb_s *rtcb = current_task(cpu);
+          rtcb = current_task(cpu);
           DEBUGASSERT(rtcb != NULL);
           DEBUGASSERT((g_cpu_irqset & (1 << cpu)) != 0);
 
@@ -492,8 +536,18 @@ void leave_critical_section(irqstate_t flags)
         }
     }
   else
+    {
+      leave_critical_section_nonirq(flags);
+    }
+}
+
+inline_function void leave_critical_section_nonirq(irqstate_t flags)
 {
   FAR struct tcb_s *rtcb;
+  int cpu;
+
+  DEBUGASSERT(g_nx_initstate >= OSINIT_TASKLISTS);
+  DEBUGASSERT(!up_interrupt_context());
 
   /* Get the TCB of the currently executing task on this CPU (avoid
    * using this_task() which can recurse.
@@ -544,11 +598,6 @@ void leave_critical_section(irqstate_t flags)
 
       /* Have all CPUs released the lock? */
     }
-    }
-
-  /* Restore the previous interrupt state which may still be interrupts
-   * disabled (but we don't have a mechanism to verify that now)
-   */
 
   up_irq_restore(flags);
 }
@@ -562,8 +611,17 @@ void leave_critical_section(irqstate_t flags)
    */
 
   if (!up_interrupt_context())
+    {
+      leave_critical_section_nonirq(flags);
+    }
+}
+
+inline_function void leave_critical_section_nonirq(irqstate_t flags)
 {
   FAR struct tcb_s *rtcb = this_task();
+
+  DEBUGASSERT(g_nx_initstate >= OSINIT_TASKLISTS);
+  DEBUGASSERT(!up_interrupt_context());
   DEBUGASSERT(rtcb != NULL);
 
   /* Have we left entered the critical section?  Or are we still
@@ -582,9 +640,6 @@ void leave_critical_section(irqstate_t flags)
       sched_note_csection(rtcb, false);
 #endif
     }
-    }
-
-  /* Restore the previous interrupt state. */
 
   up_irq_restore(flags);
 }

sched/mqueue/mq_receive.c

@@ -96,7 +96,7 @@ ssize_t file_mq_receive(FAR struct file *mq, FAR char *msg, size_t msglen,
    * because messages can be sent from interrupt level.
    */
 
-  flags = enter_critical_section();
+  flags = enter_critical_section_nonirq();
 
   /* Get the message from the message queue */
 
@@ -114,7 +114,7 @@ ssize_t file_mq_receive(FAR struct file *mq, FAR char *msg, size_t msglen,
       ret = nxmq_do_receive(msgq, mqmsg, msg, prio);
     }
 
-  leave_critical_section(flags);
+  leave_critical_section_nonirq(flags);
 
   return ret;
 }

sched/mqueue/mq_timedreceive.c

@@ -156,7 +156,7 @@ file_mq_timedreceive_internal(FAR struct file *mq, FAR char *msg,
    * because messages can be sent from interrupt level.
    */
 
-  flags = enter_critical_section();
+  flags = enter_critical_section_nonirq();
 
   /* Check if the message queue is empty.  If it is NOT empty, then we
    * will not need to start timer.
@@ -231,7 +231,7 @@ file_mq_timedreceive_internal(FAR struct file *mq, FAR char *msg,
   /* We can now restore interrupts */
 
 errout_in_critical_section:
-  leave_critical_section(flags);
+  leave_critical_section_nonirq(flags);
 
   return ret;
 }

sched/mqueue/mq_timedsend.c

@@ -144,8 +144,6 @@ file_mq_timedsend_internal(FAR struct file *mq, FAR const char *msg,
   irqstate_t flags;
   int ret;
 
-  DEBUGASSERT(up_interrupt_context() == false);
-
   /* Verify the input parameters on any failures to verify. */
 
   ret = nxmq_verify_send(mq, msg, msglen, prio);

sched/semaphore/sem_clockwait.c

@@ -107,7 +107,7 @@ int nxsem_clockwait(FAR sem_t *sem, clockid_t clockid,
    * enabled while we are blocked waiting for the semaphore.
    */
 
-  flags = enter_critical_section();
+  flags = enter_critical_section_nonirq();
 
   /* Try to take the semaphore without waiting. */
 
@@ -173,7 +173,7 @@ int nxsem_clockwait(FAR sem_t *sem, clockid_t clockid,
   /* We can now restore interrupts and delete the watchdog */
 
 out:
-  leave_critical_section(flags);
+  leave_critical_section_nonirq(flags);
   return ret;
 }
 

sched/semaphore/sem_tickwait.c

@@ -80,7 +80,7 @@ int nxsem_tickwait(FAR sem_t *sem, uint32_t delay)
    * enabled while we are blocked waiting for the semaphore.
    */
 
-  flags = enter_critical_section();
+  flags = enter_critical_section_nonirq();
 
   /* Try to take the semaphore without waiting. */
 
@@ -118,7 +118,7 @@ int nxsem_tickwait(FAR sem_t *sem, uint32_t delay)
   /* We can now restore interrupts */
 
 out:
-  leave_critical_section(flags);
+  leave_critical_section_nonirq(flags);
   return ret;
 }
 

sched/semaphore/sem_wait.c

@@ -84,7 +84,7 @@ int nxsem_wait(FAR sem_t *sem)
    * handler.
    */
 
-  flags = enter_critical_section();
+  flags = enter_critical_section_nonirq();
 
   /* Make sure we were supplied with a valid semaphore. */
 
@@ -215,7 +215,7 @@ int nxsem_wait(FAR sem_t *sem)
 #endif
     }
 
-  leave_critical_section(flags);
+  leave_critical_section_nonirq(flags);
   return ret;
 }