libc/semaphore: Move fast mutex wait/post paths to libc

This avoids unnecessary syscalls in memory protected builds, when mutex
lock/unlock can be done with only atomic counter access

Signed-off-by: Jukka Laitinen <jukka.laitinen@tii.ae>

include/nuttx/semaphore.h

@@ -59,6 +59,10 @@
      {(c), (f), SEM_WAITLIST_INITIALIZER}
 #endif /* CONFIG_PRIORITY_INHERITANCE */
 
+/* Macro to retrieve sem count */
+
+#define NXSEM_COUNT(s) ((FAR atomic_t *)&(s)->semcount)
+
 /****************************************************************************
  * Public Type Definitions
  ****************************************************************************/
@@ -153,7 +157,7 @@ int nxsem_init(FAR sem_t *sem, int pshared, unsigned int value);
 int nxsem_destroy(FAR sem_t *sem);
 
 /****************************************************************************
- * Name: nxsem_wait
+ * Name: nxsem_wait / nxsem_wait_slow
  *
  * Description:
  *   This function attempts to lock the semaphore referenced by 'sem'.  If
@@ -181,9 +185,10 @@ int nxsem_destroy(FAR sem_t *sem);
  ****************************************************************************/
 
 int nxsem_wait(FAR sem_t *sem);
+int nxsem_wait_slow(FAR sem_t *sem);
 
 /****************************************************************************
- * Name: nxsem_trywait
+ * Name: nxsem_trywait / nxsem_trywait_slow
  *
  * Description:
  *   This function locks the specified semaphore only if the semaphore is
@@ -207,6 +212,7 @@ int nxsem_wait(FAR sem_t *sem);
  ****************************************************************************/
 
 int nxsem_trywait(FAR sem_t *sem);
+int nxsem_trywait_slow(FAR sem_t *sem);
 
 /****************************************************************************
  * Name: nxsem_timedwait
@@ -328,7 +334,7 @@ int nxsem_clockwait(FAR sem_t *sem, clockid_t clockid,
 int nxsem_tickwait(FAR sem_t *sem, uint32_t delay);
 
 /****************************************************************************
- * Name: nxsem_post
+ * Name: nxsem_post / nxsem_post_slow
  *
  * Description:
  *   When a kernel thread has finished with a semaphore, it will call
@@ -357,6 +363,7 @@ int nxsem_tickwait(FAR sem_t *sem, uint32_t delay);
  ****************************************************************************/
 
 int nxsem_post(FAR sem_t *sem);
+int nxsem_post_slow(FAR sem_t *sem);
 
 /****************************************************************************
  * Name:  nxsem_get_value

include/sys/syscall_lookup.h

@@ -78,11 +78,11 @@ SYSCALL_LOOKUP(sethostname,                2)
 /* Semaphores */
 
 SYSCALL_LOOKUP(nxsem_destroy,              1)
-SYSCALL_LOOKUP(nxsem_post,                 1)
+SYSCALL_LOOKUP(nxsem_post_slow,            1)
 SYSCALL_LOOKUP(nxsem_clockwait,            3)
 SYSCALL_LOOKUP(nxsem_timedwait,            2)
-SYSCALL_LOOKUP(nxsem_trywait,              1)
-SYSCALL_LOOKUP(nxsem_wait,                 1)
+SYSCALL_LOOKUP(nxsem_trywait_slow,         1)
+SYSCALL_LOOKUP(nxsem_wait_slow,            1)
 
 #ifdef CONFIG_PRIORITY_INHERITANCE
   SYSCALL_LOOKUP(nxsem_set_protocol,       2)

libs/libc/semaphore/sem_post.c

@@ -27,8 +27,10 @@
 #include <nuttx/config.h>
 
 #include <errno.h>
+#include <assert.h>
 
 #include <nuttx/semaphore.h>
+#include <nuttx/atomic.h>
 
 /****************************************************************************
  * Public Functions
@@ -84,3 +86,61 @@ int sem_post(FAR sem_t *sem)
 
   return ret;
 }
+
+/****************************************************************************
+ * Name: nxsem_post
+ *
+ * Description:
+ *   When a kernel thread has finished with a semaphore, it will call
+ *   nxsem_post().  This function unlocks the semaphore referenced by sem
+ *   by performing the semaphore unlock operation on that semaphore.
+ *
+ *   If the semaphore value resulting from this operation is positive, then
+ *   no tasks were blocked waiting for the semaphore to become unlocked; the
+ *   semaphore is simply incremented.
+ *
+ *   If the value of the semaphore resulting from this operation is zero,
+ *   then one of the tasks blocked waiting for the semaphore shall be
+ *   allowed to return successfully from its call to nxsem_wait().
+ *
+ * Input Parameters:
+ *   sem - Semaphore descriptor
+ *
+ * Returned Value:
+ *   This is an internal OS interface and should not be used by applications.
+ *   It follows the NuttX internal error return policy:  Zero (OK) is
+ *   returned on success.  A negated errno value is returned on failure.
+ *
+ * Assumptions:
+ *   This function may be called from an interrupt handler.
+ *
+ ****************************************************************************/
+
+int nxsem_post(FAR sem_t *sem)
+{
+  DEBUGASSERT(sem != NULL);
+
+  /* We don't do atomic fast path in case of LIBC_ARCH_ATOMIC because that
+   * uses spinlocks, which can't be called from userspace. Also in the kernel
+   * taking the slow path directly is faster than locking first in here
+   */
+
+#ifndef CONFIG_LIBC_ARCH_ATOMIC
+
+  if ((sem->flags & SEM_TYPE_MUTEX)
+#  if defined(CONFIG_PRIORITY_PROTECT) || defined(CONFIG_PRIORITY_INHERITANCE)
+      && (sem->flags & SEM_PRIO_MASK) == SEM_PRIO_NONE
+#  endif
+      )
+    {
+      int32_t old = 0;
+      if (atomic_try_cmpxchg_release(NXSEM_COUNT(sem), &old, 1))
+        {
+          return OK;
+        }
+    }
+
+#endif
+
+  return nxsem_post_slow(sem);
+}

libs/libc/semaphore/sem_trywait.c

@@ -27,8 +27,13 @@
 #include <nuttx/config.h>
 
 #include <errno.h>
+#include <assert.h>
+#include <sched.h>
 
+#include <nuttx/init.h>
 #include <nuttx/semaphore.h>
+#include <nuttx/atomic.h>
+#include <nuttx/irq.h>
 
 /****************************************************************************
  * Public Functions
@@ -76,3 +81,59 @@ int sem_trywait(FAR sem_t *sem)
 
   return ret;
 }
+
+/****************************************************************************
+ * Name: nxsem_trywait
+ *
+ * Description:
+ *   This function locks the specified semaphore only if the semaphore is
+ *   currently not locked.  In either case, the call returns without
+ *   blocking.
+ *
+ * Input Parameters:
+ *   sem - the semaphore descriptor
+ *
+ * Returned Value:
+ *   This is an internal OS interface and should not be used by applications.
+ *   It follows the NuttX internal error return policy:  Zero (OK) is
+ *   returned on success.  A negated errno value is returned on failure.
+ *   Possible returned errors:
+ *
+ *     - EINVAL - Invalid attempt to get the semaphore
+ *     - EAGAIN - The semaphore is not available.
+ *
+ ****************************************************************************/
+
+int nxsem_trywait(FAR sem_t *sem)
+{
+  DEBUGASSERT(sem != NULL);
+
+  /* This API should not be called from the idleloop or interrupt */
+
+#if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__)
+  DEBUGASSERT(!OSINIT_IDLELOOP() || !sched_idletask() ||
+              up_interrupt_context());
+#endif
+
+  /* We don't do atomic fast path in case of LIBC_ARCH_ATOMIC because that
+   * uses spinlocks, which can't be called from userspace. Also in the kernel
+   * taking the slow path directly is faster than locking first in here
+   */
+
+#ifndef CONFIG_LIBC_ARCH_ATOMIC
+
+  if ((sem->flags & SEM_TYPE_MUTEX)
+#if defined(CONFIG_PRIORITY_PROTECT) || defined(CONFIG_PRIORITY_INHERITANCE)
+      && (sem->flags & SEM_PRIO_MASK) == SEM_PRIO_NONE
+#endif
+      )
+    {
+      int32_t old = 1;
+      return atomic_try_cmpxchg_acquire(NXSEM_COUNT(sem), &old, 0) ?
+        OK : -EAGAIN;
+    }
+
+#endif
+
+  return nxsem_trywait_slow(sem);
+}

libs/libc/semaphore/sem_wait.c

@@ -27,9 +27,14 @@
 #include <nuttx/config.h>
 
 #include <errno.h>
+#include <assert.h>
+#include <sched.h>
 
+#include <nuttx/init.h>
 #include <nuttx/cancelpt.h>
 #include <nuttx/semaphore.h>
+#include <nuttx/atomic.h>
+#include <nuttx/irq.h>
 
 /****************************************************************************
  * Public Functions
@@ -98,3 +103,67 @@ errout_with_cancelpt:
   leave_cancellation_point();
   return ERROR;
 }
+
+/****************************************************************************
+ * Name: nxsem_wait
+ *
+ * Description:
+ *   This function attempts to lock the semaphore referenced by 'sem'.  If
+ *   the semaphore value is (<=) zero, then the calling task will not return
+ *   until it successfully acquires the lock.
+ *
+ *   This is an internal OS interface.  It is functionally equivalent to
+ *   sem_wait except that:
+ *
+ *   - It is not a cancellation point, and
+ *   - It does not modify the errno value.
+ *
+ * Input Parameters:
+ *   sem - Semaphore descriptor.
+ *
+ * Returned Value:
+ *   This is an internal OS interface and should not be used by applications.
+ *   It follows the NuttX internal error return policy:  Zero (OK) is
+ *   returned on success.  A negated errno value is returned on failure.
+ *   Possible returned errors:
+ *
+ *   - EINVAL:  Invalid attempt to get the semaphore
+ *   - EINTR:   The wait was interrupted by the receipt of a signal.
+ *
+ ****************************************************************************/
+
+int nxsem_wait(FAR sem_t *sem)
+{
+  DEBUGASSERT(sem != NULL);
+
+  /* This API should not be called from the idleloop or interrupt */
+
+#if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__)
+  DEBUGASSERT(!OSINIT_IDLELOOP() || !sched_idletask() ||
+              up_interrupt_context());
+#endif
+
+  /* We don't do atomic fast path in case of LIBC_ARCH_ATOMIC because that
+   * uses spinlocks, which can't be called from userspace. Also in the kernel
+   * taking the slow path directly is faster than locking first in here
+   */
+
+#ifndef CONFIG_LIBC_ARCH_ATOMIC
+
+  if ((sem->flags & SEM_TYPE_MUTEX)
+#  if defined(CONFIG_PRIORITY_PROTECT) || defined(CONFIG_PRIORITY_INHERITANCE)
+      && (sem->flags & SEM_PRIO_MASK) == SEM_PRIO_NONE
+#  endif
+      )
+    {
+      int32_t old = 1;
+      if (atomic_try_cmpxchg_acquire(NXSEM_COUNT(sem), &old, 0))
+        {
+          return OK;
+        }
+    }
+
+#endif
+
+  return nxsem_wait_slow(sem);
+}

sched/semaphore/sem_post.c

@@ -37,7 +37,7 @@
 #include "semaphore/semaphore.h"
 
 /****************************************************************************
- * Private Functions
+ * Public Functions
  ****************************************************************************/
 
 /****************************************************************************
@@ -69,7 +69,7 @@
  *
  ****************************************************************************/
 
-static int nxsem_post_slow(FAR sem_t *sem)
+int nxsem_post_slow(FAR sem_t *sem)
 {
   FAR struct tcb_s *stcb = NULL;
   irqstate_t flags;
@@ -217,60 +217,3 @@ static int nxsem_post_slow(FAR sem_t *sem)
 
   return OK;
 }
-
-/****************************************************************************
- * Public Functions
- ****************************************************************************/
-
-/****************************************************************************
- * Name: nxsem_post
- *
- * Description:
- *   When a kernel thread has finished with a semaphore, it will call
- *   nxsem_post().  This function unlocks the semaphore referenced by sem
- *   by performing the semaphore unlock operation on that semaphore.
- *
- *   If the semaphore value resulting from this operation is positive, then
- *   no tasks were blocked waiting for the semaphore to become unlocked; the
- *   semaphore is simply incremented.
- *
- *   If the value of the semaphore resulting from this operation is zero,
- *   then one of the tasks blocked waiting for the semaphore shall be
- *   allowed to return successfully from its call to nxsem_wait().
- *
- * Input Parameters:
- *   sem - Semaphore descriptor
- *
- * Returned Value:
- *   This is an internal OS interface and should not be used by applications.
- *   It follows the NuttX internal error return policy:  Zero (OK) is
- *   returned on success.  A negated errno value is returned on failure.
- *
- * Assumptions:
- *   This function may be called from an interrupt handler.
- *
- ****************************************************************************/
-
-int nxsem_post(FAR sem_t *sem)
-{
-  DEBUGASSERT(sem != NULL);
-
-  /* If this is a mutex, we can try to unlock the mutex in fast mode,
-   * else try to get it in slow mode.
-   */
-
-  if ((sem->flags & SEM_TYPE_MUTEX)
-#if defined(CONFIG_PRIORITY_PROTECT) || defined(CONFIG_PRIORITY_INHERITANCE)
-      && (sem->flags & SEM_PRIO_MASK) == SEM_PRIO_NONE
-#endif
-      )
-    {
-      int32_t old = 0;
-      if (atomic_try_cmpxchg_release(NXSEM_COUNT(sem), &old, 1))
-        {
-          return OK;
-        }
-    }
-
-  return nxsem_post_slow(sem);
-}

sched/semaphore/sem_trywait.c

@@ -39,7 +39,7 @@
 #include "semaphore/semaphore.h"
 
 /****************************************************************************
- * Private Functions
+ * Public Functions
  ****************************************************************************/
 
 /****************************************************************************
@@ -60,7 +60,7 @@
  *
  ****************************************************************************/
 
-static int nxsem_trywait_slow(FAR sem_t *sem)
+int nxsem_trywait_slow(FAR sem_t *sem)
 {
   irqstate_t flags;
   int32_t semcount;
@@ -103,61 +103,3 @@ static int nxsem_trywait_slow(FAR sem_t *sem)
   leave_critical_section(flags);
   return ret;
 }
-
-/****************************************************************************
- * Public Functions
- ****************************************************************************/
-
-/****************************************************************************
- * Name: nxsem_trywait
- *
- * Description:
- *   This function locks the specified semaphore only if the semaphore is
- *   currently not locked.  In either case, the call returns without
- *   blocking.
- *
- * Input Parameters:
- *   sem - the semaphore descriptor
- *
- * Returned Value:
- *   This is an internal OS interface and should not be used by applications.
- *   It follows the NuttX internal error return policy:  Zero (OK) is
- *   returned on success.  A negated errno value is returned on failure.
- *   Possible returned errors:
- *
- *     EINVAL - Invalid attempt to get the semaphore
- *     EAGAIN - The semaphore is not available.
- *
- * Assumptions:
- *
- ****************************************************************************/
-
-int nxsem_trywait(FAR sem_t *sem)
-{
-  /* This API should not be called from the idleloop */
-
-  DEBUGASSERT(sem != NULL);
-  DEBUGASSERT(!OSINIT_IDLELOOP() || !sched_idletask() ||
-              up_interrupt_context());
-
-  /* If this is a mutex, we can try to get the mutex in fast mode,
-   * else try to get it in slow mode.
-   */
-
-  if ((sem->flags & SEM_TYPE_MUTEX)
-#if defined(CONFIG_PRIORITY_PROTECT) || defined(CONFIG_PRIORITY_INHERITANCE)
-      && (sem->flags & SEM_PRIO_MASK) == SEM_PRIO_NONE
-#endif
-      )
-    {
-      int32_t old = 1;
-      if (atomic_try_cmpxchg_acquire(NXSEM_COUNT(sem), &old, 0))
-        {
-          return OK;
-        }
-
-      return -EAGAIN;
-    }
-
-  return nxsem_trywait_slow(sem);
-}

sched/semaphore/sem_wait.c

@@ -39,7 +39,7 @@
 #include "semaphore/semaphore.h"
 
 /****************************************************************************
- * Private Functions
+ * Public Functions
  ****************************************************************************/
 
 /****************************************************************************
@@ -69,7 +69,7 @@
  *
  ****************************************************************************/
 
-static int nxsem_wait_slow(FAR sem_t *sem)
+int nxsem_wait_slow(FAR sem_t *sem)
 {
   FAR struct tcb_s *rtcb;
   irqstate_t flags;
@@ -222,65 +222,6 @@ static int nxsem_wait_slow(FAR sem_t *sem)
   return ret;
 }
 
-/****************************************************************************
- * Public Functions
- ****************************************************************************/
-
-/****************************************************************************
- * Name: nxsem_wait
- *
- * Description:
- *   This function attempts to lock the semaphore referenced by 'sem'.  If
- *   the semaphore value is (<=) zero, then the calling task will not return
- *   until it successfully acquires the lock.
- *
- *   This is an internal OS interface.  It is functionally equivalent to
- *   sem_wait except that:
- *
- *   - It is not a cancellation point, and
- *   - It does not modify the errno value.
- *
- * Input Parameters:
- *   sem - Semaphore descriptor.
- *
- * Returned Value:
- *   This is an internal OS interface and should not be used by applications.
- *   It follows the NuttX internal error return policy:  Zero (OK) is
- *   returned on success.  A negated errno value is returned on failure.
- *   Possible returned errors:
- *
- *   - EINVAL:  Invalid attempt to get the semaphore
- *   - EINTR:   The wait was interrupted by the receipt of a signal.
- *
- ****************************************************************************/
-
-int nxsem_wait(FAR sem_t *sem)
-{
-  /* This API should not be called from interrupt handlers & idleloop */
-
-  DEBUGASSERT(sem != NULL && up_interrupt_context() == false);
-  DEBUGASSERT(!OSINIT_IDLELOOP() || !sched_idletask());
-
-  /* If this is a mutex, we can try to get the mutex in fast mode,
-   * else try to get it in slow mode.
-   */
-
-  if ((sem->flags & SEM_TYPE_MUTEX)
-#if defined(CONFIG_PRIORITY_PROTECT) || defined(CONFIG_PRIORITY_INHERITANCE)
-      && (sem->flags & SEM_PRIO_MASK) == SEM_PRIO_NONE
-#endif
-      )
-    {
-      int32_t old = 1;
-      if (atomic_try_cmpxchg_acquire(NXSEM_COUNT(sem), &old, 0))
-        {
-          return OK;
-        }
-    }
-
-  return nxsem_wait_slow(sem);
-}
-
 /****************************************************************************
  * Name: nxsem_wait_uninterruptible
  *

sched/semaphore/semaphore.h

@@ -31,7 +31,6 @@
 #include <nuttx/compiler.h>
 #include <nuttx/semaphore.h>
 #include <nuttx/sched.h>
-#include <nuttx/atomic.h>
 
 #include <stdint.h>
 #include <stdbool.h>
@@ -40,8 +39,6 @@
  * Pre-processor Definitions
  ****************************************************************************/
 
-#define NXSEM_COUNT(s) ((FAR atomic_t *)&(s)->semcount)
-
 /****************************************************************************
  * Public Function Prototypes
  ****************************************************************************/

syscall/syscall.csv

@@ -92,13 +92,13 @@
 "nxsem_destroy","nuttx/semaphore.h","","int","FAR sem_t *"
 "nxsem_getprioceiling","nuttx/semaphore.h","defined(CONFIG_PRIORITY_PROTECT)","int","FAR const sem_t *","FAR int *"
 "nxsem_open","nuttx/semaphore.h","defined(CONFIG_FS_NAMED_SEMAPHORES)","int","FAR sem_t **","FAR const char *","int","...","mode_t","unsigned int"
-"nxsem_post","nuttx/semaphore.h","","int","FAR sem_t *"
+"nxsem_post_slow","nuttx/semaphore.h","","int","FAR sem_t *"
 "nxsem_set_protocol","nuttx/semaphore.h","defined(CONFIG_PRIORITY_INHERITANCE)","int","FAR sem_t *","int"
 "nxsem_setprioceiling","nuttx/semaphore.h","defined(CONFIG_PRIORITY_PROTECT)","int","FAR sem_t *","int","FAR int *"
 "nxsem_timedwait","nuttx/semaphore.h","","int","FAR sem_t *","FAR const struct timespec *"
-"nxsem_trywait","nuttx/semaphore.h","","int","FAR sem_t *"
+"nxsem_trywait_slow","nuttx/semaphore.h","","int","FAR sem_t *"
 "nxsem_unlink","nuttx/semaphore.h","defined(CONFIG_FS_NAMED_SEMAPHORES)","int","FAR const char *"
-"nxsem_wait","nuttx/semaphore.h","","int","FAR sem_t *"
+"nxsem_wait_slow","nuttx/semaphore.h","","int","FAR sem_t *"
 "open","fcntl.h","","int","FAR const char *","int","...","mode_t"
 "pgalloc", "nuttx/arch.h", "defined(CONFIG_BUILD_KERNEL)", "uintptr_t", "uintptr_t", "unsigned int"
 "pipe2","unistd.h","defined(CONFIG_PIPES) && CONFIG_DEV_PIPE_SIZE > 0","int","int [2]|FAR int *","int"