Cancellation points: Close up some logic to eliminte some race conditions.

sched/mqueue/mq_rcvinternal.c

@@ -50,6 +50,7 @@
 
 #include <nuttx/irq.h>
 #include <nuttx/arch.h>
+#include <nuttx/pthread.h>
 
 #include "sched/sched.h"
 #include "mqueue/mqueue.h"
@@ -141,6 +142,21 @@ FAR struct mqueue_msg_s *mq_waitreceive(mqd_t mqdes)
   FAR struct mqueue_inode_s *msgq;
   FAR struct mqueue_msg_s *rcvmsg;
 
+  /* mq_waitreceive() is not a cancellation point, but it is always called
+   * from a cancellation point.
+   */
+
+  if (enter_cancellation_point())
+    {
+      /* If there is a pending cancellation, then do not perform
+       * the wait.  Exit now with ECANCELED.
+       */
+
+      set_errno(ECANCELED);
+      leave_cancellation_point();
+      return NULL;
+    }
+
   /* Get a pointer to the message queue */
 
   msgq = mqdes->msgq;
@@ -195,6 +211,7 @@ FAR struct mqueue_msg_s *mq_waitreceive(mqd_t mqdes)
       msgq->nmsgs--;
     }
 
+  leave_cancellation_point();
   return rcvmsg;
 }
 

sched/mqueue/mq_receive.c

@@ -106,7 +106,7 @@ ssize_t mq_receive(mqd_t mqdes, FAR char *msg, size_t msglen,
 
   /* mq_receive() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Verify the input parameters and, in case of an error, set
    * errno appropriately.

sched/mqueue/mq_send.c

@@ -106,7 +106,7 @@ int mq_send(mqd_t mqdes, FAR const char *msg, size_t msglen, int prio)
 
   /* mq_send() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Verify the input parameters -- setting errno appropriately
    * on any failures to verify.

sched/mqueue/mq_sndinternal.c

@@ -1,5 +1,5 @@
 /****************************************************************************
- *  sched/mqueue/mq_send.c
+ *  sched/mqueue/mq_sndinternal.c
  *
  *   Copyright (C) 2007, 2009, 2013-2016 Gregory Nutt. All rights reserved.
  *   Author: Gregory Nutt <gnutt@nuttx.org>
@@ -53,6 +53,7 @@
 #include <nuttx/arch.h>
 #include <nuttx/sched.h>
 #include <nuttx/signal.h>
+#include <nuttx/pthread.h>
 
 #include "sched/sched.h"
 #ifndef CONFIG_DISABLE_SIGNALS
@@ -232,6 +233,21 @@ int mq_waitsend(mqd_t mqdes)
   FAR struct tcb_s *rtcb;
   FAR struct mqueue_inode_s *msgq;
 
+  /* mq_waitsend() is not a cancellation point, but it is always called from
+   * a cancellation point.
+   */
+
+  if (enter_cancellation_point())
+    {
+      /* If there is a pending cancellation, then do not perform
+       * the wait.  Exit now with ECANCELED.
+       */
+
+      set_errno(ECANCELED);
+      leave_cancellation_point();
+      return ERROR;
+    }
+
   /* Get a pointer to the message queue */
 
   msgq = mqdes->msgq;
@@ -249,6 +265,7 @@ int mq_waitsend(mqd_t mqdes)
           /* No... We will return an error to the caller. */
 
           set_errno(EAGAIN);
+          leave_cancellation_point();
           return ERROR;
         }
 
@@ -283,12 +300,14 @@ int mq_waitsend(mqd_t mqdes)
 
               if (get_errno() != OK)
                 {
+                  leave_cancellation_point();
                   return ERROR;
                 }
             }
         }
     }
 
+  leave_cancellation_point();
   return OK;
 }
 

sched/mqueue/mq_timedreceive.c

@@ -177,7 +177,7 @@ ssize_t mq_timedreceive(mqd_t mqdes, FAR char *msg, size_t msglen,
 
   /* mq_timedreceive() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Verify the input parameters and, in case of an error, set
    * errno appropriately.

sched/mqueue/mq_timedsend.c

@@ -181,7 +181,7 @@ int mq_timedsend(mqd_t mqdes, FAR const char *msg, size_t msglen, int prio,
 
   /* mq_timedsend() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Verify the input parameters -- setting errno appropriately
    * on any failures to verify.

sched/pthread/pthread_condtimedwait.c

@@ -181,7 +181,7 @@ int pthread_cond_timedwait(FAR pthread_cond_t *cond, FAR pthread_mutex_t *mutex,
 
   /* pthread_cond_timedwait() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Make sure that non-NULL references were provided. */
 

sched/pthread/pthread_condwait.c

@@ -77,7 +77,7 @@ int pthread_cond_wait(FAR pthread_cond_t *cond, FAR pthread_mutex_t *mutex)
 
   /* pthread_cond_wait() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Make sure that non-NULL references were provided. */
 

sched/pthread/pthread_join.c

@@ -96,7 +96,7 @@ int pthread_join(pthread_t thread, FAR pthread_addr_t *pexit_value)
 
   /* pthread_join() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* First make sure that this is not an attempt to join to
    * ourself.

sched/pthread/pthread_testcancel.c

@@ -62,6 +62,6 @@
 
 void pthread_testcancel(void)
 {
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
   leave_cancellation_point();
 }

sched/sched/sched_waitid.c

@@ -167,7 +167,7 @@ int waitid(idtype_t idtype, id_t id, FAR siginfo_t *info, int options)
 
   /* waitid() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* MISSING LOGIC:   If WNOHANG is provided in the options, then this function
    * should returned immediately.  However, there is no mechanism available now

sched/sched/sched_waitpid.c

@@ -188,7 +188,7 @@ pid_t waitpid(pid_t pid, int *stat_loc, int options)
 
   /* waitpid() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* None of the options are supported */
 
@@ -317,7 +317,7 @@ pid_t waitpid(pid_t pid, int *stat_loc, int options)
 
   /* waitpid() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* None of the options are supported */
 

sched/semaphore/sem_timedwait.c

@@ -106,7 +106,7 @@ int sem_timedwait(FAR sem_t *sem, FAR const struct timespec *abstime)
 
   /* sem_timedwait() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Verify the input parameters and, in case of an error, set
    * errno appropriately.

sched/semaphore/sem_wait.c

@@ -87,21 +87,31 @@ int sem_wait(FAR sem_t *sem)
 
   DEBUGASSERT(sem != NULL && up_interrupt_context() == false);
 
-  /* sem_wait is a cancellation point */
+  /* The following operations must be performed with interrupts
+   * disabled because sem_post() may be called from an interrupt
+   * handler.
+   */
 
-  enter_cancellation_point();
+  flags = enter_critical_section();
+
+  /* sem_wait() is a cancellation point */
+
+  if (enter_cancellation_point())
+    {
+      /* If there is a pending cancellation, then do not perform
+       * the wait.  Exit now with ECANCELED.
+       */
+
+      set_errno(ECANCELED);
+      leave_cancellation_point();
+      leave_critical_section(flags);
+      return ERROR;
+    }
 
   /* Make sure we were supplied with a valid semaphore. */
 
   if (sem != NULL)
     {
-      /* The following operations must be performed with interrupts
-       * disabled because sem_post() may be called from an interrupt
-       * handler.
-       */
-
-      flags = enter_critical_section();
-
       /* Check if the lock is available */
 
       if (sem->semcount > 0)
@@ -191,10 +201,6 @@ int sem_wait(FAR sem_t *sem)
           sched_unlock();
 #endif
         }
-
-      /* Interrupts may now be enabled. */
-
-      leave_critical_section(flags);
     }
   else
     {
@@ -202,5 +208,6 @@ int sem_wait(FAR sem_t *sem)
     }
 
   leave_cancellation_point();
+  leave_critical_section(flags);
   return ret;
 }

sched/signal/sig_nanosleep.c

@@ -115,7 +115,7 @@ int nanosleep(FAR const struct timespec *rqtp, FAR struct timespec *rmtp)
 
   /* nanosleep() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   if (!rqtp || rqtp->tv_nsec < 0 || rqtp->tv_nsec >= 1000000000)
     {

sched/signal/sig_pause.c

@@ -80,7 +80,7 @@ int pause(void)
 
   /* pause() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Set up for the sleep.  Using the empty set means that we are not
    * waiting for any particular signal.  However, any unmasked signal

sched/signal/sig_suspend.c

@@ -101,7 +101,7 @@ int sigsuspend(FAR const sigset_t *set)
 
   /* sigsuspend() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Several operations must be performed below:  We must determine if any
    * signal is pending and, if not, wait for the signal.  Since signals can

sched/signal/sig_timedwait.c

@@ -176,7 +176,7 @@ int sigtimedwait(FAR const sigset_t *set, FAR struct siginfo *info,
 
   /* sigtimedwait() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
   sched_lock();  /* Not necessary */
 
   /* Several operations must be performed below:  We must determine if any

sched/signal/sig_waitinfo.c

@@ -72,7 +72,7 @@ int sigwaitinfo(FAR const sigset_t *set, FAR struct siginfo *info)
 
   /* sigwaitinfo() is a cancellation point */
 
-  enter_cancellation_point();
+  (void)enter_cancellation_point();
 
   /* Just a wrapper around sigtimedwait() */
 

sched/task/task_cancelpt.c

@@ -99,11 +99,19 @@
  *      pending cancellation and, if so, calls either exit() or
  *      pthread_exit(), depending upon the type of the thread.
  *
+ * Input Parameters:
+ *   None
+ *
+ * Returned Value
+ *   true is returned if a cancellation is pending but cannot be performed
+ *   now due to the nesting level.
+ *
  ****************************************************************************/
 
-void enter_cancellation_point(void)
+bool enter_cancellation_point(void)
 {
   FAR struct tcb_s *tcb = this_task();
+  bool ret = false;
 
   /* Disabling pre-emption should provide sufficient protection.  We only
    * need the TCB to be stationary (no interrupt level modification is
@@ -126,24 +134,32 @@ void enter_cancellation_point(void)
        (tcb->flags & TCB_FLAG_CANCEL_DEFERRED) != 0) ||
        tcb->cpcount > 0)
     {
-      /* If there is a pending cancellation and we are at the outermost
-       * nesting level of cancellation function calls, then just exit
-       * according to the type of the thread.
-       */
+      /* Check if there is a pending cancellation */
 
-      if ((tcb->flags & TCB_FLAG_CANCEL_PENDING) != 0 &&
-          tcb->cpcount == 0)
+      if ((tcb->flags & TCB_FLAG_CANCEL_PENDING) != 0)
         {
+          /* Yes... return true (if we don't exit here) */
+
+          ret = true;
+
+          /* If there is a pending cancellation and we are at the outermost
+           * nesting level of cancellation function calls, then exit
+           * according to the type of the thread.
+           */
+
+          if (tcb->cpcount == 0)
+            {
 #ifndef CONFIG_DISABLE_PTHREAD
-           if ((tcb->flags & TCB_FLAG_TTYPE_MASK) == TCB_FLAG_TTYPE_PTHREAD)
-             {
-               pthread_exit(PTHREAD_CANCELED);
-             }
-           else
+               if ((tcb->flags & TCB_FLAG_TTYPE_MASK) == TCB_FLAG_TTYPE_PTHREAD)
+                 {
+                   pthread_exit(PTHREAD_CANCELED);
+                 }
+               else
 #endif
-             {
-               exit(EXIT_FAILURE);
-             }
+                 {
+                   exit(EXIT_FAILURE);
+                 }
+            }
         }
 
       /* Otherwise, indicate that we are at a cancellation point by
@@ -156,6 +172,7 @@ void enter_cancellation_point(void)
     }
 
   sched_unlock();
+  return ret;
 }
 
 /****************************************************************************
@@ -173,6 +190,12 @@ void enter_cancellation_point(void)
  *      pending cancellation and, if so, calls either exit() or
  *      pthread_exit(), depending upon the type of the thread.
  *
+ * Input Parameters:
+ *   None
+ *
+ * Returned Value
+ *   None
+ *
  ****************************************************************************/
 
 void leave_cancellation_point(void)