pthread_mutex_unlock(): Missing check for the case where pthread_mutex_lock() is called when the mutex is not locked. In that case, it would increment the underlying semaphore avove 1. This is the fix for a problem noted by initialkjc@yahoo.com.

sched/pthread/pthread_mutexunlock.c

@@ -42,11 +42,45 @@
 #include <unistd.h>
 #include <pthread.h>
 #include <sched.h>
+#include <assert.h>
 #include <errno.h>
 #include <debug.h>
 
 #include "pthread/pthread.h"
 
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: pthread_mutex_unlock
+ *
+ * Description:
+ *   Return true is the mutex is locked.
+ *
+ * Parameters:
+ *   None
+ *
+ * Return Value:
+ *   Returns true if the mutex is locked
+ *
+ ****************************************************************************/
+
+static inline bool pthread_mutex_islocked(FAR struct pthread_mutex_s *mutex)
+{
+  int semcount = mutex->sem.semcount;
+
+  /* The underlying semaphore should have a count less than 2:
+   *
+   *  1 == mutex is unlocked.
+   *  0 == mutex is locked with no waiters
+   * -n == mutex is locked with 'n' waiters.
+   */
+
+  DEBUGASSERT(semcount < 2);
+  return semcount < 1;
+}
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -80,17 +114,27 @@
 
 int pthread_mutex_unlock(FAR pthread_mutex_t *mutex)
 {
-  int ret = EINVAL;
+  int ret = OK;
 
   sinfo("mutex=0x%p\n", mutex);
   DEBUGASSERT(mutex != NULL);
+  if (mutex == NULL)
+    {
+      return -EINVAL;
+    }
 
   /* Make sure the semaphore is stable while we make the following checks.
    * This all needs to be one atomic action.
    */
 
   sched_lock();
-  if (mutex != NULL)
+
+  /* The unlock operation is only performed if the mutex is actually locked.
+   * If the mutex is not locked, then SUCCESS will be returned (there is
+   * not error return value specified for this case).
+   */
+
+  if (pthread_mutex_islocked(mutex))
     {
 #if !defined(CONFIG_PTHREAD_MUTEX_UNSAFE) || defined(CONFIG_PTHREAD_MUTEX_TYPES)
       /* Does the calling thread own the semaphore?  If no, should we return