pthreads: Re-order some operations so that mutexes are placed in the inconsistent state BEFORE the clean-up callbacks are called.

sched/pthread/pthread_cancel.c

@@ -145,6 +145,12 @@ int pthread_cancel(pthread_t thread)
       pthread_exit(PTHREAD_CANCELED);
     }
 
+#ifndef CONFIG_PTHREAD_MUTEX_UNSAFE
+  /* Recover any mutexes still held by the canceled thread */
+
+  pthread_mutex_inconsistent(tcb);
+#endif
+
 #ifdef CONFIG_PTHREAD_CLEANUP
   /* Perform any stack pthread clean-up callbacks.
    *
@@ -162,12 +168,6 @@ int pthread_cancel(pthread_t thread)
 
   (void)pthread_completejoin((pid_t)thread, PTHREAD_CANCELED);
 
-#ifndef CONFIG_PTHREAD_MUTEX_UNSAFE
-  /* Recover any mutexes still held by the canceled thread */
-
-  pthread_mutex_inconsistent(tcb);
-#endif
-
   /* Then let task_terminate do the real work */
 
   return task_terminate((pid_t)thread, false);

sched/pthread/pthread_exit.c

@@ -105,6 +105,12 @@ void pthread_exit(FAR void *exit_value)
    tcb->cpcount = 0;
 #endif
 
+#ifndef CONFIG_PTHREAD_MUTEX_UNSAFE
+  /* Recover any mutexes still held by the canceled thread */
+
+  pthread_mutex_inconsistent((FAR struct pthread_tcb_s *)tcb);
+#endif
+
 #ifdef CONFIG_PTHREAD_CLEANUP
    /* Perform any stack pthread clean-up callbacks */
 
@@ -123,12 +129,6 @@ void pthread_exit(FAR void *exit_value)
       exit(EXIT_FAILURE);
     }
 
-#ifndef CONFIG_PTHREAD_MUTEX_UNSAFE
-  /* Recover any mutexes still held by the canceled thread */
-
-  pthread_mutex_inconsistent((FAR struct pthread_tcb_s *)tcb);
-#endif
-
   /* Perform common task termination logic.  This will get called again later
    * through logic kicked off by _exit().  However, we need to call it before
    * calling _exit() in order certain operations if this is the last thread

sched/pthread/pthread_mutexconsistent.c

@@ -119,7 +119,6 @@ int pthread_mutex_consistent(FAR pthread_mutex_t *mutex)
               /* The thread associated with the PID no longer exists */
 
               mutex->pid    = -1;
-              mutex->flags &= _PTHREAD_MFLAGS_ROBUST;
 #ifdef CONFIG_PTHREAD_MUTEX_TYPES
               mutex->nlocks = 0;
 #endif
@@ -132,6 +131,9 @@ int pthread_mutex_consistent(FAR pthread_mutex_t *mutex)
             }
         }
 
+      /* Clear the inconsistent flag in any case */
+
+      mutex->flags &= _PTHREAD_MFLAGS_ROBUST;
       sched_unlock();
       ret = OK;
     }