score: Turn thread lock into thread wait lock

The _Thread_Lock_acquire() function had a potentially infinite run-time
due to the lack of fairness at atomic operations level.

Update #2412.
Update #2556.
Update #2765.

cpukit/libmisc/monitor/mon-task.c

@@ -20,17 +20,16 @@ rtems_monitor_task_wait_info(
     Thread_Control       *rtems_thread
 )
 {
-    ISR_lock_Context  lock_context;
-    void             *lock;
+    Thread_queue_Context queue_context;
 
-    lock = _Thread_Lock_acquire( rtems_thread, &lock_context );
+    _Thread_Wait_acquire( rtems_thread, &queue_context );
 
     canonical_task->state = rtems_thread->current_state;
     canonical_task->wait_id = _Thread_Wait_get_id( rtems_thread );
     canonical_task->wait_queue = rtems_thread->Wait.queue;
     canonical_task->wait_operations = rtems_thread->Wait.operations;
 
-    _Thread_Lock_release( lock, &lock_context );
+    _Thread_Wait_release( rtems_thread, &queue_context );
 }
 
 void

cpukit/posix/src/pthreadgetschedparam.c

@@ -36,7 +36,7 @@ int pthread_getschedparam(
 )
 {
   Thread_Control          *the_thread;
-  ISR_lock_Context         lock_context;
+  Thread_queue_Context     queue_context;
   POSIX_API_Control       *api;
   const Scheduler_Control *scheduler;
   Priority_Control         priority;
@@ -45,7 +45,7 @@ int pthread_getschedparam(
     return EINVAL;
   }
 
-  the_thread = _Thread_Get( thread, &lock_context );
+  the_thread = _Thread_Get( thread, &queue_context.Lock_context );
 
   if ( the_thread == NULL ) {
     return ESRCH;
@@ -53,7 +53,7 @@ int pthread_getschedparam(
 
   api = the_thread->API_Extensions[ THREAD_API_POSIX ];
 
-  _Thread_Lock_acquire_default_critical( the_thread, &lock_context );
+  _Thread_Wait_acquire_critical( the_thread, &queue_context );
 
   *policy = api->Attributes.schedpolicy;
   *param  = api->Attributes.schedparam;
@@ -61,7 +61,7 @@ int pthread_getschedparam(
   scheduler = _Scheduler_Get_own( the_thread );
   priority = the_thread->real_priority;
 
-  _Thread_Lock_release_default( the_thread, &lock_context );
+  _Thread_Wait_release( the_thread, &queue_context );
 
   param->sched_priority = _POSIX_Priority_From_core( scheduler, priority );
   return 0;

cpukit/rtems/include/rtems/rtems/ratemonimpl.h

@@ -73,7 +73,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Acquire_critical(
   ISR_lock_Context *lock_context
 )
 {
-  _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+  _Thread_Wait_acquire_default_critical( the_thread, lock_context );
 }
 
 RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release(
@@ -81,7 +81,7 @@ RTEMS_INLINE_ROUTINE void _Rate_monotonic_Release(
   ISR_lock_Context *lock_context
 )
 {
-  _Thread_Lock_release_default( the_thread, lock_context );
+  _Thread_Wait_release_default( the_thread, lock_context );
 }
 
 RTEMS_INLINE_ROUTINE Rate_monotonic_Control *_Rate_monotonic_Get(

cpukit/rtems/src/eventreceive.c

@@ -38,7 +38,7 @@ rtems_status_code rtems_event_receive(
     RTEMS_API_Control *api;
     Event_Control     *event;
 
-    executing = _Thread_Lock_acquire_default_for_executing( &lock_context );
+    executing = _Thread_Wait_acquire_default_for_executing( &lock_context );
     api = executing->API_Extensions[ THREAD_API_RTEMS ];
     event = &api->Event;
 
@@ -56,7 +56,7 @@ rtems_status_code rtems_event_receive(
       );
     } else {
       *event_out = event->pending_events;
-      _Thread_Lock_release_default( executing, &lock_context );
+      _Thread_Wait_release_default( executing, &lock_context );
       sc = RTEMS_SUCCESSFUL;
     }
   } else {

cpukit/rtems/src/eventseize.c

@@ -50,13 +50,13 @@ rtems_status_code _Event_Seize(
        (seized_events == event_in || _Options_Is_any( option_set )) ) {
     event->pending_events =
       _Event_sets_Clear( pending_events, seized_events );
-    _Thread_Lock_release_default( executing, lock_context );
+    _Thread_Wait_release_default( executing, lock_context );
     *event_out = seized_events;
     return RTEMS_SUCCESSFUL;
   }
 
   if ( _Options_Is_no_wait( option_set ) ) {
-    _Thread_Lock_release_default( executing, lock_context );
+    _Thread_Wait_release_default( executing, lock_context );
     *event_out = seized_events;
     return RTEMS_UNSATISFIED;
   }
@@ -78,7 +78,7 @@ rtems_status_code _Event_Seize(
   _Thread_Wait_flags_set( executing, intend_to_block );
 
   cpu_self = _Thread_Dispatch_disable_critical( lock_context );
-  _Thread_Lock_release_default( executing, lock_context );
+  _Thread_Wait_release_default( executing, lock_context );
 
   if ( ticks ) {
     _Thread_Timer_insert_relative(

cpukit/rtems/src/eventsurrender.c

@@ -77,7 +77,7 @@ rtems_status_code _Event_Surrender(
   rtems_event_set seized_events;
   bool            unblock;
 
-  _Thread_Lock_acquire_default_critical( the_thread, lock_context );
+  _Thread_Wait_acquire_default_critical( the_thread, lock_context );
 
   _Event_sets_Post( event_in, &event->pending_events );
   pending_events = event->pending_events;
@@ -116,14 +116,14 @@ rtems_status_code _Event_Surrender(
     Per_CPU_Control *cpu_self;
 
     cpu_self = _Thread_Dispatch_disable_critical( lock_context );
-    _Thread_Lock_release_default( the_thread, lock_context );
+    _Thread_Wait_release_default( the_thread, lock_context );
 
     _Thread_Timer_remove( the_thread );
     _Thread_Unblock( the_thread );
 
     _Thread_Dispatch_enable( cpu_self );
   } else {
-    _Thread_Lock_release_default( the_thread, lock_context );
+    _Thread_Wait_release_default( the_thread, lock_context );
   }
 
   return RTEMS_SUCCESSFUL;

cpukit/rtems/src/systemeventreceive.c

@@ -44,7 +44,7 @@ rtems_status_code rtems_event_system_receive(
     RTEMS_API_Control *api;
     Event_Control     *event;
 
-    executing = _Thread_Lock_acquire_default_for_executing( &lock_context );
+    executing = _Thread_Wait_acquire_default_for_executing( &lock_context );
     api = executing->API_Extensions[ THREAD_API_RTEMS ];
     event = &api->System_event;
 
@@ -62,7 +62,7 @@ rtems_status_code rtems_event_system_receive(
       );
     } else {
       *event_out = event->pending_events;
-      _Thread_Lock_release_default( executing, &lock_context );
+      _Thread_Wait_release_default( executing, &lock_context );
       sc = RTEMS_SUCCESSFUL;
     }
   } else {

cpukit/rtems/src/tasksetscheduler.c

@@ -28,10 +28,9 @@ rtems_status_code rtems_task_set_scheduler(
 {
   const Scheduler_Control *scheduler;
   Thread_Control          *the_thread;
-  ISR_lock_Context         lock_context;
-  ISR_lock_Context         state_lock_context;
+  Thread_queue_Context     queue_context;
+  ISR_lock_Context         state_context;
   Per_CPU_Control         *cpu_self;
-  void                    *lock;
   bool                     valid;
   Priority_Control         core_priority;
   Status_Control           status;
@@ -45,7 +44,7 @@ rtems_status_code rtems_task_set_scheduler(
     return RTEMS_INVALID_PRIORITY;
   }
 
-  the_thread = _Thread_Get( task_id, &lock_context );
+  the_thread = _Thread_Get( task_id, &queue_context.Lock_context );
 
   if ( the_thread == NULL ) {
 #if defined(RTEMS_MULTIPROCESSING)
@@ -57,16 +56,15 @@ rtems_status_code rtems_task_set_scheduler(
     return RTEMS_INVALID_ID;
   }
 
-  cpu_self = _Thread_Dispatch_disable_critical( &lock_context );
-  _ISR_lock_ISR_enable( &lock_context );
+  cpu_self = _Thread_Dispatch_disable_critical( &queue_context.Lock_context );
 
-  lock = _Thread_Lock_acquire( the_thread, &lock_context );
-  _Thread_State_acquire_critical( the_thread, &state_lock_context );
+  _Thread_Wait_acquire_critical( the_thread, &queue_context );
+  _Thread_State_acquire_critical( the_thread, &state_context );
 
   status = _Scheduler_Set( scheduler, the_thread, core_priority );
 
-  _Thread_State_release_critical( the_thread, &state_lock_context );
-  _Thread_Lock_release( lock, &lock_context );
+  _Thread_State_release_critical( the_thread, &state_context );
+  _Thread_Wait_release( the_thread, &queue_context );
   _Thread_Dispatch_enable( cpu_self );
   return _Status_Get( status );
 }

cpukit/score/include/rtems/score/schedulerimpl.h

@@ -825,6 +825,20 @@ RTEMS_INLINE_ROUTINE void _Scheduler_Node_set_priority(
 #endif
 }
 
+RTEMS_INLINE_ROUTINE void _Scheduler_Thread_set_priority(
+  Thread_Control   *the_thread,
+  Priority_Control  new_priority,
+  bool              prepend_it
+)
+{
+  Scheduler_Node *own_node;
+
+  own_node = _Scheduler_Thread_get_own_node( the_thread );
+  _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
+
+  the_thread->current_priority = new_priority;
+}
+
 #if defined(RTEMS_SMP)
 /**
  * @brief Gets an idle thread from the scheduler instance.

cpukit/score/include/rtems/score/thread.h

@@ -11,7 +11,7 @@
  *  COPYRIGHT (c) 1989-2014.
  *  On-Line Applications Research Corporation (OAR).
  *
- *  Copyright (c) 2014 embedded brains GmbH.
+ *  Copyright (c) 2014, 2016 embedded brains GmbH.
  *
  *  The license and distribution terms for this file may be
  *  found in the file LICENSE in this distribution or at
@@ -287,16 +287,6 @@ typedef struct {
    */
   uint32_t              return_code;
 
-  /**
-   * @brief The current thread queue.
-   *
-   * In case this field is @c NULL, then the thread is not blocked on a thread
-   * queue.  This field is protected by the thread lock.
-   *
-   * @see _Thread_Lock_set() and _Thread_Wait_set_queue().
-   */
-  Thread_queue_Queue *queue;
-
   /**
    * @brief This field contains several flags used to control the wait class
    * and state of a thread in case fine-grained locking is used.
@@ -307,12 +297,54 @@ typedef struct {
   Thread_Wait_flags     flags;
 #endif
 
+#if defined(RTEMS_SMP)
+  /**
+   * @brief Thread wait lock control block.
+   *
+   * Parts of the thread wait information are protected by the thread wait
+   * default lock and additionally a thread queue lock in case the thread
+   * is enqueued on a thread queue.
+   *
+   * The thread wait lock mechanism protects the following thread variables
+   *  - POSIX_API_Control::Attributes,
+   *  - Thread_Control::current_priority,
+   *  - Thread_Control::Wait::Lock::Pending_requests,
+   *  - Thread_Control::Wait::queue, and
+   *  - Thread_Control::Wait::operations.
+   *
+   * @see _Thread_Wait_acquire(), _Thread_Wait_release(), _Thread_Wait_claim(),
+   *   _Thread_Wait_restore_default() and _Thread_Wait_tranquilize().
+   */
+  struct {
+    /**
+     * @brief Thread wait default lock.
+     */
+    ISR_lock_Control Default;
+
+    /**
+     * @brief The pending thread wait lock acquire or tranquilize requests in
+     * case the thread is enqueued on a thread queue.
+     */
+    Chain_Control Pending_requests;
+  } Lock;
+#endif
+
+  /**
+   * @brief The current thread queue.
+   *
+   * If this field is NULL the thread is not enqueued on a thread queue.  This
+   * field is protected by the thread wait default lock.
+   *
+   * @see _Thread_Wait_claim().
+   */
+  Thread_queue_Queue *queue;
+
   /**
    * @brief The current thread queue operations.
    *
-   * This field is protected by the thread lock.
+   * This field is protected by the thread lock wait default lock.
    *
-   * @see _Thread_Lock_set() and _Thread_Wait_set_operations().
+   * @see _Thread_Wait_claim().
    */
   const Thread_queue_Operations *operations;
 
@@ -639,66 +671,6 @@ typedef struct  {
   void *        control;
 }Thread_Capture_control;
 
-#if defined(RTEMS_SMP)
-/**
- * @brief Thread lock control.
- *
- * The thread lock is either the default lock or the lock of the resource on
- * which the thread is currently blocked.  The generation number takes care
- * that the up to date lock is used.  Only resources using fine grained locking
- * provide their own lock.
- *
- * The thread lock protects the following thread variables
- *  - POSIX_API_Control::Attributes,
- *  - Thread_Control::current_priority,
- *  - Thread_Control::Wait::queue, and
- *  - Thread_Control::Wait::operations.
- *
- * @see _Thread_Lock_acquire(), _Thread_Lock_release(), _Thread_Lock_set() and
- * _Thread_Lock_restore_default().
- */
-typedef struct {
-  /**
-   * @brief The current thread lock.
-   *
-   * This is a plain ticket lock without SMP lock statistics support.  This
-   * enables external libraries to use thread locks since they are independent
-   * of the actual RTEMS build configuration, e.g. profiling enabled or
-   * disabled.
-   */
-  union {
-    /**
-     * @brief The current thread lock as an atomic unsigned integer pointer value.
-     */
-    Atomic_Uintptr atomic;
-
-    /**
-     * @brief The current thread lock as a normal pointer.
-     *
-     * Only provided for debugging purposes.
-     */
-    SMP_ticket_lock_Control *normal;
-  } current;
-
-  /**
-   * @brief The default thread lock in case the thread is not blocked on a
-   * resource.
-   */
-  SMP_ticket_lock_Control Default;
-
-#if defined(RTEMS_PROFILING)
-  /**
-   * @brief The thread lock statistics.
-   *
-   * These statistics are used by the executing thread in case it acquires a
-   * thread lock.  Thus the statistics are an aggregation of acquire and
-   * release operations of different locks.
-   */
-  SMP_lock_Stats Stats;
-#endif
-} Thread_Lock_control;
-#endif
-
 /**
  *  This structure defines the Thread Control Block (TCB).
  *
@@ -770,13 +742,6 @@ struct _Thread_Control {
 #endif
      /*================= end of common block =================*/
 
-#if defined(RTEMS_SMP)
-  /**
-   * @brief Thread lock control.
-   */
-  Thread_Lock_control Lock;
-#endif
-
 #if defined(RTEMS_SMP) && defined(RTEMS_PROFILING)
   /**
    * @brief Potpourri lock statistics.

cpukit/score/include/rtems/score/threadq.h

@@ -43,6 +43,10 @@ extern "C" {
 
 typedef struct _Thread_Control Thread_Control;
 
+typedef struct Thread_queue_Queue Thread_queue_Queue;
+
+typedef struct Thread_queue_Operations Thread_queue_Operations;
+
 typedef struct Thread_queue_Path Thread_queue_Path;
 
 #if defined(RTEMS_MULTIPROCESSING)
@@ -53,6 +57,8 @@ typedef struct Thread_queue_Path Thread_queue_Path;
  *   control is actually a thread proxy if and only if
  *   _Objects_Is_local_id( the_proxy->Object.id ) is false.
  * @param mp_id Object identifier of the object containing the thread queue.
+ *
+ * @see _Thread_queue_Context_set_MP_callout().
  */
 typedef void ( *Thread_queue_MP_callout )(
   Thread_Control *the_proxy,
@@ -60,6 +66,24 @@ typedef void ( *Thread_queue_MP_callout )(
 );
 #endif
 
+#if defined(RTEMS_SMP)
+/**
+ * @brief The thread queue gate is an SMP synchronization means.
+ *
+ * The gates are added to a list of requests.  A busy wait is performed to make
+ * sure that preceding requests are carried out.  Each predecessor notifies its
+ * successor about on request completion.
+ *
+ * @see _Thread_queue_Gate_add(), _Thread_queue_Gate_wait(), and
+ *   _Thread_queue_Gate_open().
+ */
+typedef struct {
+  Chain_Node Node;
+
+  Atomic_Uint go_ahead;
+} Thread_queue_Gate;
+#endif
+
 /**
  * @brief Thread queue context for the thread queue methods.
  *
@@ -105,8 +129,64 @@ typedef struct {
 #if defined(RTEMS_MULTIPROCESSING)
   Thread_queue_MP_callout mp_callout;
 #endif
+
+#if defined(RTEMS_SMP)
+  /**
+   * @brief Data to support thread queue enqueue operations.
+   */
+  struct {
+    /**
+     * @brief Gate to synchronize thread wait lock requests.
+     *
+     * @see _Thread_Wait_acquire_critical() and _Thread_Wait_tranquilize().
+     */
+    Thread_queue_Gate Gate;
+
+    /**
+     * @brief The thread queue lock in case the thread is blocked on a thread
+     * queue at thread wait lock acquire time.
+     */
+    SMP_ticket_lock_Control *queue_lock;
+
+    /**
+     * @brief The thread queue after thread wait lock acquire.
+     *
+     * In case the thread queue is NULL and the thread queue lock is non-NULL
+     * in this context, then we have a stale thread queue.  This happens in
+     * case the thread wait default is restored while we wait on the thread
+     * queue lock, e.g. during a mutex ownership transfer.
+     *
+     * @see _Thread_Wait_restore_default().
+     */
+    Thread_queue_Queue *queue;
+
+    /**
+     * @brief The thread queue operations after thread wait lock acquire.
+     */
+    const Thread_queue_Operations *operations;
+  } Wait;
+#endif
 } Thread_queue_Context;
 
+#if defined(RTEMS_SMP)
+/**
+ * @brief A thread queue link from one thread to another specified by the
+ * thread queue owner and thread wait queue relationships.
+ */
+typedef struct {
+  /**
+   * @brief The owner of this thread queue link.
+   */
+  Thread_Control *owner;
+
+  /**
+   * @brief The queue context used to acquire the thread wait lock of the
+   * owner.
+   */
+  Thread_queue_Context Queue_context;
+} Thread_queue_Link;
+#endif
+
 /**
  * @brief Thread priority queue.
  */
@@ -191,7 +271,7 @@ typedef struct _Thread_queue_Heads {
     sizeof( Thread_queue_Heads )
 #endif
 
-typedef struct {
+struct Thread_queue_Queue {
   /**
    * @brief Lock to protect this thread queue.
    *
@@ -221,13 +301,15 @@ typedef struct {
    * @brief The thread queue owner.
    */
   Thread_Control *owner;
-} Thread_queue_Queue;
+};
 
 /**
  * @brief Thread queue priority change operation.
  *
  * @param[in] the_thread The thread.
  * @param[in] new_priority The new priority value.
+ * @param[in] prepend_it In case this is true, then the thread is prepended to
+ *   its priority group in its scheduler instance, otherwise it is appended.
  * @param[in] queue The actual thread queue.
  *
  * @see Thread_queue_Operations.
@@ -235,6 +317,7 @@ typedef struct {
 typedef void ( *Thread_queue_Priority_change_operation )(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 );
 
@@ -247,8 +330,6 @@ typedef void ( *Thread_queue_Priority_change_operation )(
  *
  * @param[in] queue The actual thread queue.
  * @param[in] the_thread The thread to enqueue on the queue.
- *
- * @see _Thread_Wait_set_operations().
  */
 typedef void ( *Thread_queue_Enqueue_operation )(
   Thread_queue_Queue *queue,
@@ -261,8 +342,6 @@ typedef void ( *Thread_queue_Enqueue_operation )(
  *
  * @param[in] queue The actual thread queue.
  * @param[in] the_thread The thread to extract from the thread queue.
- *
- * @see _Thread_Wait_set_operations().
  */
 typedef void ( *Thread_queue_Extract_operation )(
   Thread_queue_Queue *queue,
@@ -277,8 +356,6 @@ typedef void ( *Thread_queue_Extract_operation )(
  * @retval NULL No thread is present on the thread queue.
  * @retval first The first thread of the thread queue according to the insert
  * order.  This thread remains on the thread queue.
- *
- * @see _Thread_Wait_set_operations().
  */
 typedef Thread_Control *( *Thread_queue_First_operation )(
   Thread_queue_Heads *heads
@@ -289,7 +366,7 @@ typedef Thread_Control *( *Thread_queue_First_operation )(
  *
  * @see _Thread_wait_Set_operations().
  */
-typedef struct {
+struct Thread_queue_Operations {
   /**
    * @brief Thread queue priority change operation.
    *
@@ -320,7 +397,7 @@ typedef struct {
    * @brief Thread queue first operation.
    */
   Thread_queue_First_operation first;
-} Thread_queue_Operations;
+};
 
 /**
  *  This is the structure used to manage sets of tasks which are blocked

cpukit/score/include/rtems/score/threadqimpl.h

@@ -26,6 +26,10 @@
 #include <rtems/score/smp.h>
 #include <rtems/score/thread.h>
 
+#if defined(RTEMS_DEBUG)
+#include <string.h>
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -46,6 +50,13 @@ extern "C" {
  * relationships.
  */
 struct Thread_queue_Path {
+#if defined(RTEMS_SMP)
+  /**
+   * @brief The start of a thread queue path.
+   */
+  Thread_queue_Link Start;
+#endif
+
   /**
    * @brief A potential thread to update the priority via
    * _Thread_Update_priority().
@@ -84,10 +95,8 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_initialize(
 )
 {
 #if defined(RTEMS_DEBUG)
+  memset( queue_context, 0, sizeof( *queue_context ) );
   queue_context->expected_thread_dispatch_disable_level = 0xdeadbeef;
-#if defined(RTEMS_MULTIPROCESSING)
-  queue_context->mp_callout = NULL;
-#endif
 #else
   (void) queue_context;
 #endif
@@ -187,6 +196,135 @@ RTEMS_INLINE_ROUTINE void _Thread_queue_Context_set_MP_callout(
   } while ( 0 )
 #endif
 
+/**
+ * @brief Gets the thread wait queue of the thread queue context.
+ *
+ * On SMP configurations, the value is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \
+  ( queue_context )->Wait.queue
+#else
+#define _Thread_queue_Context_get_queue( queue_context, the_thread ) \
+  ( the_thread )->Wait.queue
+#endif
+
+/**
+ * @brief Gets the thread wait operations of the thread queue context.
+ *
+ * On SMP configurations, the value is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \
+  ( queue_context )->Wait.operations
+#else
+#define _Thread_queue_Context_get_operations( queue_context, the_thread ) \
+  ( the_thread )->Wait.operations
+#endif
+
+/**
+ * @brief Thread priority change by means of the thread queue context.
+ *
+ * On SMP configurations, the used data is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ * @param new_priority The new thread priority.
+ * @param prepend_it Prepend it to its priority group or not.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_priority_change( \
+    queue_context, \
+    the_thread, \
+    new_priority, \
+    prepend_it \
+  ) \
+    ( *( queue_context )->Wait.operations->priority_change )( \
+      the_thread, \
+      new_priority, \
+      prepend_it, \
+      ( queue_context )->Wait.queue \
+    )
+#else
+#define _Thread_queue_Context_priority_change( \
+    queue_context, \
+    the_thread, \
+    new_priority, \
+    prepend_it \
+  ) \
+    ( *( the_thread )->Wait.operations->priority_change )( \
+      the_thread, \
+      new_priority, \
+      prepend_it, \
+      ( the_thread )->Wait.queue \
+    )
+#endif
+
+/**
+ * @brief Thread queue extract by means of the thread queue context.
+ *
+ * On SMP configurations, the used data is stored in the thread queue context,
+ * otherwise in the thread itself.
+ *
+ * @param queue_context The thread queue context.
+ * @param the_thread The thread.
+ */
+#if defined(RTEMS_SMP)
+#define _Thread_queue_Context_extract( \
+    queue_context, \
+    the_thread \
+  ) \
+    ( *( queue_context )->Wait.operations->extract )( \
+      ( queue_context )->Wait.queue, \
+      the_thread \
+    )
+#else
+#define _Thread_queue_Context_extract( \
+    queue_context, \
+    the_thread \
+  ) \
+    ( *( the_thread )->Wait.operations->extract )( \
+      ( the_thread )->Wait.queue, \
+      the_thread \
+    )
+#endif
+
+#if defined(RTEMS_SMP)
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_add(
+  Chain_Control     *chain,
+  Thread_queue_Gate *gate
+)
+{
+  _Atomic_Store_uint( &gate->go_ahead, 0, ATOMIC_ORDER_RELAXED );
+  _Chain_Append_unprotected( chain, &gate->Node );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_open(
+  Thread_queue_Gate *gate
+)
+{
+  _Atomic_Store_uint( &gate->go_ahead, 1, ATOMIC_ORDER_RELAXED );
+}
+
+RTEMS_INLINE_ROUTINE void _Thread_queue_Gate_wait(
+  Thread_queue_Gate *gate
+)
+{
+  while ( _Atomic_Load_uint( &gate->go_ahead, ATOMIC_ORDER_RELAXED ) == 0 ) {
+    /* Wait */
+  }
+}
+#endif
+
 RTEMS_INLINE_ROUTINE void _Thread_queue_Heads_initialize(
   Thread_queue_Heads *heads
 )
@@ -911,6 +1049,10 @@ extern const Thread_queue_Operations _Thread_queue_Operations_priority;
 
 extern const Thread_queue_Operations _Thread_queue_Operations_priority_inherit;
 
+#if defined(RTEMS_SMP)
+extern const Thread_queue_Operations _Thread_queue_Operations_stale_queue;
+#endif
+
 /**@}*/
 
 #ifdef __cplusplus

cpukit/score/src/threadchangepriority.c

@@ -27,7 +27,8 @@ static Thread_Control *_Thread_Apply_priority_locked(
   Priority_Control               new_priority,
   void                          *arg,
   Thread_Change_priority_filter  filter,
-  bool                           prepend_it
+  bool                           prepend_it,
+  Thread_queue_Context          *queue_context
 )
 {
   /*
@@ -45,17 +46,11 @@ static Thread_Control *_Thread_Apply_priority_locked(
    *  we are not REALLY changing priority.
    */
   if ( ( *filter )( the_thread, &new_priority, arg ) ) {
-    Scheduler_Node *own_node;
-
-    own_node = _Scheduler_Thread_get_own_node( the_thread );
-    _Scheduler_Node_set_priority( own_node, new_priority, prepend_it );
-
-    the_thread->current_priority = new_priority;
-
-    ( *the_thread->Wait.operations->priority_change )(
+    _Thread_queue_Context_priority_change(
+      queue_context,
       the_thread,
       new_priority,
-      the_thread->Wait.queue
+      prepend_it
     );
   } else {
     the_thread = NULL;
@@ -72,19 +67,20 @@ Thread_Control *_Thread_Apply_priority(
   bool                           prepend_it
 )
 {
-  ISR_lock_Context  lock_context;
-  ISR_lock_Control *lock;
+  Thread_queue_Context  queue_context;
+  Thread_Control       *the_thread_to_update;
 
-  lock = _Thread_Lock_acquire( the_thread, &lock_context );
-  the_thread = _Thread_Apply_priority_locked(
+  _Thread_Wait_acquire( the_thread, &queue_context );
+  the_thread_to_update = _Thread_Apply_priority_locked(
     the_thread,
     new_priority,
     arg,
     filter,
-    prepend_it
+    prepend_it,
+    &queue_context
   );
-  _Thread_Lock_release( lock, &lock_context );
-  return the_thread;
+  _Thread_Wait_release( the_thread, &queue_context );
+  return the_thread_to_update;
 }
 
 void _Thread_Update_priority( Thread_Control *the_thread )

cpukit/score/src/threadinitialize.c

@@ -182,13 +182,11 @@ bool _Thread_Initialize(
   the_thread->Scheduler.control = scheduler;
   the_thread->Scheduler.own_node = scheduler_node;
   _Resource_Node_initialize( &the_thread->Resource_node );
-  _Atomic_Store_uintptr(
-    &the_thread->Lock.current.atomic,
-    (uintptr_t) &the_thread->Lock.Default,
-    ATOMIC_ORDER_RELAXED
+  _ISR_lock_Initialize(
+    &the_thread->Wait.Lock.Default,
+    "Thread Wait Default Lock"
   );
-  _SMP_ticket_lock_Initialize( &the_thread->Lock.Default );
-  _SMP_lock_Stats_initialize( &the_thread->Lock.Stats, "Thread Lock" );
+  _Chain_Initialize_empty( &the_thread->Wait.Lock.Pending_requests );
   _SMP_lock_Stats_initialize( &the_thread->Potpourri_stats, "Thread Potpourri" );
 #endif
 

cpukit/score/src/threadqenqueue.c

@@ -34,6 +34,51 @@
 #define THREAD_QUEUE_READY_AGAIN \
   (THREAD_WAIT_CLASS_OBJECT | THREAD_WAIT_STATE_READY_AGAIN)
 
+static void _Thread_queue_Path_release( Thread_queue_Path *path )
+{
+#if defined(RTEMS_SMP)
+  Thread_queue_Link *link;
+
+  link = &path->Start;
+
+  if ( link->owner != NULL ) {
+    _Thread_Wait_release_critical( link->owner, &link->Queue_context );
+  }
+#else
+  (void) path;
+#endif
+}
+
+static void _Thread_queue_Path_acquire(
+  Thread_Control     *the_thread,
+  Thread_queue_Queue *queue,
+  Thread_queue_Path  *path
+)
+{
+#if defined(RTEMS_SMP)
+  Thread_Control     *owner;
+  Thread_queue_Link  *link;
+
+  owner = queue->owner;
+
+  if ( owner == NULL ) {
+    return;
+  }
+
+  link = &path->Start;
+  link->owner = owner;
+
+  _Thread_Wait_acquire_default_critical(
+    owner,
+    &link->Queue_context.Lock_context
+  );
+#else
+  (void) the_thread;
+  (void) queue;
+  (void) path;
+#endif
+}
+
 void _Thread_queue_Enqueue_critical(
   Thread_queue_Queue            *queue,
   const Thread_queue_Operations *operations,
@@ -52,14 +97,13 @@ void _Thread_queue_Enqueue_critical(
   }
 #endif
 
-  _Thread_Lock_set( the_thread, &queue->Lock );
+  _Thread_Wait_claim( the_thread, queue, operations );
+
+  _Thread_queue_Path_acquire( the_thread, queue, &path );
+  ( *operations->enqueue )( queue, the_thread, &path );
+  _Thread_queue_Path_release( &path );
 
   the_thread->Wait.return_code = STATUS_SUCCESSFUL;
-  _Thread_Wait_set_queue( the_thread, queue );
-  _Thread_Wait_set_operations( the_thread, operations );
-
-  ( *operations->enqueue )( queue, the_thread, &path );
-
   _Thread_Wait_flags_set( the_thread, THREAD_QUEUE_INTEND_TO_BLOCK );
   cpu_self = _Thread_Dispatch_disable_critical( &queue_context->Lock_context );
   _Thread_queue_Queue_release( queue, &queue_context->Lock_context );
@@ -173,9 +217,7 @@ bool _Thread_queue_Do_extract_locked(
     unblock = true;
   }
 
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
-  _Thread_Lock_restore_default( the_thread );
+  _Thread_Wait_restore_default( the_thread );
 
   return unblock;
 }
@@ -227,30 +269,35 @@ void _Thread_queue_Extract_critical(
 
 void _Thread_queue_Extract( Thread_Control *the_thread )
 {
-  Thread_queue_Context  queue_context;
-  void                 *lock;
-  Thread_queue_Queue   *queue;
+  Thread_queue_Context queue_context;
 
   _Thread_queue_Context_initialize( &queue_context );
-  lock = _Thread_Lock_acquire( the_thread, &queue_context.Lock_context );
+  _Thread_Wait_acquire( the_thread, &queue_context );
 
-  queue = the_thread->Wait.queue;
-
-  if ( queue != NULL ) {
-    _SMP_Assert( lock == &queue->Lock );
+  if (
+    _Thread_queue_Context_get_queue( &queue_context, the_thread ) != NULL
+  ) {
+    bool unblock;
 
+    _Thread_Wait_remove_request( the_thread, &queue_context );
     _Thread_queue_Context_set_MP_callout(
       &queue_context,
       _Thread_queue_MP_callout_do_nothing
     );
-    _Thread_queue_Extract_critical(
-      queue,
-      the_thread->Wait.operations,
+    unblock = _Thread_queue_Extract_locked(
+      _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+      _Thread_queue_Context_get_operations( &queue_context, the_thread ),
       the_thread,
-      &queue_context
+      &queue_context.Lock_context
+    );
+    _Thread_queue_Unblock_critical(
+      unblock,
+      _Thread_queue_Context_get_queue( &queue_context, the_thread ),
+      the_thread,
+      &queue_context.Lock_context
     );
   } else {
-    _Thread_Lock_release( lock, &queue_context.Lock_context );
+    _Thread_Wait_release( the_thread, &queue_context );
   }
 }
 
@@ -273,8 +320,6 @@ Thread_Control *_Thread_queue_Do_dequeue(
   the_thread = _Thread_queue_First_locked( the_thread_queue, operations );
 
   if ( the_thread != NULL ) {
-    _SMP_Assert( the_thread->Lock.current.normal == &the_thread_queue->Queue.Lock );
-
     _Thread_queue_Extract_critical(
       &the_thread_queue->Queue,
       operations,

cpukit/score/src/threadqops.c

@@ -22,13 +22,16 @@
 #include <rtems/score/rbtreeimpl.h>
 #include <rtems/score/schedulerimpl.h>
 
-static void _Thread_queue_Do_nothing_priority_change(
+static void _Thread_queue_Default_priority_change(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 )
 {
-  /* Do nothing */
+  (void) queue;
+
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
 }
 
 static void _Thread_queue_Do_nothing_extract(
@@ -39,6 +42,31 @@ static void _Thread_queue_Do_nothing_extract(
   /* Do nothing */
 }
 
+#if defined(RTEMS_SMP)
+static void _Thread_queue_Stale_queue_priority_change(
+  Thread_Control     *the_thread,
+  Priority_Control    new_priority,
+  bool                prepend_it,
+  Thread_queue_Queue *queue
+)
+{
+  ISR_lock_Context lock_context;
+
+  (void) queue;
+
+  /*
+   * This operation is used to change the priority in case we have a thread
+   * queue context with a stale thread queue.  We own the thread queue lock of
+   * the former thread queue.  In addition, we need the thread wait default
+   * lock, see _Thread_Wait_restore_default().
+   */
+
+  _Thread_Wait_acquire_default_critical( the_thread, &lock_context );
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+  _Thread_Wait_release_default_critical( the_thread, &lock_context );
+}
+#endif
+
 static Thread_queue_Heads *_Thread_queue_Queue_enqueue(
   Thread_queue_Queue *queue,
   Thread_Control     *the_thread,
@@ -190,6 +218,7 @@ static bool _Thread_queue_Priority_less(
 static void _Thread_queue_Priority_priority_change(
   Thread_Control     *the_thread,
   Priority_Control    new_priority,
+  bool                prepend_it,
   Thread_queue_Queue *queue
 )
 {
@@ -198,6 +227,8 @@ static void _Thread_queue_Priority_priority_change(
 
   _Assert( heads != NULL );
 
+  _Scheduler_Thread_set_priority( the_thread, new_priority, prepend_it );
+
   priority_queue = _Thread_queue_Priority_queue( heads, the_thread );
 
   _RBTree_Extract(
@@ -353,22 +384,16 @@ static void _Thread_queue_Priority_inherit_enqueue(
 #endif
 
   if ( priority < owner->current_priority ) {
-    Scheduler_Node *own_node;
-
     path->update_priority = owner;
 
     owner->priority_restore_hint = true;
     _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
 
-    own_node = _Scheduler_Thread_get_own_node( owner );
-    _Scheduler_Node_set_priority( own_node, priority, false );
-
-    owner->current_priority = priority;
-
-    ( *owner->Wait.operations->priority_change )(
+    _Thread_queue_Context_priority_change(
+      &path->Start.Queue_context,
       owner,
       priority,
-      owner->Wait.queue
+      false
     );
   } else {
     path->update_priority = NULL;
@@ -385,24 +410,21 @@ void _Thread_queue_Boost_priority(
 
   if ( !_Chain_Has_only_one_node( &heads->Heads.Fifo ) ) {
     const Scheduler_Control *scheduler;
-    Scheduler_Node          *own_node;
     Priority_Control         boost_priority;
 
     the_thread->priority_restore_hint = true;
     _Atomic_Fence( ATOMIC_ORDER_ACQ_REL );
 
     scheduler = _Scheduler_Get_own( the_thread );
-    own_node = _Scheduler_Thread_get_own_node( the_thread );
     boost_priority = _Scheduler_Map_priority( scheduler, PRIORITY_PSEUDO_ISR );
-    _Scheduler_Node_set_priority( own_node, boost_priority, false );
 
-    the_thread->current_priority = boost_priority;
+    _Scheduler_Thread_set_priority( the_thread, boost_priority, false );
   }
 }
 #endif
 
 const Thread_queue_Operations _Thread_queue_Operations_default = {
-  .priority_change = _Thread_queue_Do_nothing_priority_change,
+  .priority_change = _Thread_queue_Default_priority_change,
   .extract = _Thread_queue_Do_nothing_extract
   /*
    * The default operations are only used in _Thread_Change_priority() and
@@ -412,7 +434,7 @@ const Thread_queue_Operations _Thread_queue_Operations_default = {
 };
 
 const Thread_queue_Operations _Thread_queue_Operations_FIFO = {
-  .priority_change = _Thread_queue_Do_nothing_priority_change,
+  .priority_change = _Thread_queue_Default_priority_change,
   .enqueue = _Thread_queue_FIFO_enqueue,
   .extract = _Thread_queue_FIFO_extract,
   .first = _Thread_queue_FIFO_first
@@ -431,3 +453,10 @@ const Thread_queue_Operations _Thread_queue_Operations_priority_inherit = {
   .extract = _Thread_queue_Priority_extract,
   .first = _Thread_queue_Priority_first
 };
+
+#if defined(RTEMS_SMP)
+const Thread_queue_Operations _Thread_queue_Operations_stale_queue = {
+  .priority_change = _Thread_queue_Stale_queue_priority_change,
+  .extract = _Thread_queue_Do_nothing_extract
+};
+#endif

cpukit/score/src/threadrestart.c

@@ -215,8 +215,7 @@ static void _Thread_Free( Thread_Control *the_thread )
   _Workspace_Free( the_thread->Start.tls_area );
 
 #if defined(RTEMS_SMP)
-  _SMP_ticket_lock_Destroy( &the_thread->Lock.Default );
-  _SMP_lock_Stats_destroy( &the_thread->Lock.Stats );
+  _ISR_lock_Destroy( &the_thread->Wait.Lock.Default );
   _SMP_lock_Stats_destroy( &the_thread->Potpourri_stats );
 #endif
 

cpukit/score/src/threadtimeout.c

@@ -22,28 +22,15 @@
 #include <rtems/score/threadimpl.h>
 #include <rtems/score/status.h>
 
-static void _Thread_Do_timeout( Thread_Control *the_thread )
-{
-  the_thread->Wait.return_code = STATUS_TIMEOUT;
-  ( *the_thread->Wait.operations->extract )(
-    the_thread->Wait.queue,
-    the_thread
-  );
-  _Thread_Wait_set_queue( the_thread, NULL );
-  _Thread_Wait_restore_default_operations( the_thread );
-  _Thread_Lock_restore_default( the_thread );
-}
-
 void _Thread_Timeout( Watchdog_Control *watchdog )
 {
-  Thread_Control    *the_thread;
-  void              *thread_lock;
-  ISR_lock_Context   lock_context;
-  Thread_Wait_flags  wait_flags;
-  bool               unblock;
+  Thread_Control       *the_thread;
+  Thread_queue_Context  queue_context;
+  Thread_Wait_flags     wait_flags;
+  bool                  unblock;
 
   the_thread = RTEMS_CONTAINER_OF( watchdog, Thread_Control, Timer.Watchdog );
-  thread_lock = _Thread_Lock_acquire( the_thread, &lock_context );
+  _Thread_Wait_acquire( the_thread, &queue_context );
 
   wait_flags = _Thread_Wait_flags_get( the_thread );
 
@@ -52,7 +39,9 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
     Thread_Wait_flags ready_again;
     bool              success;
 
-    _Thread_Do_timeout( the_thread );
+    _Thread_Wait_cancel( the_thread, &queue_context );
+
+    the_thread->Wait.return_code = STATUS_TIMEOUT;
 
     wait_class = wait_flags & THREAD_WAIT_CLASS_MASK;
     ready_again = wait_class | THREAD_WAIT_STATE_READY_AGAIN;
@@ -76,9 +65,10 @@ void _Thread_Timeout( Watchdog_Control *watchdog )
     unblock = false;
   }
 
-  _Thread_Lock_release( thread_lock, &lock_context );
+  _Thread_Wait_release( the_thread, &queue_context );
 
   if ( unblock ) {
+    _Thread_Wait_tranquilize( the_thread );
     _Thread_Unblock( the_thread );
 
 #if defined(RTEMS_MULTIPROCESSING)

testsuites/sptests/spthreadq01/init.c

@@ -68,13 +68,12 @@ static void wake_up_master(test_context *ctx)
 
 static rtems_id get_wait_id(test_context *ctx)
 {
-  ISR_lock_Context lock_context;
-  void *lock;
+  Thread_queue_Context queue_context;
   rtems_id id;
 
-  lock = _Thread_Lock_acquire(ctx->master, &lock_context);
+  _Thread_Wait_acquire(ctx->master, &queue_context);
   id = _Thread_Wait_get_id(ctx->master);
-  _Thread_Lock_release(lock, &lock_context);
+  _Thread_Wait_release(ctx->master, &queue_context);
 
   return id;
 }