score: Robust thread dispatch

On SMP configurations, it is a fatal error to call blocking operating
system with interrupts disabled, since this prevents delivery of
inter-processor interrupts. This could lead to executing threads which
are not allowed to execute resulting in undefined behaviour.

The ARM Cortex-M port has a similar problem, since the interrupt state
is not a part of the thread context.

Update #2811.

cpukit/sapi/src/interrtext.c

@@ -57,7 +57,8 @@ static const char *const internal_error_text[] = {
   "INTERNAL_ERROR_POSIX_INIT_THREAD_ENTRY_IS_NULL",
   "INTERNAL_ERROR_THREAD_QUEUE_DEADLOCK",
   "INTERNAL_ERROR_THREAD_QUEUE_ENQUEUE_STICKY_FROM_BAD_STATE",
-  "INTERNAL_ERROR_BAD_THREAD_DISPATCH_DISABLE_LEVEL"
+  "INTERNAL_ERROR_BAD_THREAD_DISPATCH_DISABLE_LEVEL",
+  "INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT"
 };
 
 const char *rtems_internal_error_text( rtems_fatal_code error )

cpukit/score/cpu/arm/rtems/score/cpu.h

@@ -130,6 +130,12 @@
 
 #define CPU_USE_DEFERRED_FP_SWITCH FALSE
 
+#if defined(ARM_MULTILIB_ARCH_V7M)
+  #define CPU_ENABLE_ROBUST_THREAD_DISPATCH TRUE
+#else
+  #define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+#endif
+
 #if defined(ARM_MULTILIB_HAS_WFI)
   #define CPU_PROVIDES_IDLE_THREAD_BODY TRUE
 #else

cpukit/score/cpu/bfin/rtems/score/cpu.h

@@ -233,6 +233,8 @@ extern "C" {
  */
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /**
  * Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/epiphany/rtems/score/cpu.h

@@ -205,6 +205,8 @@ extern "C" {
 
 #define CPU_USE_DEFERRED_FP_SWITCH       FALSE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/i386/rtems/score/cpu.h

@@ -92,6 +92,8 @@ extern "C" {
 #endif
 #endif /* __SSE__ */
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 #define CPU_STACK_GROWS_UP               FALSE
 
 /* FIXME: The Pentium 4 used 128 bytes, it this processor still relevant? */

cpukit/score/cpu/lm32/rtems/score/cpu.h

@@ -227,6 +227,8 @@ extern "C" {
  */
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /**
  * Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/m32c/rtems/score/cpu.h

@@ -249,6 +249,8 @@ extern "C" {
  */
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /**
  * Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/m68k/rtems/score/cpu.h

@@ -94,6 +94,7 @@ extern "C" {
 #define CPU_ALL_TASKS_ARE_FP             FALSE
 #define CPU_IDLE_TASK_IS_FP              FALSE
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
 
 #define CPU_PROVIDES_IDLE_THREAD_BODY    TRUE
 #define CPU_STACK_GROWS_UP               FALSE

cpukit/score/cpu/mips/rtems/score/cpu.h

@@ -226,6 +226,8 @@ extern "C" {
 
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/moxie/rtems/score/cpu.h

@@ -196,6 +196,8 @@ extern "C" {
  */
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/nios2/rtems/score/cpu.h

@@ -55,6 +55,8 @@ extern "C" {
 
 #define CPU_USE_DEFERRED_FP_SWITCH FALSE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 #define CPU_PROVIDES_IDLE_THREAD_BODY FALSE
 
 #define CPU_STACK_GROWS_UP FALSE

cpukit/score/cpu/no_cpu/rtems/score/cpu.h

@@ -248,6 +248,18 @@ extern "C" {
  */
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+/**
+ * @brief Enables a robust thread dispatch if set to TRUE.
+ *
+ * In general, it is an application bug to call blocking operating system
+ * services with interrupts disabled.  In most situations this only increases
+ * the interrupt latency.  However, on SMP configurations or on some CPU port
+ * like ARM Cortex-M it leads to undefined system behaviour.  It order to ease
+ * the application development, this error condition is checked at run-time in
+ * case this CPU port option is defined to TRUE.
+ */
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /**
  * Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/or1k/rtems/score/cpu.h

@@ -197,6 +197,8 @@ extern "C" {
 
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/powerpc/rtems/score/cpu.h

@@ -555,6 +555,8 @@ typedef struct {
 /* conservative setting (FALSE); probably doesn't affect performance too much */
 #define CPU_USE_DEFERRED_FP_SWITCH       FALSE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Processor defined structures required for cpukit/score.
  */

cpukit/score/cpu/sh/rtems/score/cpu.h

@@ -177,6 +177,8 @@ extern "C" {
 #define CPU_USE_DEFERRED_FP_SWITCH	TRUE
 #endif
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/sparc/rtems/score/cpu.h

@@ -173,6 +173,8 @@ extern "C" {
   #define CPU_USE_DEFERRED_FP_SWITCH TRUE
 #endif
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /**
  * Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/sparc64/rtems/score/cpu.h

@@ -136,6 +136,8 @@ extern "C" {
 
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /*
  *  Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/cpu/v850/rtems/score/cpu.h

@@ -225,6 +225,8 @@ extern "C" {
  */
 #define CPU_USE_DEFERRED_FP_SWITCH       TRUE
 
+#define CPU_ENABLE_ROBUST_THREAD_DISPATCH FALSE
+
 /**
  * Does this port provide a CPU dependent IDLE task implementation?
  *

cpukit/score/include/rtems/score/interr.h

@@ -166,7 +166,8 @@ typedef enum {
   INTERNAL_ERROR_POSIX_INIT_THREAD_ENTRY_IS_NULL,
   INTERNAL_ERROR_THREAD_QUEUE_DEADLOCK,
   INTERNAL_ERROR_THREAD_QUEUE_ENQUEUE_STICKY_FROM_BAD_STATE,
-  INTERNAL_ERROR_BAD_THREAD_DISPATCH_DISABLE_LEVEL
+  INTERNAL_ERROR_BAD_THREAD_DISPATCH_DISABLE_LEVEL,
+  INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT
 } Internal_errors_Core_list;
 
 typedef CPU_Uint32ptr Internal_errors_t;

cpukit/score/include/rtems/score/threaddispatch.h

@@ -28,6 +28,18 @@ extern "C" {
  * @{
  */
 
+#if defined(RTEMS_SMP) || ( CPU_ENABLE_ROBUST_THREAD_DISPATCH == TRUE )
+/**
+ * @brief Enables a robust thread dispatch.
+ *
+ * On each change of the thread dispatch disable level from one to zero the
+ * interrupt status is checked.  In case interrupts are disabled and SMP is
+ * enabled or the CPU port needs it, then the system terminates with the fatal
+ * internal error INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT.
+ */
+#define RTEMS_SCORE_ROBUST_THREAD_DISPATCH
+#endif
+
 /**
  * @brief Indicates if the executing thread is inside a thread dispatch
  * critical section.
@@ -206,7 +218,12 @@ RTEMS_INLINE_ROUTINE void _Thread_Dispatch_enable( Per_CPU_Control *cpu_self )
 
     _ISR_Local_disable( level );
 
-    if ( cpu_self->dispatch_necessary ) {
+    if (
+      cpu_self->dispatch_necessary
+#if defined(RTEMS_SCORE_ROBUST_THREAD_DISPATCH)
+        || !_ISR_Is_enabled( level )
+#endif
+    ) {
       _Thread_Do_dispatch( cpu_self, level );
     } else {
       cpu_self->thread_dispatch_disable_level = 0;

cpukit/score/src/threaddispatch.c

@@ -147,6 +147,21 @@ void _Thread_Do_dispatch( Per_CPU_Control *cpu_self, ISR_Level level )
 
   _Assert( cpu_self->thread_dispatch_disable_level == 1 );
 
+#if defined(RTEMS_SCORE_ROBUST_THREAD_DISPATCH)
+  if (
+    !_ISR_Is_enabled( level )
+#if defined(RTEMS_SMP)
+      && rtems_configuration_is_smp_enabled()
+#endif
+  ) {
+    _Terminate(
+      INTERNAL_ERROR_CORE,
+      false,
+      INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT
+    );
+  }
+#endif
+
   executing = cpu_self->executing;
 
   do {

testsuites/smptests/Makefile.am

@@ -20,6 +20,7 @@ SUBDIRS += smpfatal02
 SUBDIRS += smpfatal03
 SUBDIRS += smpfatal04
 SUBDIRS += smpfatal05
+SUBDIRS += smpfatal06
 SUBDIRS += smpfatal08
 SUBDIRS += smpipi01
 SUBDIRS += smpload01

testsuites/smptests/configure.ac

@@ -78,6 +78,7 @@ smpfatal02/Makefile
 smpfatal03/Makefile
 smpfatal04/Makefile
 smpfatal05/Makefile
+smpfatal06/Makefile
 smpfatal08/Makefile
 smpipi01/Makefile
 smpload01/Makefile

testsuites/smptests/smpfatal06/Makefile.am

@@ -0,0 +1,19 @@
+rtems_tests_PROGRAMS = smpfatal06
+smpfatal06_SOURCES = init.c
+
+dist_rtems_tests_DATA = smpfatal06.scn smpfatal06.doc
+
+include $(RTEMS_ROOT)/make/custom/@RTEMS_BSP@.cfg
+include $(top_srcdir)/../automake/compile.am
+include $(top_srcdir)/../automake/leaf.am
+
+AM_CPPFLAGS += -I$(top_srcdir)/../support/include
+
+LINK_OBJS = $(smpfatal06_OBJECTS)
+LINK_LIBS = $(smpfatal06_LDLIBS)
+
+smpfatal06$(EXEEXT): $(smpfatal06_OBJECTS) $(smpfatal06_DEPENDENCIES)
+	@rm -f smpfatal06$(EXEEXT)
+	$(make-exe)
+
+include $(top_srcdir)/../automake/local.am

testsuites/smptests/smpfatal06/init.c

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2016 embedded brains GmbH.  All rights reserved.
+ *
+ *  embedded brains GmbH
+ *  Dornierstr. 4
+ *  82178 Puchheim
+ *  Germany
+ *  <rtems@embedded-brains.de>
+ *
+ * The license and distribution terms for this file may be
+ * found in the file LICENSE in this distribution or at
+ * http://www.rtems.org/license/LICENSE.
+ */
+
+#ifdef HAVE_CONFIG_H
+  #include "config.h"
+#endif
+
+#define TESTS_USE_PRINTK
+#include "tmacros.h"
+
+#include <rtems.h>
+
+const char rtems_test_name[] = "SMPFATAL 6";
+
+static void Init(rtems_task_argument arg)
+{
+  rtems_interrupt_level level;
+
+  TEST_BEGIN();
+
+  rtems_interrupt_local_disable(level);
+  (void) level;
+  rtems_task_wake_after(1);
+}
+
+static void fatal_extension(
+  rtems_fatal_source source,
+  bool is_internal,
+  rtems_fatal_code code
+)
+{
+  if (
+    source == INTERNAL_ERROR_CORE
+      && !is_internal
+      && code == INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT
+  ) {
+    TEST_END();
+  }
+}
+
+#define CONFIGURE_APPLICATION_NEEDS_CLOCK_DRIVER
+#define CONFIGURE_APPLICATION_NEEDS_CONSOLE_DRIVER
+
+#define CONFIGURE_INITIAL_EXTENSIONS \
+  { .fatal = fatal_extension }, \
+  RTEMS_TEST_INITIAL_EXTENSION
+
+#define CONFIGURE_SMP_APPLICATION
+
+#define CONFIGURE_SMP_MAXIMUM_PROCESSORS 1
+
+#define CONFIGURE_MAXIMUM_TASKS 1
+
+#define CONFIGURE_RTEMS_INIT_TASKS_TABLE
+
+#define CONFIGURE_INIT
+
+#include <rtems/confdefs.h>

testsuites/smptests/smpfatal06/smpfatal06.doc

@@ -0,0 +1,13 @@
+This file describes the directives and concepts tested by this test set.
+
+test set name: smpfatal06
+
+directives:
+
+  - _Thread_Do_dispatch()
+
+concepts:
+
+  - Ensure that the fatal error INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT
+    occurs in case a blocking operating system service is called with
+    interrupts disabled.

testsuites/smptests/smpfatal06/smpfatal06.scn

@@ -0,0 +1,2 @@
+*** BEGIN OF TEST SMPFATAL 6 ***
+*** END OF TEST SMPFATAL 6 ***

testsuites/sptests/spinternalerror02/init.c

@@ -36,7 +36,7 @@ static void test_internal_error_text(void)
   } while ( text != text_last );
 
   rtems_test_assert(
-    error - 3 == INTERNAL_ERROR_BAD_THREAD_DISPATCH_DISABLE_LEVEL
+    error - 3 == INTERNAL_ERROR_BAD_THREAD_DISPATCH_ENVIRONMENT
   );
 }