Protected mode: Redesign how the user space heap is accessed from the kernel code. It used to call memory management functions in user space via function pointers in the userspace interface. That is inefficient because the first thing that those memory management functions do is to trap back into the kernel to get the current PID. Worse, that operation can be fatal is certain fragile situations such as when a task is exitting.

The solution is to remove all of the memory management function calls from the interface.  Instead, the interface exports the userspace heap structure and then kernel size implementations of those memory management functions will operate on the userspace heap structure.  This avoids the unnecessary system calls and, more importantly, failures do to freeing memory when a test exits.

include/nuttx/kmalloc.h

@@ -89,31 +89,11 @@ extern "C"
 #define kumm_trysemaphore()      umm_trysemaphore()
 #define kumm_givesemaphore()     umm_givesemaphore()
 
-#ifdef CONFIG_BUILD_PROTECTED
-/* In the kernel-phase of the protected build, the these macros are defined
- * in userspace.h.  These macros version call into user-space via a header
- * at the beginning of the user-space blob.
- */
-
-#  define kumm_malloc(s)         umm_malloc(s)
-#  define kumm_zalloc(s)         umm_zalloc(s)
-#  define kumm_realloc(p,s)      umm_realloc(p,s)
-#  define kumm_memalign(a,s)     umm_memalign(a,s)
-#  define kumm_free(p)           umm_free(p)
-
-#else
-/* In the flat build (CONFIG_BUILD_FLAT) and in the kernel build
- * (CONFIG_BUILD_KERNEL), the following are declared in stdlib.h and are
- * directly callable.
- */
-
-# define kumm_malloc(s)          malloc(s)
-# define kumm_zalloc(s)          zalloc(s)
-# define kumm_realloc(p,s)       realloc(p,s)
-# define kumm_memalign(a,s)      memalign(a,s)
-# define kumm_free(p)            free(p)
-
-#endif
+#define kumm_malloc(s)           malloc(s)
+#define kumm_zalloc(s)           zalloc(s)
+#define kumm_realloc(p,s)        realloc(p,s)
+#define kumm_memalign(a,s)       memalign(a,s)
+#define kumm_free(p)             free(p)
 
 /* This family of allocators is used to manage kernel protected memory */
 

include/nuttx/mm/mm.h

@@ -276,7 +276,6 @@ extern "C"
 #define EXTERN extern
 #endif
 
-#if !defined(CONFIG_BUILD_PROTECTED) || !defined(__KERNEL__)
 /* User heap structure:
  *
  * - Flat build:  In the FLAT build, the user heap structure is a globally
@@ -288,6 +287,23 @@ extern "C"
  *   no global user heap structure.
  */
 
+ #if defined(CONFIG_ARCH_ADDRENV) && defined(CONFIG_BUILD_KERNEL)
+/* In the kernel build, there a multiple user heaps; one for each task
+ * group.  In this build configuration, the user heap structure lies
+ * in a reserved region at the beginning of the .bss/.data address
+ * space (CONFIG_ARCH_DATA_VBASE).  The size of that region is given by
+ * ARCH_DATA_RESERVE_SIZE
+ */
+
+#elif defined(CONFIG_BUILD_PROTECTED) && defined(__KERNEL__)
+/* In the protected mode, there are two heaps:  A kernel heap and a single
+ * user heap.  In that case the user heap structure lies in the user space
+ * (with a reference in the userspace interface).
+ */
+
+#else
+/* Otherwise, the user heap data structures are in common .bss */
+
 EXTERN struct mm_heap_s g_mmheap;
 #endif
 

include/nuttx/userspace.h

@@ -83,22 +83,13 @@
  * they can be called through the userspace structure.
  */
 
-#if defined(CONFIG_BUILD_PROTECTED) && defined(__KERNEL__)
-#  define umm_initialize(b,s) USERSPACE->mm_initialize(b,s)
-#  define umm_addregion(b,s)  USERSPACE->mm_addregion(b,s)
-#  define umm_trysemaphore()  USERSPACE->mm_trysemaphore()
-#  define umm_givesemaphore() USERSPACE->mm_givesemaphore()
-#  define umm_malloc(s)       USERSPACE->mm_malloc(s)
-#  define umm_zalloc(s)       USERSPACE->mm_zalloc(s)
-#  define umm_realloc(p,s)    USERSPACE->mm_realloc(p,s)
-#  define umm_memalign(a,s)   USERSPACE->mm_memalign(a,s)
-#  define umm_free(p)         USERSPACE->mm_free(p)
-#endif
-
 /****************************************************************************
- * Type Definitions
+ * Public Type Definitions
  ****************************************************************************/
-/* Every user-space blob starts with a header that provides information about
+
+struct mm_heaps_s; /* Forward reference */
+
+ /* Every user-space blob starts with a header that provides information about
  * the blob.  The form of that header is provided by struct userspace_s.  An
  * instance of this structure is expected to reside at CONFIG_NUTTX_USERSPACE.
  */
@@ -116,6 +107,10 @@ struct userspace_s
   uintptr_t us_bssstart;
   uintptr_t us_bssend;
 
+  /* Memory manager heap structure */
+
+  FAR struct mm_heap_s *us_heap;
+
   /* Task/thread startup routines */
 
   void (*task_startup)(main_t entrypt, int argc, FAR char *argv[])
@@ -132,21 +127,6 @@ struct userspace_s
     FAR siginfo_t *info, FAR void *ucontext);
 #endif
 
-  /* Memory manager entry points */
-
-  void (*mm_initialize)(FAR void *heap_start, size_t heap_size);
-  void (*mm_addregion)(FAR void *heap_start, size_t heap_size);
-  int  (*mm_trysemaphore)(void);
-  void (*mm_givesemaphore)(void);
-
-  FAR void *(*mm_malloc)(size_t size);
-  FAR void *(*mm_realloc)(FAR void *oldmem, size_t newsize);
-#if 0 /* Not yet integrated */
-  FAR void *(*mm_memalign)(size_t alignment, size_t size);
-#endif
-  FAR void *(*mm_zalloc)(size_t size);
-  void (*mm_free)(FAR void *mem);
-
   /* User-space work queue support */
 
 #ifdef CONFIG_LIB_USRWORK