[rt-smart] kernel virtual memory management layer (#6809)

synchronize virtual memory system works.
adding kernel virtual memory management layer for page-based MMU enabled architecture
porting libcpu MMU codes
porting lwp memory related codes

libcpu/arm/cortex-a/context_gcc.S

@@ -53,7 +53,7 @@ rt_hw_context_switch_to:
 #ifdef RT_USING_SMART
     bl      rt_thread_self
     mov     r4, r0
-    bl      lwp_mmu_switch
+    bl      lwp_aspace_switch
     mov     r0, r4
     bl      lwp_user_setting_restore
 #endif
@@ -116,7 +116,7 @@ rt_hw_context_switch:
 #ifdef RT_USING_SMART
     bl      rt_thread_self
     mov     r4, r0
-    bl      lwp_mmu_switch
+    bl      lwp_aspace_switch
     mov     r0, r4
     bl      lwp_user_setting_restore
 #endif

libcpu/arm/cortex-a/cpuport.h

@@ -10,6 +10,8 @@
 #ifndef  CPUPORT_H__
 #define  CPUPORT_H__
 
+#include <rtthread.h>
+
 /* the exception stack without VFP registers */
 struct rt_hw_exp_stack
 {

libcpu/arm/cortex-a/interrupt.c

@@ -70,8 +70,8 @@ void rt_hw_interrupt_init(void)
 
     /* initialize ARM GIC */
 #ifdef RT_USING_SMART
-    gic_dist_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_RW);
-    gic_cpu_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_cpu_base(), 0x1000, MMU_MAP_K_RW);
+    gic_dist_base = (uint32_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000);
+    gic_cpu_base = (uint32_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000);
 #else
     gic_dist_base = platform_get_gic_dist_base();
     gic_cpu_base = platform_get_gic_cpu_base();
@@ -100,7 +100,8 @@ void rt_hw_interrupt_init(void)
 
     /* initialize ARM GIC */
 #ifdef RT_USING_SMART
-    gic_dist_base = (uint32_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_RW);
+    gic_dist_base = (uint32_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000);
+    gic_cpu_base = (uint32_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000);
 #else
     gic_dist_base = platform_get_gic_dist_base();
 #endif

libcpu/arm/cortex-a/mmu.h

@@ -11,6 +11,7 @@
 #define __MMU_H_
 
 #include <rtthread.h>
+#include <mm_aspace.h>
 
 #define DESC_SEC       (0x2)
 #define MEMWBWA        ((1<<12)|(3<<2))     /* write back, write allocate */
@@ -87,27 +88,43 @@ struct mem_desc
 #define ARCH_TYPE_SUPERSECTION (1 << 18)
 
 #define ARCH_ADDRESS_WIDTH_BITS 32
+#define ARCH_VADDR_WIDTH        32
 
-typedef struct
-{
-    size_t *vtable;
-    size_t vstart;
-    size_t vend;
-    size_t pv_off;
-} rt_mmu_info;
+/**
+ * *info it's possible to map (-1ul & ~ARCH_PAGE_MASK) but a not aligned -1 is
+ * never returned on a successful mapping
+ */
+#define ARCH_MAP_FAILED ((void *)-1)
 
-int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off);
-int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size);
+int rt_hw_mmu_ioremap_init(struct rt_aspace *aspace, void *v_address, size_t size);
 void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_uint32_t size);
 
-#ifdef RT_USING_SMART
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr);
-void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr);
-#else
-void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr);
-#endif
+void rt_hw_mmu_setup(struct rt_aspace *aspace, struct mem_desc *mdesc, int desc_nr);
 
-void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size);
-void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr);
+int rt_hw_mmu_map_init(struct rt_aspace *aspace, void *v_address, size_t size, size_t *vtable, size_t pv_off);
+void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, size_t size, size_t attr);
+void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size);
+
+void rt_hw_aspace_switch(struct rt_aspace *aspace);
+void rt_hw_mmu_switch(void *tbl);
+
+void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr);
+void rt_hw_mmu_kernel_map_init(struct rt_aspace *aspace, size_t vaddr_start, size_t size);
+void *rt_hw_mmu_tbl_get();
+
+static inline void *_rt_kmem_v2p(void *vaddr)
+{
+    return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
+}
+
+static inline void *rt_kmem_v2p(void *vaddr)
+{
+    MM_PGTBL_LOCK(&rt_kernel_space);
+    void *paddr = _rt_kmem_v2p(vaddr);
+    MM_PGTBL_UNLOCK(&rt_kernel_space);
+    return paddr;
+}
+
+int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, enum rt_mmu_cntl cmd);
 
 #endif

libcpu/arm/cortex-a/start_gcc.S

@@ -258,34 +258,27 @@ rt_hw_set_process_id:
     MCR p15, 0, r0, c13, c0, 1
     mov pc, lr
 
+#endif
 .global rt_hw_mmu_switch
 rt_hw_mmu_switch:
-    mov r3, #0
-    mcr p15, 0, r3, c13, c0, 1   /* set contextid = 0, for synchronization*/ 
-    isb
-    
     orr r0, #0x18
-    mcr p15, 0, r0, c2, c0, 0    /* ttbr0 */
+    mcr p15, 0, r0, c2, c0, 0       // ttbr0
 
-    isb
-    mov r1, r1, LSL #0x8         
-    and r2, r2, #0xff             
-    orr r1, r1, r2                /* contextid.PROCID = pid, contextid.ASID = asid*/
-    mcr p15, 0, r1, c13, c0, 1    /* set contextid = r1*/ 
-    isb
-
-    mcr p15, 0, r0, c7, c5, 0    /* iciallu */
-    mcr p15, 0, r0, c7, c5, 6    /* bpiall */
+                                    //invalid tlb
+    mov r0, #0
+    mcr p15, 0, r0, c8, c7, 0
+    mcr p15, 0, r0, c7, c5, 0       //iciallu
+    mcr p15, 0, r0, c7, c5, 6       //bpiall 
 
     dsb
     isb
     mov pc, lr
+
 .global rt_hw_mmu_tbl_get
 rt_hw_mmu_tbl_get:
     mrc p15, 0, r0, c2, c0, 0    /* ttbr0 */
     bic r0, #0x18
     mov pc, lr
-#endif
 
 _halt:
     wfe
@@ -459,7 +452,7 @@ rt_hw_context_switch_interrupt_do:
     bl      rt_thread_self
 #ifdef RT_USING_SMART
     mov     r4, r0
-    bl      lwp_mmu_switch
+    bl      lwp_aspace_switch
     mov     r0, r4
     bl      lwp_user_setting_restore
 #endif

libcpu/arm/cortex-a/tlb.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2006-2022, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date           Author       Notes
+ * 2022-12-12     WangXiaoyao  the first version
+ */
+#ifndef __TLB_H__
+#define __TLB_H__
+
+#include "mm_aspace.h"
+#include <rtthread.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define dsb(scope)      __asm__ volatile("dsb " #scope : : : "memory")
+#define isb()           __asm__ volatile("isb" : : : "memory")
+#define STORE_CP32(r, name...) "mcr " RT_STRINGIFY(CP32(%r, name)) ";"
+
+#define WRITE_CP32(v, name...) do {                             \
+    register uint32_t _r = (v);                                 \
+    asm volatile(STORE_CP32(0, name) : : "r" (_r));             \
+} while (0)
+
+static inline void rt_hw_tlb_invalidate_all(void)
+{
+    asm volatile ("mcr p15, 0, r0, c8, c7, 0\ndsb\nisb" ::: "memory");
+}
+
+static inline void rt_hw_tlb_invalidate_all_local(void)
+{
+    rt_hw_tlb_invalidate_all();
+}
+
+static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace)
+{
+    rt_hw_tlb_invalidate_all();
+}
+
+static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
+                                              size_t size, size_t stride)
+{
+    rt_hw_tlb_invalidate_all();
+}
+
+#endif /* __TLB_H__ */

libcpu/arm/cortex-a/trap.c

@@ -8,12 +8,13 @@
  * 2013-07-20     Bernard      first version
  */
 
-#include <rtthread.h>
-#include <rthw.h>
-#include <board.h>
 #include <backtrace.h>
+#include <board.h>
+#include <rthw.h>
+#include <rtthread.h>
 
 #include "interrupt.h"
+#include "mm_fault.h"
 
 #ifdef RT_USING_FINSH
 extern long list_thread(void);
@@ -44,14 +45,23 @@ void check_user_fault(struct rt_hw_exp_stack *regs, uint32_t pc_adj, char *info)
 
 int check_user_stack(struct rt_hw_exp_stack *regs)
 {
-    void* dfar = RT_NULL;
+    void *dfar = RT_NULL;
+    asm volatile("MRC p15, 0, %0, c6, c0, 0" : "=r"(dfar));
 
-    asm volatile ("MRC p15, 0, %0, c6, c0, 0":"=r"(dfar));
-    if (arch_expand_user_stack(dfar))
+    if ((dfar >= (void *)USER_STACK_VSTART) && (dfar < (void *)USER_STACK_VEND))
     {
-        regs->pc -= 8;
-        return 1;
+        struct rt_mm_fault_msg msg = {
+            .fault_op = MM_FAULT_OP_WRITE,
+            .fault_type = MM_FAULT_TYPE_PAGE_FAULT,
+            .vaddr = dfar,
+        };
+        if (rt_mm_fault_try_fix(&msg))
+        {
+            regs->pc -= 8;
+            return 1;
+        }
     }
+
     return 0;
 }
 #endif
@@ -79,7 +89,7 @@ void rt_hw_show_register(struct rt_hw_exp_stack *regs)
         rt_kprintf("ttbr0:0x%08x\n", v);
         asm volatile ("MRC p15, 0, %0, c6, c0, 0":"=r"(v));
         rt_kprintf("dfar:0x%08x\n", v);
-        rt_kprintf("0x%08x -> 0x%08x\n", v, rt_hw_mmu_v2p(&mmu_info, (void *)v));
+        rt_kprintf("0x%08x -> 0x%08x\n", v, rt_kmem_v2p((void *)v));
     }
 #endif
 }