Mirrors_Framework/rt-thread
1
0
Fork 0
mirror of https://github.com/RT-Thread/rt-thread.git synced 2026-09-24 05:04:10 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

[rt-smart] kernel virtual memory management layer (#6809)

Browse Source 
synchronize virtual memory system works.
adding kernel virtual memory management layer for page-based MMU enabled architecture
porting libcpu MMU codes
porting lwp memory related codes
This commit is contained in:
Shell
2023-01-08 21:08:55 -05:00
committed by GitHub
parent 7f9ccd3c80
commit 7450ef6c4d
121 changed files with 5947 additions and 7041 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libcpu/aarch64/common/context_gcc.S
+1 -26
View File
@@ -183,7 +183,7 @@ rt_hw_get_gtimer_frq:
#ifdef RT_USING_LWP
BL rt_thread_self
MOV X19, X0
BL lwp_mmu_switch
BL lwp_aspace_switch
MOV X0, X19
BL lwp_user_setting_restore
#endif
@@ -514,28 +514,3 @@ vector_serror:
STP X0, X1, [SP, #-0x10]!
BL rt_hw_trap_serror
b .
.global rt_hw_mmu_switch
rt_hw_mmu_switch:
MSR TTBR0_EL1, X0
MRS X1, TCR_EL1
CMP X0, XZR
ORR X1, X1, #(1 << 7)
BEQ 1f
BIC X1, X1, #(1 << 7)
1:
MSR TCR_EL1, X1
DSB SY
ISB
TLBI VMALLE1
DSB SY
ISB
IC IALLUIS
DSB SY
ISB
RET
.global rt_hw_mmu_tbl_get
rt_hw_mmu_tbl_get:
MRS X0, TTBR0_EL1
RET
libcpu/aarch64/common/cpu_ops_common.h
+1 -1
View File
@@ -8,7 +8,7 @@
static inline rt_uint64_t get_secondary_entry_pa(void)
{
rt_uint64_t secondary_entry_pa = (rt_uint64_t)rt_hw_mmu_v2p(&mmu_info, _secondary_cpu_entry);
rt_uint64_t secondary_entry_pa = (rt_uint64_t)rt_kmem_v2p(_secondary_cpu_entry);
if (!secondary_entry_pa)
{
libcpu/aarch64/common/cpuport.h
+13 -2
View File
@@ -11,6 +11,7 @@
#define CPUPORT_H__
#include <armv8.h>
#include <rtdef.h>
#ifdef RT_USING_SMP
typedef union {
@@ -29,12 +30,22 @@ rt_inline void rt_hw_isb(void)
rt_inline void rt_hw_dmb(void)
{
asm volatile ("dmb sy":::"memory");
asm volatile ("dmb ish":::"memory");
}
rt_inline void rt_hw_wmb(void)
{
asm volatile ("dmb ishst":::"memory");
}
rt_inline void rt_hw_rmb(void)
{
asm volatile ("dmb ishld":::"memory");
}
rt_inline void rt_hw_dsb(void)
{
asm volatile ("dsb sy":::"memory");
asm volatile ("dsb ish":::"memory");
}
#endif /*CPUPORT_H__*/
libcpu/aarch64/common/interrupt.c
+10 -4
View File
@@ -15,6 +15,12 @@
#include "gic.h"
#include "gicv3.h"
#ifdef RT_USING_SMART
#include "ioremap.h"
#else
#define rt_ioremap(x, ...) (x)
#endif
/* exception and interrupt handler table */
struct rt_irq_desc isr_table[MAX_HANDLERS];
@@ -103,11 +109,11 @@ void rt_hw_interrupt_init(void)
/* initialize ARM GIC */
#ifdef RT_USING_SMART
gic_dist_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_dist_base(), 0x2000, MMU_MAP_K_DEVICE);
gic_cpu_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_cpu_base(), 0x1000, MMU_MAP_K_DEVICE);
gic_dist_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_dist_base(), 0x2000);
gic_cpu_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_cpu_base(), 0x1000);
#ifdef BSP_USING_GICV3
gic_rdist_base = (rt_uint64_t)rt_hw_mmu_map(&mmu_info, 0, (void*)platform_get_gic_redist_base(),
RT_CPUS_NR * (2 << 16), MMU_MAP_K_DEVICE);
gic_rdist_base = (rt_uint64_t)rt_ioremap((void*)platform_get_gic_redist_base(),
RT_CPUS_NR * (2 << 16));
#endif
#else
gic_dist_base = platform_get_gic_dist_base();
libcpu/aarch64/common/mmu.c
+384 -734
View File
File diff suppressed because it is too large Load Diff
libcpu/aarch64/common/mmu.h
+52 -81
View File
@@ -11,6 +11,7 @@
#define __MMU_H_
#include <rtthread.h>
#include <mm_aspace.h>
/* normal memory wra mapping type */
#define NORMAL_MEM 0
@@ -25,6 +26,7 @@ struct mem_desc
unsigned long vaddr_end;
unsigned long paddr_start;
unsigned long attr;
struct rt_varea varea;
};
#define MMU_AF_SHIFT 10
@@ -37,60 +39,17 @@ struct mem_desc
#define MMU_AP_KRUN 2UL /* kernel r, user none */
#define MMU_AP_KRUR 3UL /* kernel r, user r */
#define MMU_MAP_K_RO (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KRUN << MMU_AP_SHIFT) |\
(NORMAL_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_K_RWCB (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KAUN << MMU_AP_SHIFT) |\
(NORMAL_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_K_RW (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KAUN << MMU_AP_SHIFT) |\
(NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_K_DEVICE (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KAUN << MMU_AP_SHIFT) |\
(DEVICE_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_U_RO (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KRUR << MMU_AP_SHIFT) |\
(NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_U_RWCB (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KAUA << MMU_AP_SHIFT) |\
(NORMAL_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_U_RW (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KAUA << MMU_AP_SHIFT) |\
(NORMAL_NOCACHE_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_U_DEVICE (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
(MMU_AP_KAUA << MMU_AP_SHIFT) |\
(DEVICE_MEM << MMU_MA_SHIFT)\
)
#define MMU_MAP_CUSTOM(ap, mtype) (\
(0x1UL << MMU_AF_SHIFT) |\
(0x2UL << MMU_SHARED_SHIFT) |\
((ap) << MMU_AP_SHIFT) |\
((mtype) << MMU_MA_SHIFT)\
)
#define MMU_MAP_CUSTOM(ap, mtype) \
((0x1UL << MMU_AF_SHIFT) | (0x2UL << MMU_SHARED_SHIFT) | \
((ap) << MMU_AP_SHIFT) | ((mtype) << MMU_MA_SHIFT))
#define MMU_MAP_K_RO MMU_MAP_CUSTOM(MMU_AP_KRUN, NORMAL_MEM)
#define MMU_MAP_K_RWCB MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM)
#define MMU_MAP_K_RW MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_NOCACHE_MEM)
#define MMU_MAP_K_DEVICE MMU_MAP_CUSTOM(MMU_AP_KAUN, DEVICE_MEM)
#define MMU_MAP_U_RO MMU_MAP_CUSTOM(MMU_AP_KRUR, NORMAL_NOCACHE_MEM)
#define MMU_MAP_U_RWCB MMU_MAP_CUSTOM(MMU_AP_KAUA, NORMAL_MEM)
#define MMU_MAP_U_RW MMU_MAP_CUSTOM(MMU_AP_KAUA, NORMAL_NOCACHE_MEM)
#define MMU_MAP_U_DEVICE MMU_MAP_CUSTOM(MMU_AP_KAUA, DEVICE_MEM)
#define ARCH_SECTION_SHIFT 21
#define ARCH_SECTION_SIZE (1 << ARCH_SECTION_SHIFT)
@@ -102,6 +61,7 @@ struct mem_desc
#define ARCH_PAGE_TBL_SIZE (1 << ARCH_PAGE_TBL_SHIFT)
#define ARCH_PAGE_TBL_MASK (ARCH_PAGE_TBL_SIZE - 1)
#define ARCH_VADDR_WIDTH 48
#define ARCH_ADDRESS_WIDTH_BITS 64
#define MMU_MAP_ERROR_VANOTALIGN -1
@@ -109,37 +69,48 @@ struct mem_desc
#define MMU_MAP_ERROR_NOPAGE -3
#define MMU_MAP_ERROR_CONFLICT -4
typedef struct
{
size_t *vtable;
size_t vstart;
size_t vend;
size_t pv_off;
} rt_mmu_info;
#define ARCH_MAP_FAILED ((void *)0x1ffffffffffff)
void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off);
void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr);
int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void* v_address, size_t size, size_t *vtable, size_t pv_off);
int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void* v_address, size_t size);
#ifdef RT_USING_SMART
void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void* p_addr, size_t size, size_t attr);
void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr);
#else
void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr);
#endif
void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void* v_addr, size_t size);
void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void* v_addr);
struct rt_aspace;
void rt_hw_mmu_ktbl_set(unsigned long tbl);
void *rt_hw_mmu_tbl_get();
void rt_hw_mmu_switch(void *mmu_table);
void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1,
unsigned long size, unsigned long pv_off);
void rt_hw_mmu_setup(struct rt_aspace *aspace, struct mem_desc *mdesc,
int desc_nr);
void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr);
void rt_hw_mmu_init(void);
int rt_hw_mmu_map_init(struct rt_aspace *aspace, void *v_address,
rt_size_t size, rt_size_t *vtable, rt_size_t pv_off);
void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr,
size_t size, size_t attr);
void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size);
void rt_hw_aspace_switch(struct rt_aspace *aspace);
void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr);
void rt_hw_mmu_kernel_map_init(struct rt_aspace *aspace, rt_size_t vaddr_start,
rt_size_t size);
void rt_hw_mmu_ktbl_set(unsigned long tbl);
extern rt_mmu_info mmu_info;
static inline void *rt_hw_mmu_tbl_get()
{
uintptr_t tbl;
__asm__ volatile("MRS %0, TTBR0_EL1" : "=r"(tbl));
return (void *)(tbl & ((1ul << 48) - 2));
}
static inline void *_rt_kmem_v2p(void *vaddr)
{
return rt_hw_mmu_v2p(&rt_kernel_space, vaddr);
}
static inline void *rt_kmem_v2p(void *vaddr)
{
MM_PGTBL_LOCK(&rt_kernel_space);
void *paddr = _rt_kmem_v2p(vaddr);
MM_PGTBL_UNLOCK(&rt_kernel_space);
return paddr;
}
int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
enum rt_mmu_cntl cmd);
#endif
libcpu/aarch64/common/tlb.h
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright (c) 2006-2022, RT-Thread Development Team
*
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
* Date Author Notes
* 2022-11-28 WangXiaoyao the first version
*/
#ifndef __TLB_H__
#define __TLB_H__
#include <rtthread.h>
#include <stddef.h>
#include <stdint.h>
#include "mm_aspace.h"
#include "mmu.h"
#define TLBI_ARG(addr, asid) \
({ \
uintptr_t arg = (uintptr_t)(addr) >> 12; \
arg &= (1ull << 44) - 1; \
arg |= (uintptr_t)(asid) << 48; \
(void *)arg; \
})
static inline void rt_hw_tlb_invalidate_all(void)
{
__asm__ volatile(
// ensure updates to pte completed
"dsb ishst\n"
"tlbi vmalle1is\n"
"dsb ish\n"
// after tlb in new context, refresh inst
"isb\n" ::
: "memory");
}
static inline void rt_hw_tlb_invalidate_all_local(void)
{
__asm__ volatile(
// ensure updates to pte completed
"dsb nshst\n"
"tlbi vmalle1is\n"
"dsb nsh\n"
// after tlb in new context, refresh inst
"isb\n" ::
: "memory");
}
static inline void rt_hw_tlb_invalidate_aspace(rt_aspace_t aspace)
{
rt_hw_tlb_invalidate_all_local();
}
static inline void rt_hw_tlb_invalidate_page(rt_aspace_t aspace, void *start)
{
start = TLBI_ARG(start, 0);
__asm__ volatile(
"dsb ishst\n"
"tlbi vaae1is, %0\n"
"dsb ish\n"
"isb\n" ::"r"(start)
: "memory");
}
static inline void rt_hw_tlb_invalidate_range(rt_aspace_t aspace, void *start,
size_t size, size_t stride)
{
if (size < ARCH_PAGE_SIZE)
{
rt_hw_tlb_invalidate_page(aspace, start);
}
else
{
rt_hw_tlb_invalidate_aspace(aspace);
}
}
#endif /* __TLB_H__ */
libcpu/aarch64/common/trap.c
+45 -4
View File
@@ -14,6 +14,7 @@
#include <armv8.h>
#include "interrupt.h"
#include "mm_fault.h"
#include <backtrace.h>
@@ -48,6 +49,29 @@ void check_user_fault(struct rt_hw_exp_stack *regs, uint32_t pc_adj, char *info)
}
}
int _get_type(unsigned long esr)
{
int ret;
int fsc = esr & 0x3f;
switch (fsc)
{
case 0x4:
case 0x5:
case 0x6:
case 0x7:
ret = MM_FAULT_TYPE_PAGE_FAULT;
break;
case 0x9:
case 0xa:
case 0xb:
ret = MM_FAULT_TYPE_ACCESS_FAULT;
break;
default:
ret = MM_FAULT_TYPE_GENERIC;
}
return ret;
}
int check_user_stack(unsigned long esr, struct rt_hw_exp_stack *regs)
{
unsigned char ec;
@@ -55,19 +79,36 @@ int check_user_stack(unsigned long esr, struct rt_hw_exp_stack *regs)
int ret = 0;
ec = (unsigned char)((esr >> 26) & 0x3fU);
enum rt_mm_fault_op fault_op;
enum rt_mm_fault_type fault_type;
switch (ec)
{
case 0x20:
fault_op = MM_FAULT_OP_EXECUTE;
fault_type = _get_type(esr);
break;
case 0x21:
case 0x24:
fault_op = MM_FAULT_OP_WRITE;
fault_type = _get_type(esr);
break;
default:
fault_op = 0;
break;
}
if (fault_op)
{
asm volatile("mrs %0, far_el1":"=r"(dfar));
if (arch_expand_user_stack(dfar))
struct rt_mm_fault_msg msg = {
.fault_op = fault_op,
.fault_type = fault_type,
.vaddr = dfar,
};
if (rt_mm_fault_try_fix(&msg))
{
ret = 1;
}
break;
default:
break;
}
return ret;
}
libcpu/aarch64/cortex-a/entry_point.S
+10 -9
View File
@@ -91,10 +91,11 @@ __start:
msr cpacr_el1, x1
/* clear bss */
ldr x1, =__bss_start /* get bss start address */
ldr x2, =__bss_end
adrp x1, __bss_start /* get bss start address */
add x1, x1, #:lo12:__bss_start
adrp x2, __bss_end
add x1, x1, #:lo12:__bss_end
sub x2, x2, x1 /* get bss size */
add x1, x1, x9
and x3, x2, #7 /* x3 is < 7 */
ldr x4, =~0x7
@@ -140,14 +141,14 @@ __start:
orr x1, x1, #(1 << 0) /* M */
msr sctlr_el1, x1 /* enable MMU */
dsb sy
isb sy
dsb ish
isb
ic ialluis /* Invalidate all instruction caches in Inner Shareable domain to Point of Unification */
dsb sy
isb sy
dsb ish
isb
tlbi vmalle1 /* Invalidate all stage 1 translations used at EL1 with the current VMID */
dsb sy
isb sy
dsb ish
isb
ret
after_mmu_enable: