Mirrors_Framework/rt-thread
1
0
Fork 0
mirror of https://github.com/RT-Thread/rt-thread.git synced 2026-09-21 02:06:14 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

[components/mm] support for scalable memory management (#7277)

Browse Source 
* [mm/page] multi-list page manager
[mm/page] page debugger
[libcpu/aarch64] hugepage support

* [quality] remove void-arith

* [format] remove kasan codes
This commit is contained in:
Shell
2023-04-22 23:59:11 +08:00
committed by GitHub
parent 90c69a0588
commit 470454d201
20 changed files with 603 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libcpu/aarch64/common/cache.h
+4 -1
View File
@@ -23,7 +23,10 @@ void rt_hw_cpu_dcache_invalidate(void *start_addr, unsigned long size);
static inline void rt_hw_icache_invalidate_all(void)
{
__asm_invalidate_icache_all();
/* wait for any modification complete */
__asm__ volatile ("dsb ishst");
__asm__ volatile ("ic iallu");
__asm__ volatile ("isb");
}
void rt_hw_cpu_icache_invalidate(void *addr, rt_size_t size);
libcpu/aarch64/common/cpu.c
+4 -1
View File
@@ -132,7 +132,10 @@ static rt_uint64_t _read_be_number(void *start, int size)
{
rt_uint64_t buf = 0;
for (; size > 0; size--)
buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start++);
{
buf = (buf << 32) | fdt32_to_cpu(*(uint32_t *)start);
start = (uint32_t *)start + 1;
}
return buf;
}
libcpu/aarch64/common/mmu.c
+134 -39
View File
@@ -1,13 +1,14 @@
/*
* Copyright (c) 2006-2018, RT-Thread Development Team
* Copyright (c) 2006-2023, RT-Thread Development Team
*
* SPDX-License-Identifier: Apache-2.0
*
* Change Logs:
* Date Author Notes
* 2012-01-10 bernard porting to AM1808
* 2021-11-28 GuEe-GUI first version
* 2022-12-10 WangXiaoyao porting to MM
*/
#include <board.h>
#include <rthw.h>
#include <rtthread.h>
@@ -79,6 +80,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
break;
}
/* next table entry in current level */
level_info[level].pos = cur_lv_tbl + off;
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
@@ -119,8 +121,7 @@ static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
return;
}
static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
unsigned long attr)
static int _kernel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
{
int ret = 0;
int level;
@@ -145,7 +146,7 @@ static int _kenrel_map_4K(unsigned long *lv0_tbl, void *vaddr, void *paddr,
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = (unsigned long)rt_pages_alloc(0);
page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!page)
{
ret = MMU_MAP_ERROR_NOPAGE;
@@ -188,19 +189,104 @@ err:
return ret;
}
static int _kernel_map_2M(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr)
{
int ret = 0;
int level;
unsigned long *cur_lv_tbl = lv0_tbl;
unsigned long page;
unsigned long off;
unsigned long va = (unsigned long)vaddr;
unsigned long pa = (unsigned long)paddr;
int level_shift = MMU_ADDRESS_BITS;
if (va & ARCH_SECTION_MASK)
{
return MMU_MAP_ERROR_VANOTALIGN;
}
if (pa & ARCH_SECTION_MASK)
{
return MMU_MAP_ERROR_PANOTALIGN;
}
for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
{
off = (va >> level_shift);
off &= MMU_LEVEL_MASK;
if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
{
page = (unsigned long)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!page)
{
ret = MMU_MAP_ERROR_NOPAGE;
goto err;
}
rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
}
else
{
page = cur_lv_tbl[off];
page &= MMU_ADDRESS_MASK;
/* page to va */
page -= PV_OFFSET;
rt_page_ref_inc((void *)page, 0);
}
page = cur_lv_tbl[off];
if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
{
/* is block! error! */
ret = MMU_MAP_ERROR_CONFLICT;
goto err;
}
cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
level_shift -= MMU_LEVEL_SHIFT;
}
/* now is level page */
attr &= MMU_ATTRIB_MASK;
pa |= (attr | MMU_TYPE_BLOCK); /* block */
off = (va >> ARCH_SECTION_SHIFT);
off &= MMU_LEVEL_MASK;
cur_lv_tbl[off] = pa;
rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
return ret;
err:
_kenrel_unmap_4K(lv0_tbl, (void *)va);
return ret;
}
void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
size_t attr)
{
int ret = -1;
void *unmap_va = v_addr;
size_t npages = size >> ARCH_PAGE_SHIFT;
size_t npages;
size_t stride;
int (*mapper)(unsigned long *lv0_tbl, void *vaddr, void *paddr, unsigned long attr);
if (((rt_ubase_t)v_addr & ARCH_SECTION_MASK) || (size & ARCH_SECTION_MASK))
{
/* legacy 4k mapping */
npages = size >> ARCH_PAGE_SHIFT;
stride = ARCH_PAGE_SIZE;
mapper = _kernel_map_4K;
}
else
{
/* 2m huge page */
npages = size >> ARCH_SECTION_SHIFT;
stride = ARCH_SECTION_SIZE;
mapper = _kernel_map_2M;
}
// TODO trying with HUGEPAGE here
while (npages--)
{
MM_PGTBL_LOCK(aspace);
ret = _kenrel_map_4K(aspace->page_table, v_addr, p_addr, attr);
ret = mapper(aspace->page_table, v_addr, p_addr, attr);
MM_PGTBL_UNLOCK(aspace);
if (ret != 0)
@@ -213,12 +299,12 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
MM_PGTBL_LOCK(aspace);
_kenrel_unmap_4K(aspace->page_table, (void *)unmap_va);
MM_PGTBL_UNLOCK(aspace);
unmap_va += ARCH_PAGE_SIZE;
unmap_va = (char *)unmap_va + stride;
}
break;
}
v_addr += ARCH_PAGE_SIZE;
p_addr += ARCH_PAGE_SIZE;
v_addr = (char *)v_addr + stride;
p_addr = (char *)p_addr + stride;
}
if (ret == 0)
@@ -244,7 +330,7 @@ void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size)
MM_PGTBL_LOCK(aspace);
_kenrel_unmap_4K(aspace->page_table, v_addr);
MM_PGTBL_UNLOCK(aspace);
v_addr += ARCH_PAGE_SIZE;
v_addr = (char *)v_addr + ARCH_PAGE_SIZE;
}
}
@@ -254,7 +340,7 @@ void rt_hw_aspace_switch(rt_aspace_t aspace)
{
void *pgtbl = aspace->page_table;
pgtbl = rt_kmem_v2p(pgtbl);
uintptr_t tcr;
rt_ubase_t tcr;
__asm__ volatile("msr ttbr0_el1, %0" ::"r"(pgtbl) : "memory");
@@ -337,20 +423,19 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
rt_page_cleanup();
}
#ifdef RT_USING_SMART
static inline void _init_region(void *vaddr, size_t size)
static void _init_region(void *vaddr, size_t size)
{
rt_ioremap_start = vaddr;
rt_ioremap_size = size;
rt_mpr_start = rt_ioremap_start - rt_mpr_size;
rt_mpr_start = (char *)rt_ioremap_start - rt_mpr_size;
}
#else
#define RTOS_VEND ((void *)0xfffffffff000UL)
#define RTOS_VEND (0xfffffffff000UL)
static inline void _init_region(void *vaddr, size_t size)
{
rt_mpr_start = RTOS_VEND - rt_mpr_size;
rt_mpr_start = (void *)(RTOS_VEND - rt_mpr_size);
}
#endif
@@ -395,7 +480,7 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, size_t size,
rt_aspace_init(aspace, (void *)KERNEL_VADDR_START, 0 - KERNEL_VADDR_START,
vtable);
#else
rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - (void *)0x1000, vtable);
rt_aspace_init(aspace, (void *)0x1000, RTOS_VEND - 0x1000ul, vtable);
#endif
_init_region(v_address, size);
@@ -586,26 +671,35 @@ void *rt_hw_mmu_v2p(rt_aspace_t aspace, void *v_addr)
{
int level_shift;
unsigned long paddr;
unsigned long *pte = _query(aspace, v_addr, &level_shift);
if (pte)
if (aspace == &rt_kernel_space)
{
paddr = *pte & MMU_ADDRESS_MASK;
paddr |= (uintptr_t)v_addr & ((1ul << level_shift) - 1);
paddr = (unsigned long)rt_hw_mmu_kernel_v2p(v_addr);
}
else
{
paddr = (unsigned long)ARCH_MAP_FAILED;
unsigned long *pte = _query(aspace, v_addr, &level_shift);
if (pte)
{
paddr = *pte & MMU_ADDRESS_MASK;
paddr |= (rt_ubase_t)v_addr & ((1ul << level_shift) - 1);
}
else
{
paddr = (unsigned long)ARCH_MAP_FAILED;
}
}
return (void *)paddr;
}
static int _noncache(uintptr_t *pte)
static int _noncache(rt_ubase_t *pte)
{
int err = 0;
const uintptr_t idx_shift = 2;
const uintptr_t idx_mask = 0x7 << idx_shift;
uintptr_t entry = *pte;
const rt_ubase_t idx_shift = 2;
const rt_ubase_t idx_mask = 0x7 << idx_shift;
rt_ubase_t entry = *pte;
if ((entry & idx_mask) == (NORMAL_MEM << idx_shift))
{
*pte = (entry & ~idx_mask) | (NORMAL_NOCACHE_MEM << idx_shift);
@@ -618,12 +712,12 @@ static int _noncache(uintptr_t *pte)
return err;
}
static int _cache(uintptr_t *pte)
static int _cache(rt_ubase_t *pte)
{
int err = 0;
const uintptr_t idx_shift = 2;
const uintptr_t idx_mask = 0x7 << idx_shift;
uintptr_t entry = *pte;
const rt_ubase_t idx_shift = 2;
const rt_ubase_t idx_mask = 0x7 << idx_shift;
rt_ubase_t entry = *pte;
if ((entry & idx_mask) == (NORMAL_NOCACHE_MEM << idx_shift))
{
*pte = (entry & ~idx_mask) | (NORMAL_MEM << idx_shift);
@@ -636,7 +730,7 @@ static int _cache(uintptr_t *pte)
return err;
}
static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = {
static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_ubase_t *pte) = {
[MMU_CNTL_CACHE] = _cache,
[MMU_CNTL_NONCACHE] = _noncache,
};
@@ -646,17 +740,18 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
{
int level_shift;
int err = -RT_EINVAL;
void *vend = vaddr + size;
rt_ubase_t vstart = (rt_ubase_t)vaddr;
rt_ubase_t vend = vstart + size;
int (*handler)(uintptr_t * pte);
int (*handler)(rt_ubase_t * pte);
if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
{
handler = control_handler[cmd];
while (vaddr < vend)
while (vstart < vend)
{
uintptr_t *pte = _query(aspace, vaddr, &level_shift);
void *range_end = vaddr + (1ul << level_shift);
rt_ubase_t *pte = _query(aspace, (void *)vstart, &level_shift);
rt_ubase_t range_end = vstart + (1ul << level_shift);
RT_ASSERT(range_end <= vend);
if (pte)
@@ -664,7 +759,7 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
err = handler(pte);
RT_ASSERT(err == RT_EOK);
}
vaddr = range_end;
vstart = range_end;
}
}
else
libcpu/aarch64/common/mmu.h
+22
View File
@@ -97,6 +97,28 @@ static inline void *rt_hw_mmu_tbl_get()
return (void *)(tbl & ((1ul << 48) - 2));
}
static inline void *rt_hw_mmu_kernel_v2p(void *v_addr)
{
rt_ubase_t par;
void *paddr;
asm volatile("at s1e1w, %0"::"r"(v_addr):"memory");
asm volatile("mrs %0, par_el1":"=r"(par)::"memory");
if (par & 0x1)
{
paddr = ARCH_MAP_FAILED;
}
else
{
#define MMU_ADDRESS_MASK 0x0000fffffffff000UL
par &= MMU_ADDRESS_MASK;
par |= (rt_ubase_t)v_addr & ARCH_PAGE_MASK;
paddr = (void *)par;
}
return paddr;
}
int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
enum rt_mmu_cntl cmd);