[components/mm] support for scalable memory management (#7277)

* [mm/page] multi-list page manager
[mm/page] page debugger
[libcpu/aarch64] hugepage support

* [quality] remove void-arith

* [format] remove kasan codes
This commit is contained in:
Shell
2023-04-22 23:59:11 +08:00
committed by GitHub
parent 90c69a0588
commit 470454d201
20 changed files with 603 additions and 230 deletions
+2 -1
View File
@@ -11,6 +11,7 @@
#include <rthw.h>
#include <rtthread.h>
#include <cpuport.h>
#include <mm_aspace.h>
#ifdef RT_USING_VIRTIO_NET
@@ -106,7 +107,7 @@ static struct pbuf *virtio_net_rx(rt_device_t dev)
#ifdef RT_USING_SMP
level = rt_spin_lock_irqsave(&virtio_dev->spinlock);
#endif
rt_memcpy(p->payload, (void *)VIRTIO_PA2VA(queue_rx->desc[id].addr), len);
rt_memcpy(p->payload, (void *)queue_rx->desc[id].addr - PV_OFFSET, len);
queue_rx->used_idx++;
@@ -26,7 +26,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
{
size_t *mmu_table;
mmu_table = (size_t *)rt_pages_alloc(0);
mmu_table = (size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!mmu_table)
{
return -RT_ENOMEM;
+1 -1
View File
@@ -91,7 +91,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
{
rt_ubase_t *mmu_table;
mmu_table = (rt_ubase_t *)rt_pages_alloc(0);
mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!mmu_table)
{
return -RT_ENOMEM;
+1 -1
View File
@@ -82,7 +82,7 @@ int arch_user_space_init(struct rt_lwp *lwp)
{
rt_size_t *mmu_table;
mmu_table = (rt_size_t *)rt_pages_alloc(0);
mmu_table = (rt_size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!mmu_table)
{
return -1;
+14 -1
View File
@@ -9,6 +9,7 @@
* 2018-11-02 heyuanjie fix complie error in iar
* 2021-02-03 lizhirui add 64-bit arch support and riscv64 arch support
* 2021-08-26 linzhenxing add lwp_setcwd\lwp_getcwd
* 2023-02-20 wangxiaoyao inv icache before new app startup
*/
#include <rthw.h>
@@ -1097,10 +1098,22 @@ static void _lwp_thread_entry(void *parameter)
icache_invalid_all();
}
/**
* without ASID support, it will be a special case when trying to run application
* and exit multiple times and a same page frame allocated to it bound to
* different text segment. Then we are in a situation where icache contains
* out-of-dated data and must be handle by the running core itself.
* with ASID support, this should be a rare case that ASID & page frame both
* identical to previous running application.
*
* For a new application loaded into memory, icache are seen as empty. And there
* should be nothing in the icache entry to match. So this icache invalidation
* operation should have barely influence.
*/
rt_hw_icache_invalidate_all();
#ifdef ARCH_MM_MMU
arch_start_umode(lwp->args, lwp->text_entry, (void *)USER_STACK_VEND, tid->stack_addr + tid->stack_size);
arch_start_umode(lwp->args, lwp->text_entry, (void *)USER_STACK_VEND, (char *)tid->stack_addr + tid->stack_size);
#else
arch_start_umode(lwp->args, lwp->text_entry, lwp->data_entry, (void *)((uint32_t)lwp->data_entry + lwp->data_size));
#endif /* ARCH_MM_MMU */
+3 -4
View File
@@ -6,6 +6,7 @@
* Change Logs:
* Date Author Notes
* 2019-10-12 Jesven first version
* 2023-02-20 wangxiaoyao adapt to mm
*/
#include <rthw.h>
#include <rtthread.h>
@@ -17,8 +18,6 @@
#include <lwp_user_mm.h>
#include <mmu.h>
#include <mm_aspace.h>
#include <mm_flag.h>
/* the kernel structure to represent a share-memory */
struct lwp_shm_struct
@@ -64,7 +63,7 @@ static void on_shm_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg
/* map all share page frames to user space in a time */
void *page = (void *)shm->addr;
void *pg_paddr = page + PV_OFFSET;
void *pg_paddr = (char *)page + PV_OFFSET;
err = rt_varea_map_range(varea, varea->start, pg_paddr, shm->size);
if (err == RT_EOK)
@@ -140,7 +139,7 @@ static int _lwp_shmget(size_t key, size_t size, int create)
/* allocate pages up to 2's exponent to cover the required size */
bit = rt_page_bits(size);
page_addr = rt_pages_alloc(bit); /* virtual address */
page_addr = rt_pages_alloc_ext(bit, PAGE_ANY_AVAILABLE); /* virtual address */
if (!page_addr)
{
goto err;
+5 -5
View File
@@ -306,7 +306,7 @@ static void _crt_thread_entry(void *parameter)
user_stack &= ~7; //align 8
#ifdef ARCH_MM_MMU
arch_crt_start_umode(parameter, tid->user_entry, (void *)user_stack, tid->stack_addr + tid->stack_size);
arch_crt_start_umode(parameter, tid->user_entry, (void *)user_stack, (char *)tid->stack_addr + tid->stack_size);
#else
set_user_context((void*)user_stack);
arch_start_umode(parameter, tid->user_entry, ((struct rt_lwp *)tid->lwp)->data_entry, (void*)user_stack);
@@ -1861,7 +1861,7 @@ static char *_insert_args(int new_argc, char *new_argv[], struct lwp_args_info *
{
goto quit;
}
page = rt_pages_alloc(0); /* 1 page */
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
if (!page)
{
goto quit;
@@ -2065,7 +2065,7 @@ int load_ldso(struct rt_lwp *lwp, char *exec_name, char *const argv[], char *con
}
}
page = rt_pages_alloc(0); /* 1 page */
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
if (!page)
{
SET_ERRNO(ENOMEM);
@@ -2252,7 +2252,7 @@ sysret_t sys_execve(const char *path, char *const argv[], char *const envp[])
SET_ERRNO(EINVAL);
goto quit;
}
page = rt_pages_alloc(0); /* 1 page */
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); /* 1 page */
if (!page)
{
SET_ERRNO(ENOMEM);
@@ -2396,7 +2396,7 @@ sysret_t sys_execve(const char *path, char *const argv[], char *const envp[])
arch_start_umode(lwp->args,
lwp->text_entry,
(void*)USER_STACK_VEND,
thread->stack_addr + thread->stack_size);
(char *)thread->stack_addr + thread->stack_size);
/* never reach here */
}
return -EINVAL;
+8 -7
View File
@@ -10,6 +10,7 @@
* 2021-02-12 lizhirui add 64-bit support for lwp_brk
* 2021-02-19 lizhirui add riscv64 support for lwp_user_accessable and lwp_get_from_user
* 2021-06-07 lizhirui modify user space bound check
* 2022-12-25 wangxiaoyao adapt to new mm
*/
#include <rtthread.h>
@@ -122,7 +123,7 @@ static void _user_do_page_fault(struct rt_varea *varea,
if (lwp_objs->source)
{
void *paddr = rt_hw_mmu_v2p(lwp_objs->source, msg->fault_vaddr);
char *paddr = rt_hw_mmu_v2p(lwp_objs->source, msg->fault_vaddr);
if (paddr != ARCH_MAP_FAILED)
{
void *vaddr;
@@ -130,7 +131,7 @@ static void _user_do_page_fault(struct rt_varea *varea,
if (!(varea->flag & MMF_TEXT))
{
void *cp = rt_pages_alloc(0);
void *cp = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (cp)
{
memcpy(cp, vaddr, ARCH_PAGE_SIZE);
@@ -220,9 +221,9 @@ int lwp_unmap_user(struct rt_lwp *lwp, void *va)
static void _dup_varea(rt_varea_t varea, struct rt_lwp *src_lwp,
rt_aspace_t dst)
{
void *vaddr = varea->start;
void *vend = vaddr + varea->size;
if (vaddr < (void *)USER_STACK_VSTART || vaddr >= (void *)USER_STACK_VEND)
char *vaddr = varea->start;
char *vend = vaddr + varea->size;
if (vaddr < (char *)USER_STACK_VSTART || vaddr >= (char *)USER_STACK_VEND)
{
while (vaddr != vend)
{
@@ -430,7 +431,7 @@ void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa,
size_t map_size, int cached)
{
int err;
void *va;
char *va;
size_t offset = 0;
if (!map_size)
@@ -458,7 +459,7 @@ void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa,
rt_size_t attr = cached ? MMU_MAP_U_RWCB : MMU_MAP_U_RW;
err =
rt_aspace_map_phy(lwp->aspace, &hint, attr, MM_PA_TO_OFF(map_pa), &va);
rt_aspace_map_phy(lwp->aspace, &hint, attr, MM_PA_TO_OFF(map_pa), (void **)&va);
if (err != RT_EOK)
{
va = RT_NULL;
+2 -2
View File
@@ -78,7 +78,7 @@ static struct rt_varea *search(struct util_avl_root *root,
{
rt_varea_t varea = VAREA_ENTRY(node);
int cmp = compare(range.start, range.end, varea->start,
varea->start + varea->size - 1);
(char *)varea->start + varea->size - 1);
if (cmp < 0)
{
@@ -118,7 +118,7 @@ rt_varea_t _aspace_bst_search_exceed(struct rt_aspace *aspace, void *start)
if (cmp < 0)
{
/* varae exceed start */
ptrdiff_t off = va_s - start;
ptrdiff_t off = (char *)va_s - (char *)start;
if (off < min_off)
{
min_off = off;
+3 -3
View File
@@ -33,12 +33,12 @@ enum ioremap_type
static void *_ioremap_type(void *paddr, size_t size, enum ioremap_type type)
{
void *v_addr = NULL;
char *v_addr = NULL;
size_t attr;
size_t lo_off;
int err;
lo_off = (uintptr_t)paddr & ARCH_PAGE_MASK;
lo_off = (rt_ubase_t)paddr & ARCH_PAGE_MASK;
struct rt_mm_va_hint hint = {
.prefer = RT_NULL,
@@ -62,7 +62,7 @@ static void *_ioremap_type(void *paddr, size_t size, enum ioremap_type type)
default:
return v_addr;
}
err = rt_aspace_map_phy(&rt_kernel_space, &hint, attr, MM_PA_TO_OFF(paddr), &v_addr);
err = rt_aspace_map_phy(&rt_kernel_space, &hint, attr, MM_PA_TO_OFF(paddr), (void **)&v_addr);
if (err)
{
+38 -43
View File
@@ -134,24 +134,16 @@ static int _do_named_map(rt_aspace_t aspace, void *vaddr, rt_size_t length,
int err = RT_EOK;
/* it's ensured by caller that (void*)end will not overflow */
void *end = vaddr + length;
void *phyaddr = (void *)(offset << MM_PAGE_SHIFT);
while (vaddr != end)
void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, length, attr);
if (ret == RT_NULL)
{
/* TODO try to map with huge TLB, when flag & HUGEPAGE */
rt_size_t pgsz = ARCH_PAGE_SIZE;
void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, pgsz, attr);
if (ret == RT_NULL)
{
err = -RT_ERROR;
break;
}
vaddr += pgsz;
phyaddr += pgsz;
err = -RT_ERROR;
}
if (err == RT_EOK)
rt_hw_tlb_invalidate_range(aspace, end - length, length, ARCH_PAGE_SIZE);
rt_hw_tlb_invalidate_range(aspace, vaddr, length, ARCH_PAGE_SIZE);
return err;
}
@@ -164,7 +156,7 @@ rt_inline void _do_page_fault(struct rt_aspace_fault_msg *msg, rt_size_t off,
msg->fault_vaddr = vaddr;
msg->fault_op = MM_FAULT_OP_READ;
msg->fault_type = MM_FAULT_TYPE_PAGE_FAULT;
msg->response.status = -1;
msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE;
msg->response.vaddr = 0;
msg->response.size = 0;
@@ -180,9 +172,9 @@ int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg)
* the page returned by handler is not checked
* cause no much assumption can make on it
*/
void *store = msg->response.vaddr;
char *store = msg->response.vaddr;
rt_size_t store_sz = msg->response.size;
if (msg->fault_vaddr + store_sz > varea->start + varea->size)
if ((char *)msg->fault_vaddr + store_sz > (char *)varea->start + varea->size)
{
LOG_W("%s: too much (0x%lx) of buffer on vaddr %p is provided",
__func__, store_sz, msg->fault_vaddr);
@@ -232,9 +224,9 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start,
int err = RT_EOK;
/* it's ensured by caller that start & size ara page-aligned */
void *end = start + size;
void *vaddr = start;
rt_size_t off = varea->offset + ((start - varea->start) >> ARCH_PAGE_SHIFT);
char *end = (char *)start + size;
char *vaddr = start;
rt_size_t off = varea->offset + ((vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT);
while (vaddr != end)
{
@@ -243,8 +235,10 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start,
_do_page_fault(&msg, off, vaddr, varea->mem_obj, varea);
if (_varea_map_with_msg(varea, &msg))
{
err = -RT_ENOMEM;
break;
}
/**
* It's hard to identify the mapping pattern on a customized handler
* So we terminate the prefetch process on that case
@@ -386,7 +380,7 @@ rt_varea_t _varea_create(void *start, rt_size_t size)
}
#define _IS_OVERFLOW(start, length) ((length) > (0ul - (uintptr_t)(start)))
#define _IS_OVERSIZE(start, length, limit_s, limit_sz) (((length) + (rt_size_t)((start) - (limit_start))) > (limit_size))
#define _IS_OVERSIZE(start, length, limit_s, limit_sz) (((length) + (rt_size_t)((char *)(start) - (char *)(limit_start))) > (limit_size))
static inline int _not_in_range(void *start, rt_size_t length,
void *limit_start, rt_size_t limit_size)
@@ -449,6 +443,10 @@ int rt_aspace_map(rt_aspace_t aspace, void **addr, rt_size_t length,
{
rt_free(varea);
}
else
{
*addr = varea->start;
}
}
else
{
@@ -461,10 +459,7 @@ int rt_aspace_map(rt_aspace_t aspace, void **addr, rt_size_t length,
{
*addr = NULL;
}
else
{
*addr = varea->start;
}
return err;
}
@@ -642,7 +637,7 @@ int rt_aspace_unmap(rt_aspace_t aspace, void *addr)
if (_not_in_range(addr, 1, aspace->start, aspace->size))
{
LOG_I("%s: %lx not in range of aspace[%lx:%lx]", __func__, addr,
aspace->start, aspace->start + aspace->size);
aspace->start, (char *)aspace->start + aspace->size);
return -RT_EINVAL;
}
@@ -658,7 +653,7 @@ static inline void *_lower(void *a, void *b)
static inline void *_align(void *va, rt_ubase_t align_mask)
{
return (void *)((rt_ubase_t)(va + ~align_mask) & align_mask);
return (void *)((rt_ubase_t)((char *)va + ~align_mask) & align_mask);
}
static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
@@ -667,17 +662,17 @@ static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
void *ret = RT_NULL;
while (varea && varea->start < limit.end)
{
void *candidate = varea->start + varea->size;
char *candidate = (char *)varea->start + varea->size;
candidate = _align(candidate, align_mask);
if (candidate > limit.end || limit.end - candidate + 1 < req_size)
if (candidate > (char *)limit.end || (char *)limit.end - candidate + 1 < req_size)
break;
rt_varea_t nx_va = ASPACE_VAREA_NEXT(varea);
if (nx_va)
{
rt_size_t gap_size =
_lower(limit.end, nx_va->start - 1) - candidate + 1;
(char *)_lower(limit.end, (char *)nx_va->start - 1) - candidate + 1;
if (gap_size >= req_size)
{
ret = candidate;
@@ -703,15 +698,15 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
rt_varea_t varea = _aspace_bst_search_exceed(aspace, limit.start);
if (varea)
{
void *candidate = _align(limit.start, align_mask);
rt_size_t gap_size = varea->start - candidate;
char *candidate = _align(limit.start, align_mask);
rt_size_t gap_size = (char *)varea->start - candidate;
if (gap_size >= req_size)
{
rt_varea_t former = _aspace_bst_search(aspace, limit.start);
if (former)
{
candidate = _align(former->start + former->size, align_mask);
gap_size = varea->start - candidate;
candidate = _align((char *)former->start + former->size, align_mask);
gap_size = (char *)varea->start - candidate;
if (gap_size >= req_size)
va = candidate;
@@ -730,12 +725,12 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
}
else
{
void *candidate;
char *candidate;
rt_size_t gap_size;
candidate = limit.start;
candidate = _align(candidate, align_mask);
gap_size = limit.end - candidate + 1;
gap_size = (char *)limit.end - candidate + 1;
if (gap_size >= req_size)
va = candidate;
@@ -750,7 +745,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
{
rt_varea_t varea = NULL;
void *va = RT_NULL;
struct _mm_range limit = {limit_start, limit_start + limit_size - 1};
struct _mm_range limit = {limit_start, (char *)limit_start + limit_size - 1};
rt_ubase_t align_mask = ~0ul;
if (flags & MMF_REQUEST_ALIGN)
@@ -762,7 +757,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
{
/* if prefer and free, just return the prefer region */
prefer = _align(prefer, align_mask);
struct _mm_range range = {prefer, prefer + req_size - 1};
struct _mm_range range = {prefer, (char *)prefer + req_size - 1};
varea = _aspace_bst_search_overlap(aspace, range);
if (!varea)
@@ -780,7 +775,7 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
if (va == RT_NULL)
{
/* rewind to first range */
limit.end = varea->start - 1;
limit.end = (char *)varea->start - 1;
va = _find_head_and_asc_search(aspace, req_size, align_mask,
limit);
}
@@ -798,7 +793,7 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage)
{
int err = RT_EOK;
rt_varea_t varea;
void *end = addr + (npage << ARCH_PAGE_SHIFT);
char *end = (char *)addr + (npage << ARCH_PAGE_SHIFT);
WR_LOCK(aspace);
varea = _aspace_bst_search(aspace, addr);
@@ -809,7 +804,7 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage)
LOG_W("%s: varea not exist", __func__);
err = -RT_ENOENT;
}
else if (addr >= end || (rt_size_t)addr & ARCH_PAGE_MASK ||
else if ((char *)addr >= end || (rt_size_t)addr & ARCH_PAGE_MASK ||
_not_in_range(addr, npage << ARCH_PAGE_SHIFT, varea->start,
varea->size))
{
@@ -938,12 +933,12 @@ static int _dump(rt_varea_t varea, void *arg)
{
if (varea->mem_obj && varea->mem_obj->get_name)
{
rt_kprintf("[%p - %p] %s\n", varea->start, varea->start + varea->size,
rt_kprintf("[%p - %p] %s\n", varea->start, (char *)varea->start + varea->size,
varea->mem_obj->get_name(varea));
}
else
{
rt_kprintf("[%p - %p] phy-map\n", varea->start, varea->start + varea->size);
rt_kprintf("[%p - %p] phy-map\n", varea->start, (char *)varea->start + varea->size);
rt_kprintf("\t\\_ paddr = %p\n", varea->offset << MM_PAGE_SHIFT);
}
return 0;
+1 -1
View File
@@ -104,7 +104,7 @@ int rt_aspace_fault_try_fix(struct rt_aspace_fault_msg *msg)
if (varea)
{
void *pa = rt_hw_mmu_v2p(aspace, msg->fault_vaddr);
msg->off = (msg->fault_vaddr - varea->start) >> ARCH_PAGE_SHIFT;
msg->off = ((char *)msg->fault_vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT;
/* permission checked by fault op */
switch (msg->fault_op)
+3 -3
View File
@@ -56,21 +56,21 @@ void rt_varea_pgmgr_pop_all(rt_varea_t varea)
void rt_varea_pgmgr_pop(rt_varea_t varea, void *vaddr, rt_size_t size)
{
void *vend = vaddr + size;
void *vend = (char *)vaddr + size;
while (vaddr != vend)
{
rt_page_t page = rt_page_addr2page(vaddr);
page->pre->next = page->next;
page->next->pre = page->pre;
rt_pages_free(vaddr, 0);
vaddr += ARCH_PAGE_SIZE;
vaddr = (char *)vaddr + ARCH_PAGE_SIZE;
}
}
static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
{
void *page;
page = rt_pages_alloc(0);
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
if (!page)
{
+309 -107
View File
File diff suppressed because it is too large Load Diff
+31
View File
@@ -23,10 +23,35 @@
union {struct {fields}; char _padding[GET_FLOOR(struct {fields})];};\
} *rt_page_t
/**
* @brief PAGE ALLOC FLAGS
*
* @info PAGE_ANY_AVAILABLE
* page allocation default to use lower region, this behavior can change by setting
* PAGE_ANY_AVAILABLE
*/
#define PAGE_ANY_AVAILABLE 0x1ul
#ifdef RT_DEBUG_PAGE_LEAK
#define DEBUG_FIELD { \
/* trace list */ \
struct rt_page *tl_next; \
struct rt_page *tl_prev; \
void *caller; \
size_t trace_size; \
}
#else
#define DEBUG_FIELD
#endif
DEF_PAGE_T(
struct rt_page *next; /* same level next */
struct rt_page *pre; /* same level pre */
DEBUG_FIELD
rt_uint32_t size_bits; /* if is ARCH_ADDRESS_WIDTH_BITS, means not free */
rt_uint32_t ref_cnt; /* page group ref count */
);
@@ -49,6 +74,8 @@ void rt_page_cleanup(void);
void *rt_pages_alloc(rt_uint32_t size_bits);
void *rt_pages_alloc_ext(rt_uint32_t size_bits, size_t flags);
void rt_page_ref_inc(void *addr, rt_uint32_t size_bits);
int rt_page_ref_get(void *addr, rt_uint32_t size_bits);
@@ -78,4 +105,8 @@ struct rt_page *rt_page_addr2page(void *addr);
*/
int rt_page_install(rt_region_t region);
void rt_page_leak_trace_start(void);
void rt_page_leak_trace_stop(void);
#endif /* __MM_PAGE_H__ */