Mirrors_Framework/rt-thread
1
0
Fork 0
mirror of https://github.com/RT-Thread/rt-thread.git synced 2026-08-18 02:58:57 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: mm: added affinity pages allocator

Browse Source 
This patch introduces a tagged pages allocator to address the existing problems
of page aliasing on specific platforms and the requirement of page coloring.
It implements an affinity-id aware page manager by separating the runtime page
list into two types: a normal single linked-list and a multi-dimensional affinity-list.

Changes:
- Introduced tagged pages allocator and managing algorithm for affinity pages list
- Modified components to support affinity-id list management
- Updated page allocation and freeing functions to handle tagged pages
- Added configuration options for page affinity block size and debugging
- Modified mmap and elf loading to respect affinity settings
- Enhanced page list management to support multi-dimensional affinity-list

Signed-off-by: Shell <smokewood@qq.com>
This commit is contained in:
Shell
2025-02-25 11:26:30 +08:00
committed by Rbb666
parent 3fdb940967
commit 9386411d13
14 changed files with 628 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/dfs/dfs_v2/include/dfs_file.h
+1
View File
@@ -137,6 +137,7 @@ struct dfs_mmap2_args
int prot;
int flags;
off_t pgoffset;
size_t min_align_size;
struct rt_lwp *lwp;
void *ret;
components/dfs/dfs_v2/src/dfs_file_mmap.c
+1
View File
@@ -76,6 +76,7 @@ static void *_map_data_to_uspace(struct dfs_mmap2_args *mmap2, void *data, rt_er
map_vaddr = (void *)((size_t)map_vaddr & ~ARCH_PAGE_MASK);
k_flags = lwp_user_mm_flag_to_kernel(mmap2->flags);
k_flags = MMF_CREATE(k_flags, mmap2->min_align_size);
k_attr = lwp_user_mm_attr_to_kernel(mmap2->prot);
map_vaddr = _do_mmap(lwp, map_vaddr, map_size, k_attr, k_flags, mmap2->pgoffset, data, code);
components/dfs/dfs_v2/src/dfs_pcache.c
+5 -4
View File
@@ -710,14 +710,15 @@ static int dfs_page_unmap(struct dfs_page *page)
return 0;
}
static struct dfs_page *dfs_page_create(void)
static struct dfs_page *dfs_page_create(off_t pos)
{
struct dfs_page *page = RT_NULL;
int affid = RT_PAGE_PICK_AFFID(pos);
page = rt_calloc(1, sizeof(struct dfs_page));
if (page)
{
page->page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
page->page = rt_pages_alloc_tagged(0, affid, PAGE_ANY_AVAILABLE);
if (page->page)
{
//memset(page->page, 0x00, ARCH_PAGE_SIZE);
@@ -1008,12 +1009,12 @@ static struct dfs_page *dfs_aspace_load_page(struct dfs_file *file, off_t pos)
struct dfs_vnode *vnode = file->vnode;
struct dfs_aspace *aspace = vnode->aspace;
page = dfs_page_create();
page = dfs_page_create(pos);
if (page)
{
page->aspace = aspace;
page->size = ARCH_PAGE_SIZE;
page->fpos = pos / ARCH_PAGE_SIZE * ARCH_PAGE_SIZE;
page->fpos = RT_ALIGN_DOWN(pos, ARCH_PAGE_SIZE);
aspace->ops->read(file, page);
page->ref_count ++;
components/lwp/lwp_elf.c
+1 -1
View File
@@ -576,7 +576,7 @@ static int elf_aux_fill(elf_load_info_t *load_info)
elf_addr_t *aux_info;
uint32_t random_value = rt_tick_get();
size_t prot = PROT_READ | PROT_WRITE;
size_t flags = MAP_PRIVATE;
size_t flags = MAP_FIXED | MAP_PRIVATE;
rt_lwp_t lwp = load_info->lwp;
void *va;
components/lwp/lwp_user_mm.c
+87 -8
View File
@@ -503,29 +503,99 @@ void *lwp_user_memory_remap_to_kernel(rt_lwp_t lwp, void *uaddr, size_t length)
return kaddr;
}
#include <dfs_dentry.h>
#define _AFFBLK_PGOFFSET (RT_PAGE_AFFINITY_BLOCK_SIZE >> MM_PAGE_SHIFT)
static rt_base_t _aligned_for_weak_mapping(off_t *ppgoff, rt_size_t *plen, rt_size_t *palign)
{
off_t aligned_pgoffset, pgoffset = *ppgoff;
rt_size_t length = *plen;
rt_size_t min_align_size = *palign;
rt_base_t aligned_size = 0;
if (pgoffset >= 0)
{
/* force an alignment */
aligned_pgoffset =
RT_ALIGN_DOWN(pgoffset, RT_PAGE_AFFINITY_BLOCK_SIZE >> MM_PAGE_SHIFT);
aligned_size = (pgoffset - aligned_pgoffset) << MM_PAGE_SHIFT;
if (aligned_pgoffset != pgoffset)
{
/**
* If requested pgoffset is not sitting on an aligned page offset,
* expand the request mapping to force an alignment.
*/
length += aligned_size;
pgoffset = aligned_pgoffset;
}
/**
* As this is a weak mapping, we can pick any reasonable address for our
* requirement.
*/
min_align_size = RT_PAGE_AFFINITY_BLOCK_SIZE;
}
else
{
RT_ASSERT(0 && "Unexpected input");
}
*ppgoff = pgoffset;
*plen = length;
*palign = min_align_size;
return aligned_size;
}
void *lwp_mmap2(struct rt_lwp *lwp, void *addr, size_t length, int prot,
int flags, int fd, off_t pgoffset)
{
rt_err_t rc;
rt_size_t k_attr;
rt_size_t k_flags;
rt_size_t k_offset;
rt_size_t k_attr, k_flags, k_offset, aligned_size = 0;
rt_size_t min_align_size = 1 << MM_PAGE_SHIFT;
rt_aspace_t uspace;
rt_mem_obj_t mem_obj;
void *ret = 0;
LOG_D("%s(addr=0x%lx,length=%ld,fd=%d)", __func__, addr, length, fd);
LOG_D("%s(addr=0x%lx,length=0x%lx,fd=%d,pgoff=0x%lx)", __func__, addr, length, fd, pgoffset);
/* alignment for affinity page block */
if (flags & MAP_FIXED)
{
if (fd != -1)
{
/* requested mapping address */
rt_base_t va_affid = RT_PAGE_PICK_AFFID(addr);
rt_base_t pgoff_affid = RT_PAGE_PICK_AFFID(pgoffset << MM_PAGE_SHIFT);
/* filter illegal align address */
if (va_affid != pgoff_affid)
{
LOG_W("Unaligned mapping address %p(pgoff=0x%lx) from fd=%d",
addr, pgoffset, fd);
}
}
else
{
/* anonymous mapping can always aligned */
}
}
else
{
/* weak address selection */
aligned_size = _aligned_for_weak_mapping(&pgoffset, &length, &min_align_size);
}
if (fd == -1)
{
/**
* todo: add threshold
*/
#ifdef RT_DEBUGGING_PAGE_THRESHOLD
if (!_memory_threshold_ok())
return (void *)-ENOMEM;
#endif /* RT_DEBUGGING_PAGE_THRESHOLD */
k_offset = MM_PA_TO_OFF(addr);
k_flags = lwp_user_mm_flag_to_kernel(flags) | MMF_MAP_PRIVATE;
k_flags = MMF_CREATE(lwp_user_mm_flag_to_kernel(flags) | MMF_MAP_PRIVATE,
min_align_size);
k_attr = lwp_user_mm_attr_to_kernel(prot);
uspace = lwp->aspace;
@@ -553,6 +623,7 @@ void *lwp_mmap2(struct rt_lwp *lwp, void *addr, size_t length, int prot,
mmap2.addr = addr;
mmap2.length = length;
mmap2.min_align_size = min_align_size;
mmap2.prot = prot;
mmap2.flags = flags;
mmap2.pgoffset = pgoffset;
@@ -572,7 +643,15 @@ void *lwp_mmap2(struct rt_lwp *lwp, void *addr, size_t length, int prot,
}
if ((long)ret <= 0)
{
LOG_D("%s() => %ld", __func__, ret);
}
else
{
ret = (char *)ret + aligned_size;
LOG_D("%s() => 0x%lx", __func__, ret);
}
return ret;
}
components/mm/Kconfig
+12
View File
@@ -1,5 +1,13 @@
menu "Memory management"
config RT_PAGE_AFFINITY_BLOCK_SIZE
hex "Affinity block size in bytes for page management"
default 0x1000
help
Page affinity block can be used to resolve the VIPT aliasing problem.
It should be set to `1ul << ((index + block) - page_offset)` in this case.
You could also exploit this as a tunning for cache coloring.
config RT_USING_MEMBLOCK
bool "Using memblock"
default n
@@ -16,4 +24,8 @@ config RT_INIT_MEMORY_REGIONS
memory into different types of regions. This variable specifies
the maximum number of regions supported by the system.
config RT_DEBUGGING_ALIASING
bool "Using aliasing paging debugger"
default n
endmenu
components/mm/ioremap.c
+6 -4
View File
@@ -38,13 +38,15 @@ static void *_ioremap_type(void *paddr, size_t size, enum ioremap_type type)
size_t attr;
size_t lo_off;
int err;
size_t pa_off = (rt_ubase_t)paddr & ~(RT_PAGE_AFFINITY_BLOCK_SIZE - 1);
lo_off = (rt_ubase_t)paddr & ARCH_PAGE_MASK;
lo_off = (rt_ubase_t)paddr - pa_off;
pa_off = MM_PA_TO_OFF(pa_off);
struct rt_mm_va_hint hint = {
.prefer = RT_NULL,
.map_size = RT_ALIGN(size + lo_off, ARCH_PAGE_SIZE),
.flags = 0,
.map_size = RT_ALIGN(size + lo_off, RT_PAGE_AFFINITY_BLOCK_SIZE),
.flags = MMF_CREATE(0, RT_PAGE_AFFINITY_BLOCK_SIZE),
.limit_start = rt_ioremap_start,
.limit_range_size = rt_ioremap_size,
};
@@ -63,7 +65,7 @@ static void *_ioremap_type(void *paddr, size_t size, enum ioremap_type type)
default:
return v_addr;
}
err = rt_aspace_map_phy(&rt_kernel_space, &hint, attr, MM_PA_TO_OFF(paddr), (void **)&v_addr);
err = rt_aspace_map_phy(&rt_kernel_space, &hint, attr, pa_off, (void **)&v_addr);
if (err)
{
components/mm/mm_anon.c
+1 -1
View File
@@ -581,7 +581,7 @@ int rt_varea_fix_private_locked(rt_varea_t ex_varea, void *pa,
}
else if (ex_obj->page_read)
{
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
page = rt_pages_alloc_tagged(0, RT_PAGE_PICK_AFFID(fault_vaddr), PAGE_ANY_AVAILABLE);
if (page)
{
/** setup message & fetch the data from source object */
components/mm/mm_aspace.c
+49 -20
View File
@@ -1149,12 +1149,17 @@ static void *_ascending_search(rt_varea_t varea, rt_size_t req_size,
rt_varea_t nx_va = ASPACE_VAREA_NEXT(varea);
if (nx_va)
{
rt_size_t gap_size =
(char *)_lower(limit.end, (char *)nx_va->start - 1) - candidate + 1;
if (gap_size >= req_size)
if (candidate < (char *)nx_va->start)
{
ret = candidate;
break;
rt_size_t gap_size =
(char *)_lower(limit.end, (char *)nx_va->start - 1) -
candidate + 1;
if (gap_size >= req_size)
{
ret = candidate;
break;
}
}
}
else
@@ -1172,15 +1177,16 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
struct _mm_range limit)
{
void *va = RT_NULL;
char *candidate = _align(limit.start, align_mask);
rt_varea_t varea = _aspace_bst_search_exceed(aspace, limit.start);
rt_varea_t varea = _aspace_bst_search_exceed(aspace, candidate);
if (varea)
{
char *candidate = _align(limit.start, align_mask);
rt_size_t gap_size = (char *)varea->start - candidate;
if (gap_size >= req_size)
{
rt_varea_t former = _aspace_bst_search(aspace, limit.start);
/* try previous memory region of varea if possible */
rt_varea_t former = ASPACE_VAREA_PREV(varea);
if (former)
{
candidate = _align((char *)former->start + former->size, align_mask);
@@ -1203,12 +1209,7 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
}
else
{
char *candidate;
rt_size_t gap_size;
candidate = limit.start;
candidate = _align(candidate, align_mask);
gap_size = (char *)limit.end - candidate + 1;
rt_size_t gap_size = (char *)limit.end - candidate + 1;
if (gap_size >= req_size)
va = candidate;
@@ -1217,6 +1218,12 @@ static void *_find_head_and_asc_search(rt_aspace_t aspace, rt_size_t req_size,
return va;
}
/**
* Find a memory region that:
* - is free
* - sits inside the limit range
* - meets the alignment requirement
*/
static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
void *limit_start, rt_size_t limit_size,
mm_flag_t flags)
@@ -1231,20 +1238,42 @@ static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size,
align_mask = ~((1 << MMF_GET_ALIGN(flags)) - 1);
}
if (prefer != RT_NULL)
if (flags & MMF_MAP_FIXED)
{
/* if prefer and free, just return the prefer region */
prefer = _align(prefer, align_mask);
struct _mm_range range = {prefer, (char *)prefer + req_size - 1};
varea = _aspace_bst_search_overlap(aspace, range);
/* caller should guarantee that the request region is legal */
RT_ASSERT(!_not_in_range(flags, prefer, req_size, limit_start, limit_size));
varea = _aspace_bst_search_overlap(aspace, range);
if (!varea)
{
va = prefer;
}
else if (flags & MMF_MAP_FIXED)
else
{
/* OVERLAP */
/* region not freed */
}
}
else if (prefer != RT_NULL)
{
struct _mm_range range;
/* ceiling the prefer address */
prefer = _align(prefer, align_mask);
if (_not_in_range(flags, prefer, req_size, limit_start, limit_size))
{
prefer = limit_start;
}
range.start = prefer;
range.end = (char *)prefer + req_size - 1;
varea = _aspace_bst_search_overlap(aspace, range);
if (!varea)
{
/* if preferred and free, just return the prefer region */
va = prefer;
}
else
{
components/mm/mm_fault.c
+8 -1
View File
@@ -185,15 +185,22 @@ int rt_aspace_fault_try_fix(rt_aspace_t aspace, struct rt_aspace_fault_msg *msg)
case MM_FAULT_OP_EXECUTE:
err = _exec_fault(varea, pa, msg);
break;
default:
LOG_D("Unhandle exception");
break;
}
}
}
else
{
LOG_I("%s: varea not found at 0x%lx", __func__, msg->fault_vaddr);
LOG_W("%s: varea not found at 0x%lx", __func__, msg->fault_vaddr);
}
RD_UNLOCK(aspace);
}
else
{
LOG_W("No aspace found");
}
return err;
}
components/mm/mm_object.c
+2 -1
View File
@@ -34,7 +34,8 @@ static const char *get_name(rt_varea_t varea)
static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
{
void *page;
page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
int affid = RT_PAGE_PICK_AFFID(msg->fault_vaddr);
page = rt_pages_alloc_tagged(0, affid, PAGE_ANY_AVAILABLE);
if (!page)
{
components/mm/mm_page.c
+435 -175
View File
File diff suppressed because it is too large Load Diff
components/mm/mm_page.h
+9 -1
View File
@@ -33,6 +33,9 @@
#define PAGE_ANY_AVAILABLE 0x1ul
#define RT_PAGE_PICK_AFFID(ptr) \
((((long)ptr) & (RT_PAGE_AFFINITY_BLOCK_SIZE - 1)) / ARCH_PAGE_SIZE)
#ifdef RT_DEBUGING_PAGE_LEAK
#define DEBUG_FIELD struct { \
/* trace list */ \
@@ -57,6 +60,7 @@ DEF_PAGE_T(
#undef GET_FLOOR
#undef DEF_PAGE_T
#undef DEBUG_FIELD
typedef struct tag_region
{
@@ -77,13 +81,17 @@ void *rt_pages_alloc(rt_uint32_t size_bits);
void *rt_pages_alloc_ext(rt_uint32_t size_bits, size_t flags);
void *rt_pages_alloc_tagged(rt_uint32_t size_bits, long tag, size_t flags);
rt_bool_t rt_page_is_member(rt_base_t page_pa);
void rt_page_ref_inc(void *addr, rt_uint32_t size_bits);
int rt_page_ref_get(void *addr, rt_uint32_t size_bits);
int rt_pages_free(void *addr, rt_uint32_t size_bits);
void rt_page_list(void);
int rt_page_list(void);
rt_size_t rt_page_bits(rt_size_t size);
libcpu/aarch64/common/mmu.c
+11
View File
@@ -296,6 +296,17 @@ void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size,
mapper = _kernel_map_2M;
}
/* check aliasing */
#ifdef RT_DEBUGGING_ALIASING
#define _ALIAS_OFFSET(addr) ((long)(addr) & (RT_PAGE_AFFINITY_BLOCK_SIZE - 1))
if (rt_page_is_member((rt_base_t)p_addr) && _ALIAS_OFFSET(v_addr) != _ALIAS_OFFSET(p_addr))
{
LOG_W("Possibly aliasing on va(0x%lx) to pa(0x%lx)", v_addr, p_addr);
rt_backtrace();
RT_ASSERT(0);
}
#endif /* RT_DEBUGGING_ALIASING */
MM_PGTBL_LOCK(aspace);
ret = mapper(aspace->page_table, v_addr, p_addr, attr);
MM_PGTBL_UNLOCK(aspace);