arm64_addrenv: Add support for 4 level MMU translations

The original code made the incorrect assumption that the amount of
translation levels is 3, but this is incorrect. The amount of levels is 4
and the amount of levels that are utilized / in use is set dynamically
from the amount of VA bits in use.

arch/arm64/src/common/arm64_addrenv.c

@@ -115,24 +115,28 @@ static void map_spgtables(arch_addrenv_t *addrenv, uintptr_t vaddr)
 {
   int       i;
   uintptr_t prev;
+  uintptr_t l0;
 
-  /* Start from L1, and connect until max level - 1 */
+  /* Get the base page table level and the page table associated with it */
 
-  prev = arm64_pgvaddr(addrenv->spgtables[0]);
+  l0   = mmu_get_base_pgt_level();
+  prev = arm64_pgvaddr(addrenv->spgtables[l0]);
 
-  /* Check if the mapping already exists */
+  /* Start from the base level, and connect until max level - 1 */
 
-  if (mmu_ln_getentry(1, prev, vaddr) != 0)
-    {
-      return;
-    }
-
-  /* No mapping yet, create it */
-
-  for (i = 0; i < (ARCH_SPGTS - 1); i++)
+  for (i = l0; i < (ARCH_SPGTS - 1); i++)
     {
       uintptr_t next = addrenv->spgtables[i + 1];
-      mmu_ln_setentry(i + 1, prev, next, vaddr, MMU_UPGT_FLAGS);
+
+      /* Check if the mapping already exists */
+
+      if (mmu_ln_getentry(i, prev, vaddr) == 0)
+        {
+          /* No mapping yet, create it */
+
+          mmu_ln_setentry(i, prev, next, vaddr, MMU_UPGT_FLAGS);
+        }
+
       prev = arm64_pgvaddr(next);
     }
 }
@@ -157,7 +161,7 @@ static int create_spgtables(arch_addrenv_t *addrenv)
   int       i;
   uintptr_t paddr;
 
-  for (i = 0; i < ARCH_SPGTS; i++)
+  for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++)
     {
       paddr = mm_pgalloc(1);
       if (!paddr)
@@ -197,20 +201,37 @@ static int create_spgtables(arch_addrenv_t *addrenv)
 
 static int copy_kernel_mappings(arch_addrenv_t *addrenv)
 {
-  uintptr_t user_mappings = arm64_pgvaddr(addrenv->spgtables[0]);
+  uintptr_t kpgt;
+  uintptr_t upgt;
+  uintptr_t l0;
 
-  /* Copy the L1 references */
+  /* Determine the base page table level */
 
-  if (user_mappings == 0)
+  l0   = mmu_get_base_pgt_level();
+  kpgt = g_kernel_mappings;
+
+  /* Don't copy L0 references, as those encompass 512GB each */
+
+  if (l0 == 0)
+    {
+      upgt = arm64_pgvaddr(addrenv->spgtables[1]);
+      kpgt = arm64_pgvaddr(mmu_pte_to_paddr(((uintptr_t *)kpgt)[0]));
+    }
+  else
+    {
+      upgt = arm64_pgvaddr(addrenv->spgtables[l0]);
+    }
+
+  if (upgt == 0 || kpgt == 0)
     {
       return -EINVAL;
     }
 
-  memcpy((void *)user_mappings, (void *)g_kernel_mappings, MMU_PAGE_SIZE);
+  memcpy((void *)upgt, (void *)kpgt, MMU_PAGE_SIZE);
 
   /* Update with memory by flushing the cache */
 
-  up_flush_dcache(user_mappings, user_mappings + MMU_PAGE_SIZE);
+  up_flush_dcache(upgt, upgt + MMU_PAGE_SIZE);
 
   return OK;
 }
@@ -248,8 +269,8 @@ static int create_region(arch_addrenv_t *addrenv, uintptr_t vaddr,
 
   nmapped   = 0;
   npages    = MM_NPAGES(size);
-  ptprev    = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
-  ptlevel   = ARCH_SPGTS;
+  ptlevel   = MMU_PGT_LEVEL_MAX - 1;
+  ptprev    = arm64_pgvaddr(addrenv->spgtables[ptlevel]);
 
   /* Create mappings for the lower level tables */
 
@@ -384,6 +405,7 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize,
   uintptr_t textbase;
   uintptr_t database;
   uintptr_t heapbase;
+  uintptr_t l0;
 
   DEBUGASSERT(addrenv);
   DEBUGASSERT(MM_ISALIGNED(ARCH_ADDRENV_VBASE));
@@ -485,7 +507,8 @@ int up_addrenv_create(size_t textsize, size_t datasize, size_t heapsize,
 
   /* Provide the ttbr0 value for context switch */
 
-  addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[0], 0);
+  l0 = mmu_get_base_pgt_level();
+  addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0);
 
   /* Synchronize data and instruction pipelines */
 
@@ -532,17 +555,14 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv)
   /* Things start from the beginning of the user virtual memory */
 
   vaddr  = ARCH_ADDRENV_VBASE;
-  pgsize = mmu_get_region_size(ARCH_SPGTS);
+  pgsize = mmu_get_region_size(MMU_PGT_LEVEL_MAX - 1);
 
   /* First destroy the allocated memory and the final level page table */
 
   ptprev = (uintptr_t *)arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
   if (ptprev)
     {
-      /* walk user space only */
-
-      i = (ARCH_SPGTS < 2) ? vaddr / pgsize : 0;
-      for (; i < ENTRIES_PER_PGT; i++, vaddr += pgsize)
+      for (i = 0; i < ENTRIES_PER_PGT; i++, vaddr += pgsize)
         {
           ptlast = (uintptr_t *)arm64_pgvaddr(mmu_pte_to_paddr(ptprev[i]));
           if (ptlast)
@@ -570,7 +590,7 @@ int up_addrenv_destroy(arch_addrenv_t *addrenv)
 
   /* Then destroy the static tables */
 
-  for (i = 0; i < ARCH_SPGTS; i++)
+  for (i = mmu_get_base_pgt_level(); i < ARCH_SPGTS; i++)
     {
       paddr = addrenv->spgtables[i];
       if (paddr)

arch/arm64/src/common/arm64_addrenv_perms.c

@@ -70,7 +70,7 @@ static int modify_region(uintptr_t vstart, uintptr_t vend, uintptr_t setmask)
 
   for (vaddr = vstart; vaddr < vend; vaddr += MM_PGSIZE)
     {
-      for (ptlevel = 1, lnvaddr = l1vaddr;
+      for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = l1vaddr;
            ptlevel < MMU_PGT_LEVEL_MAX;
            ptlevel++)
         {

arch/arm64/src/common/arm64_addrenv_pgmap.c

@@ -90,7 +90,9 @@ uintptr_t up_addrenv_find_page(arch_addrenv_t *addrenv, uintptr_t vaddr)
 
   /* Make table walk to find the page */
 
-  for (ptlevel = 1, lnvaddr = pgdir; ptlevel < MMU_PGT_LEVEL_MAX; ptlevel++)
+  for (ptlevel = mmu_get_base_pgt_level(), lnvaddr = pgdir;
+       ptlevel < MMU_PGT_LEVEL_MAX;
+       ptlevel++)
     {
       paddr = mmu_pte_to_paddr(mmu_ln_getentry(ptlevel, lnvaddr, vaddr));
       lnvaddr = arm64_pgvaddr(paddr);
@@ -189,6 +191,7 @@ int up_addrenv_kmap_init(void)
   struct arch_addrenv_s *addrenv;
   uintptr_t              next;
   uintptr_t              vaddr;
+  uintptr_t              l0;
   int                    i;
 
   /* Populate the static page tables one by one */
@@ -196,19 +199,20 @@ int up_addrenv_kmap_init(void)
   addrenv = &g_kernel_addrenv;
   next    =  g_kernel_pgt_pbase;
   vaddr   =  CONFIG_ARCH_KMAP_VBASE;
+  l0      =  mmu_get_base_pgt_level();
 
-  for (i = 0; i < ARCH_SPGTS; i++)
+  for (i = l0; i < ARCH_SPGTS; i++)
     {
       /* Connect the static page tables */
 
       uintptr_t lnvaddr = arm64_pgvaddr(next);
       addrenv->spgtables[i] = next;
-      next = mmu_pte_to_paddr(mmu_ln_getentry(i + 1, lnvaddr, vaddr));
+      next = mmu_pte_to_paddr(mmu_ln_getentry(i, lnvaddr, vaddr));
     }
 
   /* Set the page directory root */
 
-  addrenv->ttbr0 = mmu_ttbr_reg(g_kernel_pgt_pbase, 0);
+  addrenv->ttbr0 = mmu_ttbr_reg(addrenv->spgtables[l0], 0);
 
   /* When all is set and done, flush the data caches */
 

arch/arm64/src/common/arm64_addrenv_utils.c

@@ -66,8 +66,8 @@ uintptr_t arm64_get_pgtable(arch_addrenv_t *addrenv, uintptr_t vaddr)
 
   /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */
 
-  ptlevel = ARCH_SPGTS;
-  ptprev  = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
+  ptlevel = MMU_PGT_LEVEL_MAX - 1;
+  ptprev  = arm64_pgvaddr(addrenv->spgtables[ptlevel]);
   if (!ptprev)
     {
       /* Something is very wrong */
@@ -184,7 +184,10 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr,
   uintptr_t ptlevel;
   uintptr_t paddr;
 
-  ptprev  = arm64_pgvaddr(addrenv->spgtables[ARCH_SPGTS - 1]);
+  /* Get the current level MAX_LEVELS-1 entry corresponding to this vaddr */
+
+  ptlevel = MMU_PGT_LEVEL_MAX - 1;
+  ptprev  = arm64_pgvaddr(addrenv->spgtables[ptlevel]);
   if (!ptprev)
     {
       /* Something is very wrong */
@@ -192,8 +195,6 @@ int arm64_unmap_pages(arch_addrenv_t *addrenv, uintptr_t vaddr,
       return -EFAULT;
     }
 
-  ptlevel = ARCH_SPGTS;
-
   /* Remove the references from the caller's address environment */
 
   for (; npages > 0; npages--)

arch/arm64/src/common/arm64_mmu.c

@@ -787,3 +787,8 @@ size_t mmu_get_region_size(uint32_t ptlevel)
 
   return g_pgt_sizes[ptlevel];
 }
+
+uintptr_t mmu_get_base_pgt_level(void)
+{
+  return XLAT_TABLE_BASE_LEVEL;
+}

arch/arm64/src/common/arm64_mmu.h

@@ -628,6 +628,28 @@ void mmu_ln_restore(uint32_t ptlevel, uintptr_t lnvaddr, uintptr_t vaddr,
 
 size_t mmu_get_region_size(uint32_t ptlevel);
 
+/****************************************************************************
+ * Name: mmu_get_base_pgt_level
+ *
+ * Description:
+ *   Get the base translation table level. The ARM64 MMU implementation
+ *   optimizes the amount of translation table levels in use, based on the
+ *   configured virtual address range (CONFIG_ARM64_VA_BITS).
+ *
+ *   Table indices range from 0...3 and the lowest table indices are dropped
+ *   as needed. If CONFIG_ARM64_VA_BITS >= 40, all 4 translation table levels
+ *   are needed.
+ *
+ * Input Parameters:
+ *   None.
+ *
+ * Returned Value:
+ *   The base translation table level.
+ *
+ ****************************************************************************/
+
+uintptr_t mmu_get_base_pgt_level(void);
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __ARCH_ARM64_SRC_COMMON_ARM64_MMU_H */