Revert "arm64: save FPU regs every time"

This reverts commit 3c4f3c1008.

arch/arm64/src/common/arm64_arch.h

@@ -299,9 +299,6 @@ struct regs_context
   uint64_t  spsr;
   uint64_t  sp_el0;
   uint64_t  exe_depth;
-#ifdef CONFIG_ARCH_FPU
-  struct fpu_reg fpu_regs;
-#endif
 };
 
 /****************************************************************************

arch/arm64/src/common/arm64_exit.c

@@ -65,6 +65,12 @@ void up_exit(int status)
    */
 
   enter_critical_section();
+
+  /* Destroy the task at the head of the ready to run list. */
+#ifdef CONFIG_ARCH_FPU
+  arm64_destory_fpu(tcb);
+#endif
+
   nxtask_exit();
 
   /* Now, perform the context switch to the new ready-to-run task at the

arch/arm64/src/common/arm64_fpu.c

@@ -77,10 +77,11 @@ struct arm64_fpu_procfs_file_s
  * Private Data
  ***************************************************************************/
 
-#ifdef CONFIG_FS_PROCFS_REGISTER
+static struct fpu_reg g_idle_thread_fpu[CONFIG_SMP_NCPUS];
-
 static struct arm64_cpu_fpu_context g_cpu_fpu_ctx[CONFIG_SMP_NCPUS];
 
+#ifdef CONFIG_FS_PROCFS_REGISTER
+
 /* procfs methods */
 
 static int arm64_fpu_procfs_open(struct file *filep, const char *relpath,
@@ -261,6 +262,141 @@ static int arm64_fpu_procfs_stat(const char *relpath, struct stat *buf)
 }
 #endif
 
+/***************************************************************************
+ * Public Functions
+ ***************************************************************************/
+
+void arm64_init_fpu(struct tcb_s *tcb)
+{
+  if (tcb->pid < CONFIG_SMP_NCPUS)
+    {
+#ifdef CONFIG_SMP
+      int cpu = tcb->cpu;
+#else
+      int cpu = 0;
+#endif
+      memset(&g_cpu_fpu_ctx[cpu], 0,
+             sizeof(struct arm64_cpu_fpu_context));
+      g_cpu_fpu_ctx[cpu].idle_thread = tcb;
+
+      tcb->xcp.fpu_regs = (uint64_t *)&g_idle_thread_fpu[cpu];
+    }
+
+  memset(tcb->xcp.fpu_regs, 0, sizeof(struct fpu_reg));
+}
+
+void arm64_destory_fpu(struct tcb_s *tcb)
+{
+  struct tcb_s *owner;
+
+  /* save current fpu owner's context */
+
+  owner = g_cpu_fpu_ctx[this_cpu()].fpu_owner;
+
+  if (owner == tcb)
+    {
+      g_cpu_fpu_ctx[this_cpu()].fpu_owner = NULL;
+    }
+}
+
+/***************************************************************************
+ * Name: arm64_fpu_enter_exception
+ *
+ * Description:
+ *   called at every time get into a exception
+ *
+ ***************************************************************************/
+
+void arm64_fpu_enter_exception(void)
+{
+}
+
+void arm64_fpu_exit_exception(void)
+{
+}
+
+void arm64_fpu_trap(struct regs_context *regs)
+{
+  struct tcb_s *owner;
+
+  UNUSED(regs);
+
+  /* disable fpu trap access */
+
+  arm64_fpu_access_trap_disable();
+
+  /* save current fpu owner's context */
+
+  owner = g_cpu_fpu_ctx[this_cpu()].fpu_owner;
+
+  if (owner != NULL)
+    {
+      arm64_fpu_save((struct fpu_reg *)owner->xcp.fpu_regs);
+      ARM64_DSB();
+      g_cpu_fpu_ctx[this_cpu()].save_count++;
+      g_cpu_fpu_ctx[this_cpu()].fpu_owner = NULL;
+    }
+
+  if (arch_get_exception_depth() > 1)
+    {
+      /* if get_exception_depth > 1
+       * it means FPU access exception occurred in exception context
+       * switch FPU owner to idle thread
+       */
+
+      owner = g_cpu_fpu_ctx[this_cpu()].idle_thread;
+    }
+  else
+    {
+      owner = running_task();
+    }
+
+  /* restore our context */
+
+  arm64_fpu_restore((struct fpu_reg *)owner->xcp.fpu_regs);
+  g_cpu_fpu_ctx[this_cpu()].restore_count++;
+
+  /* become new owner */
+
+  g_cpu_fpu_ctx[this_cpu()].fpu_owner = owner;
+}
+
+void arm64_fpu_context_restore(void)
+{
+  struct tcb_s *new_tcb = running_task();
+
+  arm64_fpu_access_trap_disable();
+
+  /* FPU trap has happened at this task */
+
+  if (new_tcb == g_cpu_fpu_ctx[this_cpu()].fpu_owner)
+    {
+      arm64_fpu_access_trap_disable();
+    }
+  else
+    {
+      arm64_fpu_access_trap_enable();
+    }
+
+  g_cpu_fpu_ctx[this_cpu()].switch_count++;
+}
+
+#ifdef CONFIG_SMP
+void arm64_fpu_context_save(void)
+{
+  struct tcb_s *tcb = running_task();
+
+  if (tcb == g_cpu_fpu_ctx[this_cpu()].fpu_owner)
+    {
+      arm64_fpu_access_trap_disable();
+      arm64_fpu_save((struct fpu_reg *)tcb->xcp.fpu_regs);
+      ARM64_DSB();
+      g_cpu_fpu_ctx[this_cpu()].save_count++;
+      g_cpu_fpu_ctx[this_cpu()].fpu_owner = NULL;
+    }
+}
+#endif
+
 void arm64_fpu_enable(void)
 {
   irqstate_t flags = up_irq_save();

arch/arm64/src/common/arm64_initialstate.c

@@ -47,6 +47,10 @@
 #include "chip.h"
 #include "arm64_fatal.h"
 
+#ifdef CONFIG_ARCH_FPU
+#include "arm64_fpu.h"
+#endif
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -66,6 +70,17 @@ void arm64_new_task(struct tcb_s * tcb)
     }
 #endif
 
+#ifdef CONFIG_ARCH_FPU
+  struct fpu_reg *pfpuctx;
+  pfpuctx = STACK_PTR_TO_FRAME(struct fpu_reg, stack_ptr);
+  tcb->xcp.fpu_regs   = (uint64_t *)pfpuctx;
+
+  /* set fpu context */
+
+  arm64_init_fpu(tcb);
+  stack_ptr  = (uintptr_t)pfpuctx;
+#endif
+
   pinitctx = STACK_PTR_TO_FRAME(struct regs_context, stack_ptr);
   memset(pinitctx, 0, sizeof(struct regs_context));
   pinitctx->elr       = (uint64_t)tcb->start;
@@ -135,6 +150,11 @@ void up_initial_state(struct tcb_s *tcb)
       tcb->stack_base_ptr  = tcb->stack_alloc_ptr;
       tcb->adj_stack_size  = CONFIG_IDLETHREAD_STACKSIZE;
 
+#ifdef CONFIG_ARCH_FPU
+      /* set fpu context */
+
+      arm64_init_fpu(tcb);
+#endif
       /* set initialize idle thread tcb and exception depth
        * core 0, idle0
        */

arch/arm64/src/common/arm64_schedulesigaction.c

@@ -38,6 +38,10 @@
 #include "irq/irq.h"
 #include "arm64_fatal.h"
 
+#ifdef CONFIG_ARCH_FPU
+#include "arm64_fpu.h"
+#endif
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -54,6 +58,16 @@ void arm64_init_signal_process(struct tcb_s *tcb, struct regs_context *regs)
   struct regs_context  *psigctx;
   char *stack_ptr    = (char *)pctx->sp_elx - sizeof(struct regs_context);
 
+#ifdef CONFIG_ARCH_FPU
+  struct fpu_reg      *pfpuctx;
+  pfpuctx            = STACK_PTR_TO_FRAME(struct fpu_reg, stack_ptr);
+  tcb->xcp.fpu_regs  = (uint64_t *)pfpuctx;
+
+  /* set fpu context */
+
+  arm64_init_fpu(tcb);
+  stack_ptr          = (char *)pfpuctx;
+#endif
   psigctx            = STACK_PTR_TO_FRAME(struct regs_context, stack_ptr);
   memset(psigctx, 0, sizeof(struct regs_context));
   psigctx->elr       = (uint64_t)arm64_sigdeliver;
@@ -163,6 +177,9 @@ void up_schedule_sigaction(struct tcb_s *tcb, sig_deliver_t sigdeliver)
               /* create signal process context */
 
               tcb->xcp.saved_reg = up_current_regs();
+#ifdef CONFIG_ARCH_FPU
+              tcb->xcp.saved_fpu_regs = tcb->xcp.fpu_regs;
+#endif
               arm64_init_signal_process(tcb,
               (struct regs_context *)up_current_regs());
 
@@ -184,6 +201,9 @@ void up_schedule_sigaction(struct tcb_s *tcb, sig_deliver_t sigdeliver)
            * have been delivered.
            */
 
+#ifdef CONFIG_ARCH_FPU
+          tcb->xcp.saved_fpu_regs = tcb->xcp.fpu_regs;
+#endif
           /* create signal process context */
 
           tcb->xcp.saved_reg = tcb->xcp.regs;
@@ -259,6 +279,9 @@ void up_schedule_sigaction(struct tcb_s *tcb, sig_deliver_t sigdeliver)
                    * been delivered.
                    */
 
+#ifdef CONFIG_ARCH_FPU
+                  tcb->xcp.saved_fpu_regs = tcb->xcp.fpu_regs;
+#endif
                   /* create signal process context */
 
                   tcb->xcp.saved_reg = tcb->xcp.regs;
@@ -277,6 +300,9 @@ void up_schedule_sigaction(struct tcb_s *tcb, sig_deliver_t sigdeliver)
                   /* create signal process context */
 
                   tcb->xcp.saved_reg = up_current_regs();
+#ifdef CONFIG_ARCH_FPU
+                  tcb->xcp.saved_fpu_regs = tcb->xcp.fpu_regs;
+#endif
                   arm64_init_signal_process(tcb,
                   (struct regs_context *)up_current_regs());
 
@@ -313,6 +339,9 @@ void up_schedule_sigaction(struct tcb_s *tcb, sig_deliver_t sigdeliver)
            * have been delivered.
            */
 
+#ifdef CONFIG_ARCH_FPU
+          tcb->xcp.saved_fpu_regs = tcb->xcp.fpu_regs;
+#endif
           tcb->xcp.saved_reg = tcb->xcp.regs;
 
           /* create signal process context */

arch/arm64/src/common/arm64_sigdeliver.c

@@ -38,6 +38,10 @@
 #include "irq/irq.h"
 #include "arm64_fatal.h"
 
+#ifdef CONFIG_ARCH_FPU
+#include "arm64_fpu.h"
+#endif
+
 /****************************************************************************
  * Public Functions
  ****************************************************************************/
@@ -153,6 +157,11 @@ retry:
   rtcb->xcp.sigdeliver = NULL;  /* Allows next handler to be scheduled */
   rtcb->xcp.regs = rtcb->xcp.saved_reg;
 
+#ifdef CONFIG_ARCH_FPU
+  arm64_destory_fpu(rtcb);
+  rtcb->xcp.fpu_regs = rtcb->xcp.saved_fpu_regs;
+#endif
+
   /* Then restore the correct state for this thread of execution. */
 
 #ifdef CONFIG_SMP

arch/arm64/src/common/arm64_syscall.c

@@ -193,6 +193,7 @@ uint64_t *arm64_syscall_switch(uint64_t * regs)
            */
 
           ret_regs = (uint64_t *)f_regs->regs[REG_X1];
+          f_regs->regs[REG_X1] = 0; /* set the saveregs = 0 */
 
           DEBUGASSERT(ret_regs);
         }

arch/arm64/src/common/arm64_vector_table.S

@@ -45,8 +45,8 @@
  *     but only save ARM64_ESF_REGS
  */
 
-.macro arm64_enter_exception
+.macro arm64_enter_exception xreg0, xreg1
-    sub    sp, sp, #8 * XCPTCONTEXT_REGS
+    sub    sp, sp, #8 * XCPTCONTEXT_GP_REGS
 
     stp    x0,  x1,  [sp, #8 * REG_X0]
     stp    x2,  x3,  [sp, #8 * REG_X2]
@@ -65,32 +65,30 @@
     stp    x28, x29, [sp, #8 * REG_X28]
 
     /* Save the current task's SP_ELx and x30 */
-    add    x0, sp, #8 * XCPTCONTEXT_REGS
+    add    \xreg0, sp, #8 * XCPTCONTEXT_GP_REGS
-    stp    x30, x0,  [sp, #8 * REG_X30]
+    stp    x30, \xreg0,  [sp, #8 * REG_X30]
 
     /* ELR and SPSR */
 #if CONFIG_ARCH_ARM64_EXCEPTION_LEVEL == 3
-    mrs    x0, elr_el3
+    mrs    \xreg0, elr_el3
-    mrs    x1, spsr_el3
+    mrs    \xreg1, spsr_el3
 #else
-    mrs    x0, elr_el1
+    mrs    \xreg0, elr_el1
-    mrs    x1, spsr_el1
+    mrs    \xreg1, spsr_el1
 #endif
-    stp    x0, x1,  [sp, #8 * REG_ELR]
+    stp    \xreg0, \xreg1,  [sp, #8 * REG_ELR]
 
-    /* Increment exception depth */
+    /* increment exception depth */
 
-    mrs    x0, tpidrro_el0
+    mrs    \xreg0, tpidrro_el0
-    mov    x1, #1
+    mov    \xreg1, #1
-    add    x0, x0, x1
+    add    \xreg0, \xreg0, \xreg1
-    msr    tpidrro_el0, x0
+    msr    tpidrro_el0, \xreg0
 
-    /* Save the FPU registers */
-
-    add    x0, sp, #8 * XCPTCONTEXT_GP_REGS
 #ifdef CONFIG_ARCH_FPU
-    bl     arm64_fpu_save
+    bl    arm64_fpu_enter_exception
 #endif
+
 .endm
 
 /****************************************************************************
@@ -145,97 +143,97 @@ SECTION_SUBSEC_FUNC(exc_vector_table,_vector_table_section,_vector_table)
     /* Current EL with SP0 / Synchronous */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x9, x10
     b    arm64_sync_exc
 
     /* Current EL with SP0 / IRQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_irq_handler
 
     /* Current EL with SP0 / FIQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b     arm64_fiq_handler
 
     /* Current EL with SP0 / SError */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_serror_handler
 
     /* Current EL with SPx / Synchronous */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x9, x10
     b    arm64_sync_exc
 
     /* Current EL with SPx / IRQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_irq_handler
 
     /* Current EL with SPx / FIQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b     arm64_fiq_handler
 
     /* Current EL with SPx / SError */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_serror_handler
 
     /* Lower EL using AArch64 / Synchronous */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x9, x10
     b    arm64_sync_exc
 
     /* Lower EL using AArch64 / IRQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_irq_handler
 
     /* Lower EL using AArch64 / FIQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b     arm64_fiq_handler
 
     /* Lower EL using AArch64 / SError */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_serror_handler
 
     /* Lower EL using AArch32 / Synchronous */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_mode32_handler
 
     /* Lower EL using AArch32 / IRQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_mode32_handler
 
     /* Lower EL using AArch32 / FIQ */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_mode32_handler
 
     /* Lower EL using AArch32 / SError */
 
     .align 7
-    arm64_enter_exception
+    arm64_enter_exception x0, x1
     b    arm64_mode32_handler
 
 /* Restore Corruptible Registers and exception context
@@ -244,11 +242,10 @@ SECTION_SUBSEC_FUNC(exc_vector_table,_vector_table_section,_vector_table)
 
 GTEXT(arm64_exit_exception)
 SECTION_FUNC(text, arm64_exit_exception)
-
-    mov    sp, x0
-    add    x0, x0, #8 * XCPTCONTEXT_GP_REGS
 #ifdef CONFIG_ARCH_FPU
-    bl     arm64_fpu_restore
+    bl    arm64_fpu_exit_exception
+GTEXT(arm64_exit_exc_fpu_done)
+arm64_exit_exc_fpu_done:
 #endif
 
     /* restore spsr and elr at el1*/
@@ -286,6 +283,6 @@ SECTION_FUNC(text, arm64_exit_exception)
     ldp    x28, x29, [sp, #8 * REG_X28]
     ldp    x30, xzr, [sp, #8 * REG_X30]
 
-    add    sp, sp, #8 * XCPTCONTEXT_REGS
+    add    sp, sp, #8 * XCPTCONTEXT_GP_REGS
 
     eret

arch/arm64/src/common/arm64_vectors.S

@@ -99,6 +99,63 @@ SECTION_FUNC(text, up_saveusercontext)
 
     ret
 
+/****************************************************************************
+ * Function: arm64_context_switch
+ *
+ * Description:
+ *  Routine to handle context switch
+ *
+ * arm64_context_switch( x0, x1)
+ *     x0: restore thread stack context
+ *     x1: save thread stack context
+ * note:
+ *     x1 = 0, only restore x0
+ *
+ ****************************************************************************/
+
+GTEXT(arm64_context_switch)
+SECTION_FUNC(text, arm64_context_switch)
+    cmp    x1, #0x0
+    beq    restore_new
+
+#if defined(CONFIG_ARCH_FPU) && defined(CONFIG_SMP)
+    stp    x0, x1, [sp, #-16]!
+    stp    xzr, x30, [sp, #-16]!
+    bl     arm64_fpu_context_save
+    ldp    xzr, x30, [sp], #16
+    ldp    x0, x1, [sp], #16
+#endif
+
+    /* Save the current SP_ELx */
+    add    x4, sp, #8 * XCPTCONTEXT_GP_REGS
+    str    x4, [x1, #8 * REG_SP_ELX]
+
+    /* Save the current task's SP_EL0 and exception depth */
+    mrs    x4, sp_el0
+    mrs    x5, tpidrro_el0
+    stp    x4, x5, [x1, #8 * REG_SP_EL0]
+
+restore_new:
+
+    /* Restore SP_EL0 and thread's exception dept */
+    ldp    x4,  x5,  [x0, #8 * REG_SP_EL0]
+    msr    tpidrro_el0, x5
+    msr    sp_el0, x4
+
+    /* retrieve new thread's SP_ELx */
+    ldr    x4, [x0, #8 * REG_SP_ELX]
+    sub    sp, x4, #8 * XCPTCONTEXT_GP_REGS
+
+#ifdef CONFIG_ARCH_FPU
+    stp    xzr, x30, [sp, #-16]!
+    bl     arm64_fpu_context_restore
+    ldp    xzr, x30, [sp], #16
+#endif
+
+    /* Return to arm64_sync_exc() or arm64_irq_handler() */
+
+    ret
+
 /****************************************************************************
  * Function: arm64_jump_to_user
  *
@@ -142,27 +199,28 @@ SECTION_FUNC(text, arm64_sync_exc)
     /* checking the EC value to see which exception need to be handle */
 
 #if CONFIG_ARCH_ARM64_EXCEPTION_LEVEL == 3
-    mrs    x0, esr_el3
+    mrs    x9, esr_el3
 #else
-    mrs    x0, esr_el1
+    mrs    x9, esr_el1
 #endif
-    lsr    x1, x0, #26
+    lsr    x10, x9, #26
 
 #ifdef CONFIG_ARCH_FPU
     /* fpu trap */
 
-    cmp    x1, #0x07 /*Access to SIMD or floating-point */
+    cmp    x10, #0x07 /* Access to SIMD or floating-point */
     bne    1f
-    mov    x0, sp
+    mov    x0, sp     /* x0 = context */
+    bl     arm64_fpu_trap
 
     /* when the fpu trap is handled */
 
-    b      arm64_exit_exception
+    b      arm64_exit_exc_fpu_done
 1:
 #endif
     /* 0x15 = SVC system call */
 
-    cmp    x1, #0x15
+    cmp    x10, #0x15
 
     /* if this is a svc call ?*/
 
@@ -250,13 +308,17 @@ context_switch:
      */
 
     mov    x0, sp
-    str    x0, [sp, #-16]!
     bl    arm64_syscall_switch
 
     /* get save task reg context pointer */
 
-    ldr    x1, [sp], #16
+    ldr    x1, [sp, #8 * REG_X1]
+    cmp    x1, #0x0
 
+    beq do_switch
+    ldr x1, [x1]
+
+do_switch:
 #ifdef CONFIG_SMP
     /* Notes:
      * Complete any pending TLB or cache maintenance on this CPU in case
@@ -268,7 +330,15 @@ context_switch:
     dsb   ish
 #endif
 
+    bl    arm64_context_switch
+
+#ifdef CONFIG_ARCH_FPU
+    /* when the fpu trap is handled */
+
+    b    arm64_exit_exc_fpu_done
+#else
     b    arm64_exit_exception
+#endif
 
 save_context:
     arm64_exception_context_save x0 x1 sp
@@ -338,6 +408,14 @@ SECTION_FUNC(text, arm64_irq_handler)
 
     ldr    x1, [sp], #16
 
+    /* retrieve the task's stack. */
+
+    mov    sp, x1
+
+    cmp    x0, x1
+    beq    irq_exit
+
+irq_context_switch:
 #ifdef CONFIG_SMP
     /* Notes:
      * Complete any pending TLB or cache maintenance on this CPU in case
@@ -349,6 +427,19 @@ SECTION_FUNC(text, arm64_irq_handler)
 
 #endif
 
+    /*  Switch thread
+     *  - x0: restore task reg context, return by arm64_decodeirq,
+     *  - x1: save task reg context, save before call arm64_decodeirq
+     *    call arm64_context_switch(x0) to switch
+     */
+    bl    arm64_context_switch
+#ifdef CONFIG_ARCH_FPU
+    /* when the fpu trap is handled */
+
+    b     arm64_exit_exc_fpu_done
+#endif
+
+irq_exit:
     b     arm64_exit_exception
 
 /* TODO: if the arm64_fatal_handler return success, maybe need context switch */
@@ -411,6 +502,14 @@ SECTION_FUNC(text, arm64_fiq_handler)
 
     ldr    x1, [sp], #16
 
+    /* retrieve the task's stack. */
+
+    mov    sp, x1
+
+    cmp    x0, x1
+    beq    fiq_exit
+
+
 #ifdef CONFIG_SMP
     /* Notes:
      * Complete any pending TLB or cache maintenance on this CPU in case
@@ -422,5 +521,17 @@ SECTION_FUNC(text, arm64_fiq_handler)
 
 #endif
 
+    /*  Switch thread
+     *  - x0: restore task reg context, return by arm64_decodefiq,
+     *  - x1: save task reg context, save before call arm64_decodefiq
+     *    call arm64_context_switch(x0) to switch
+     */
+    bl    arm64_context_switch
+#ifdef CONFIG_ARCH_FPU
+    /* when the fpu trap is handled */
+
+    b     arm64_exit_exc_fpu_done
+#endif
+fiq_exit:
     b     arm64_exit_exception
 #endif