risc-v/qemu-rv: Supports SMP up to 8 cores

Signed-off-by: Huang Qi <huangqi3@xiaomi.com>

arch/risc-v/Kconfig

@@ -117,8 +117,9 @@ config ARCH_CHIP_QEMU_RV
 	bool "QEMU RV"
 	select ARCH_HAVE_FPU
 	select ARCH_HAVE_DPFPU
+	select ARCH_HAVE_MULTICPU
 	---help---
-		QEMU Generic RV32 processor
+		QEMU Generic RV32/RV64 processor
 
 config ARCH_CHIP_RISCV_CUSTOM
 	bool "Custom RISC-V chip"

arch/risc-v/src/qemu-rv/Make.defs

@@ -33,7 +33,12 @@ CMN_CSRCS += riscv_interruptcontext.c riscv_modifyreg32.c riscv_puts.c
 CMN_CSRCS += riscv_releasepending.c riscv_reprioritizertr.c riscv_copyfullstate.c
 CMN_CSRCS += riscv_releasestack.c riscv_stackframe.c riscv_schedulesigaction.c
 CMN_CSRCS += riscv_sigdeliver.c riscv_unblocktask.c riscv_usestack.c
-CMN_CSRCS += riscv_idle.c riscv_tcbinfo.c
+CMN_CSRCS += riscv_idle.c riscv_tcbinfo.c riscv_cpuidlestack.c
+CMN_CSRCS += riscv_fault.c
+
+ifeq ($(CONFIG_SMP), y)
+CMN_CSRCS += riscv_cpuindex.c riscv_cpupause.c riscv_cpustart.c
+endif
 
 ifeq ($(CONFIG_SCHED_BACKTRACE),y)
 CMN_CSRCS += riscv_backtrace.c

arch/risc-v/src/qemu-rv/chip.h

@@ -42,4 +42,8 @@ extern void up_serialinit(void);
 
 #include "qemu_rv_memorymap.h"
 
+#include "hardware/qemu_rv_clint.h"
+#include "hardware/qemu_rv_memorymap.h"
+#include "hardware/qemu_rv_plic.h"
+
 #endif /* __ARCH_RISCV_SRC_QEMU_RV_CHIP_H */

arch/risc-v/src/qemu-rv/hardware/qemu_rv_clint.h

@@ -25,7 +25,10 @@
  * Pre-processor Definitions
  ****************************************************************************/
 
+#define QEMU_RV_CLINT_MSIP      (QEMU_RV_CLINT_BASE + 0x0000)
 #define QEMU_RV_CLINT_MTIMECMP  (QEMU_RV_CLINT_BASE + 0x4000)
 #define QEMU_RV_CLINT_MTIME     (QEMU_RV_CLINT_BASE + 0xbff8)
 
+#define RISCV_CLINT_MSIP        QEMU_RV_CLINT_MSIP
+
 #endif /* __ARCH_RISCV_SRC_QEMU_RV_HARDWARE_QEMU_RV_CLINT_H */

arch/risc-v/src/qemu-rv/qemu_rv_head.S

@@ -38,9 +38,53 @@
 
 __start:
 
+  /* Load mhartid (cpuid) */
+
+  csrr a0, mhartid
+
   /* Set stack pointer to the idle thread stack */
 
-  la  sp, QEMU_RV_IDLESTACK_TOP
+  bnez a0, 1f
+  la   sp, QEMU_RV_IDLESTACK_TOP
+  j    2f
+1:
+
+  /* In case of single CPU config, stop here */
+
+#if !defined(CONFIG_SMP) || (CONFIG_SMP_NCPUS == 1)
+  csrw mie, zero
+  wfi
+#endif
+
+  /* To get g_cpu_basestack[mhartid], must get g_cpu_basestack first */
+
+  la   t0, g_cpu_basestack
+
+  /* Offset = pointer width * hart id */
+
+#ifdef CONFIG_ARCH_RV32
+  slli t1, a0, 2
+#else
+  slli t1, a0, 3
+#endif
+  add  t0, t0, t1
+
+  /* Load idle stack base to sp */
+
+#ifdef CONFIG_ARCH_RV32
+  lw   sp, 0(t0)
+#else
+  ld   sp, 0(t0)
+#endif
+
+  /* sp (stack top) = sp + idle stack size */
+
+  li   t0, CONFIG_IDLETHREAD_STACKSIZE
+  add  sp, sp, t0
+
+2:
+
+  /* Disable all interrupts (i.e. timer, external) in mie */
 
 	csrw	mie, zero
 

arch/risc-v/src/qemu-rv/qemu_rv_irq.c

@@ -37,8 +37,7 @@
 #include "riscv_internal.h"
 #include "riscv_arch.h"
 
-#include "hardware/qemu_rv_memorymap.h"
+#include "chip.h"
-#include "hardware/qemu_rv_plic.h"
 
 /****************************************************************************
  * Public Functions
@@ -87,6 +86,17 @@ void up_irqinitialize(void)
 
   irq_attach(RISCV_IRQ_ECALLM, riscv_swint, NULL);
 
+#ifdef CONFIG_SMP
+  /* Clear MSOFT for CPU0 */
+
+  putreg32(0, RISCV_CLINT_MSIP);
+
+  /* Setup MSOFT for CPU0 with pause handler */
+
+  irq_attach(RISCV_IRQ_MSOFT, riscv_pause_handler, NULL);
+  up_enable_irq(RISCV_IRQ_MSOFT);
+#endif
+
 #ifndef CONFIG_SUPPRESS_INTERRUPTS
 
   /* And finally, enable interrupts */
@@ -108,7 +118,13 @@ void up_disable_irq(int irq)
   int extirq;
   uint32_t oldstat;
 
-  if (irq == RISCV_IRQ_MTIMER)
+  if (irq == RISCV_IRQ_MSOFT)
+    {
+      /* Read mstatus & clear machine software interrupt enable in mie */
+
+      asm volatile ("csrrc %0, mie, %1": "=r" (oldstat) : "r"(MIE_MSIE));
+    }
+  else if (irq == RISCV_IRQ_MTIMER)
     {
       /* Read mstatus & clear machine timer interrupt enable in mie */
 
@@ -147,7 +163,13 @@ void up_enable_irq(int irq)
   int extirq;
   uint32_t oldstat;
 
-  if (irq == RISCV_IRQ_MTIMER)
+  if (irq == RISCV_IRQ_MSOFT)
+    {
+      /* Read mstatus & set machine software interrupt enable in mie */
+
+      asm volatile ("csrrs %0, mie, %1": "=r" (oldstat) : "r"(MIE_MSIE));
+    }
+  else if (irq == RISCV_IRQ_MTIMER)
     {
       /* Read mstatus & set machine timer interrupt enable in mie */
 

arch/risc-v/src/qemu-rv/qemu_rv_irq_dispatch.c

@@ -52,7 +52,7 @@
  * Public Data
  ****************************************************************************/
 
-volatile uintptr_t *g_current_regs[1];
+volatile uintptr_t *g_current_regs[CONFIG_SMP_NCPUS];
 
 /****************************************************************************
  * Public Functions
@@ -67,6 +67,11 @@ void *riscv_dispatch_irq(uintptr_t vector, uintptr_t *regs)
   uintptr_t irq = (vector >> RV_IRQ_MASK) | (vector & 0xf);
   uintptr_t *mepc = regs;
 
+  if (vector < RISCV_IRQ_ECALLM)
+    {
+      riscv_fault(irq, regs);
+    }
+
   /* Firstly, check if the irq is machine external interrupt */
 
   if (RISCV_IRQ_MEXT == irq)
@@ -97,9 +102,14 @@ void *riscv_dispatch_irq(uintptr_t vector, uintptr_t *regs)
   DEBUGASSERT(CURRENT_REGS == NULL);
   CURRENT_REGS = regs;
 
-  /* Deliver the IRQ */
+  /* MEXT means no interrupt */
 
-  irq_dispatch(irq, regs);
+  if (RISCV_IRQ_MEXT != irq)
+    {
+      /* Deliver the IRQ */
+
+      irq_dispatch(irq, regs);
+    }
 
   if (RISCV_IRQ_MEXT <= irq)
     {

arch/risc-v/src/qemu-rv/qemu_rv_start.c

@@ -60,10 +60,15 @@ uintptr_t g_idle_topstack = QEMU_RV_IDLESTACK_TOP;
  * Name: qemu_rv_start
  ****************************************************************************/
 
-void qemu_rv_start(void)
+void qemu_rv_start(int mhartid)
 {
   uint32_t *dest;
 
+  if (mhartid > 0)
+    {
+      goto cpux;
+    }
+
   /* Clear .bss.  We'll do this inline (vs. calling memset) just to be
    * certain that there are no issues with the state of global variables.
    */
@@ -89,6 +94,12 @@ void qemu_rv_start(void)
 
   nx_start();
 
+cpux:
+
+#ifdef CONFIG_SMP
+  riscv_cpu_boot(mhartid);
+#endif
+
   while (true)
     {
       asm("WFI");

boards/risc-v/qemu-rv/rv-virt/README.txt

@@ -22,11 +22,11 @@
 
 4. Run the nuttx with qemu
 
-  $ qemu-system-riscv32 -M virt -cpu rv32 -bios none -kernel nuttx -nographic
+  $ qemu-system-riscv32 -M virt -cpu rv32 -smp 8 -bios none -kernel nuttx -nographic
 
   or
 
-  $ qemu-system-riscv64 -M virt -cpu rv64 -bios none -kernel nuttx -nographic
+  $ qemu-system-riscv64 -M virt -cpu rv64 -smp 8-bios none -kernel nuttx -nographic
 
 5. TODO
 

boards/risc-v/qemu-rv/rv-virt/configs/smp/defconfig

@@ -0,0 +1,74 @@
+#
+# This file is autogenerated: PLEASE DO NOT EDIT IT.
+#
+# You can use "make menuconfig" to make any modifications to the installed .config file.
+# You can then do "make savedefconfig" to generate a new defconfig file that includes your
+# modifications.
+#
+# CONFIG_DISABLE_OS_API is not set
+# CONFIG_DISABLE_PSEUDOFS_OPERATIONS is not set
+# CONFIG_NSH_DISABLEBG is not set
+# CONFIG_NSH_DISABLE_CAT is not set
+# CONFIG_NSH_DISABLE_CD is not set
+# CONFIG_NSH_DISABLE_FREE is not set
+# CONFIG_NSH_DISABLE_HELP is not set
+# CONFIG_NSH_DISABLE_LOSMART is not set
+# CONFIG_NSH_DISABLE_LS is not set
+# CONFIG_NSH_DISABLE_MOUNT is not set
+# CONFIG_NSH_DISABLE_PS is not set
+# CONFIG_NSH_DISABLE_PSSTACKUSAGE is not set
+# CONFIG_NSH_DISABLE_UNAME is not set
+CONFIG_16550_ADDRWIDTH=0
+CONFIG_16550_UART0=y
+CONFIG_16550_UART0_BASE=0x10000000
+CONFIG_16550_UART0_CLOCK=3686400
+CONFIG_16550_UART0_IRQ=37
+CONFIG_16550_UART0_SERIAL_CONSOLE=y
+CONFIG_16550_UART=y
+CONFIG_ARCH="risc-v"
+CONFIG_ARCH_BOARD="rv-virt"
+CONFIG_ARCH_BOARD_QEMU_RV_VIRT=y
+CONFIG_ARCH_CHIP="qemu-rv"
+CONFIG_ARCH_CHIP_QEMU_RV32=y
+CONFIG_ARCH_CHIP_QEMU_RV=y
+CONFIG_ARCH_CHIP_QEMU_RV_ISA_A=y
+CONFIG_ARCH_CHIP_QEMU_RV_ISA_C=y
+CONFIG_ARCH_CHIP_QEMU_RV_ISA_M=y
+CONFIG_ARCH_INTERRUPTSTACK=2048
+CONFIG_ARCH_RISCV=y
+CONFIG_ARCH_STACKDUMP=y
+CONFIG_BINFMT_DISABLE=y
+CONFIG_BOARD_LOOPSPERMSEC=6366
+CONFIG_BUILTIN=y
+CONFIG_CLOCK_MONOTONIC=y
+CONFIG_DEBUG_FULLOPT=y
+CONFIG_DEBUG_SYMBOLS=y
+CONFIG_DEFAULT_SMALL=y
+CONFIG_DEV_ZERO=y
+CONFIG_FS_PROCFS=y
+CONFIG_IDLETHREAD_STACKSIZE=2048
+CONFIG_INIT_ENTRYPOINT="nsh_main"
+CONFIG_INTELHEX_BINARY=y
+CONFIG_LIBC_FLOATINGPOINT=y
+CONFIG_LIBC_PERROR_STDOUT=y
+CONFIG_LIBC_STRERROR=y
+CONFIG_NFILE_DESCRIPTORS_PER_BLOCK=6
+CONFIG_NSH_ARCHINIT=y
+CONFIG_NSH_BUILTIN_APPS=y
+CONFIG_NSH_FILEIOSIZE=64
+CONFIG_PREALLOC_TIMERS=0
+CONFIG_RAM_SIZE=33554432
+CONFIG_RAM_START=0x80000000
+CONFIG_RR_INTERVAL=200
+CONFIG_SCHED_WAITPID=y
+CONFIG_SERIAL_UART_ARCH_MMIO=y
+CONFIG_SMP=y
+CONFIG_SMP_NCPUS=8
+CONFIG_STACK_COLORATION=y
+CONFIG_START_MONTH=12
+CONFIG_START_YEAR=2021
+CONFIG_SYSTEM_NSH=y
+CONFIG_TESTING_OSTEST=y
+CONFIG_TESTING_OSTEST_FPUSIZE=528
+CONFIG_TESTING_SMP=y
+CONFIG_USEC_PER_TICK=1000

boards/risc-v/qemu-rv/rv-virt/configs/smp64/defconfig

@@ -0,0 +1,74 @@
+#
+# This file is autogenerated: PLEASE DO NOT EDIT IT.
+#
+# You can use "make menuconfig" to make any modifications to the installed .config file.
+# You can then do "make savedefconfig" to generate a new defconfig file that includes your
+# modifications.
+#
+# CONFIG_DISABLE_OS_API is not set
+# CONFIG_DISABLE_PSEUDOFS_OPERATIONS is not set
+# CONFIG_NSH_DISABLEBG is not set
+# CONFIG_NSH_DISABLE_CAT is not set
+# CONFIG_NSH_DISABLE_CD is not set
+# CONFIG_NSH_DISABLE_FREE is not set
+# CONFIG_NSH_DISABLE_HELP is not set
+# CONFIG_NSH_DISABLE_LOSMART is not set
+# CONFIG_NSH_DISABLE_LS is not set
+# CONFIG_NSH_DISABLE_MOUNT is not set
+# CONFIG_NSH_DISABLE_PS is not set
+# CONFIG_NSH_DISABLE_PSSTACKUSAGE is not set
+# CONFIG_NSH_DISABLE_UNAME is not set
+CONFIG_16550_ADDRWIDTH=0
+CONFIG_16550_UART0=y
+CONFIG_16550_UART0_BASE=0x10000000
+CONFIG_16550_UART0_CLOCK=3686400
+CONFIG_16550_UART0_IRQ=37
+CONFIG_16550_UART0_SERIAL_CONSOLE=y
+CONFIG_16550_UART=y
+CONFIG_ARCH="risc-v"
+CONFIG_ARCH_BOARD="rv-virt"
+CONFIG_ARCH_BOARD_QEMU_RV_VIRT=y
+CONFIG_ARCH_CHIP="qemu-rv"
+CONFIG_ARCH_CHIP_QEMU_RV64=y
+CONFIG_ARCH_CHIP_QEMU_RV=y
+CONFIG_ARCH_CHIP_QEMU_RV_ISA_A=y
+CONFIG_ARCH_CHIP_QEMU_RV_ISA_C=y
+CONFIG_ARCH_CHIP_QEMU_RV_ISA_M=y
+CONFIG_ARCH_INTERRUPTSTACK=2048
+CONFIG_ARCH_RISCV=y
+CONFIG_ARCH_STACKDUMP=y
+CONFIG_BINFMT_DISABLE=y
+CONFIG_BOARD_LOOPSPERMSEC=6366
+CONFIG_BUILTIN=y
+CONFIG_CLOCK_MONOTONIC=y
+CONFIG_DEBUG_FULLOPT=y
+CONFIG_DEBUG_SYMBOLS=y
+CONFIG_DEFAULT_SMALL=y
+CONFIG_DEV_ZERO=y
+CONFIG_FS_PROCFS=y
+CONFIG_IDLETHREAD_STACKSIZE=2048
+CONFIG_INIT_ENTRYPOINT="nsh_main"
+CONFIG_INTELHEX_BINARY=y
+CONFIG_LIBC_FLOATINGPOINT=y
+CONFIG_LIBC_PERROR_STDOUT=y
+CONFIG_LIBC_STRERROR=y
+CONFIG_NFILE_DESCRIPTORS_PER_BLOCK=6
+CONFIG_NSH_ARCHINIT=y
+CONFIG_NSH_BUILTIN_APPS=y
+CONFIG_NSH_FILEIOSIZE=64
+CONFIG_PREALLOC_TIMERS=0
+CONFIG_RAM_SIZE=33554432
+CONFIG_RAM_START=0x80000000
+CONFIG_RR_INTERVAL=200
+CONFIG_SCHED_WAITPID=y
+CONFIG_SERIAL_UART_ARCH_MMIO=y
+CONFIG_SMP=y
+CONFIG_SMP_NCPUS=8
+CONFIG_STACK_COLORATION=y
+CONFIG_START_MONTH=12
+CONFIG_START_YEAR=2021
+CONFIG_SYSTEM_NSH=y
+CONFIG_TESTING_OSTEST=y
+CONFIG_TESTING_OSTEST_FPUSIZE=264
+CONFIG_TESTING_SMP=y
+CONFIG_USEC_PER_TICK=1000