arch/arm: add barrier instruction for cache ops

Signed-off-by: zhangyuan21 <zhangyuan21@xiaomi.com>

arch/arm/src/armv7-a/arm_l2cc_pl310.c

@@ -37,6 +37,7 @@
 #include <nuttx/irq.h>
 
 #include "arm_internal.h"
+#include "barriers.h"
 #include "l2cc.h"
 #include "l2cc_pl310.h"
 
@@ -235,10 +236,6 @@
 #  define OK                       0
 #endif
 
-/* Data synchronization barrier */
-
-#define dsb(a) __asm__ __volatile__ ("dsb " #a : : : "memory")
-
 /****************************************************************************
  * Private Functions
  ****************************************************************************/
@@ -404,6 +401,8 @@ void arm_l2ccinitialize(void)
 
       l2cc_invalidate_all();
       putreg32(L2CC_CR_L2CEN, L2CC_CR);
+      ARM_DSB();
+      ARM_ISB();
     }
 
   sinfo("(%d ways) * (%d bytes/way) = %d bytes\n",
@@ -434,6 +433,8 @@ void l2cc_enable(void)
   flags = enter_critical_section();
   l2cc_invalidate_all();
   putreg32(L2CC_CR_L2CEN, L2CC_CR);
+  ARM_DSB();
+  ARM_ISB();
   leave_critical_section(flags);
 }
 
@@ -463,7 +464,8 @@ void l2cc_disable(void)
   /* Disable the L2CC-P310 L2 cache by clearing the Control Register (CR) */
 
   putreg32(0, L2CC_CR);
-  dsb();
+  ARM_DSB();
+  ARM_ISB();
   leave_critical_section(flags);
 }
 

arch/arm/src/armv7-a/cp15_cacheops.h

@@ -225,6 +225,7 @@
   mrc p15, 0, \tmp, c1, c0, 0 /* Read SCTLR */
   orr \tmp, \tmp, #(0x1 << 2) /* Enable D cache */
   mcr p15, 0, \tmp, c1, c0, 0 /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -245,6 +246,7 @@
   mrc p15, 0, \tmp, c1, c0, 0 /* Read SCTLR */
   bic \tmp, \tmp, #(0x1 << 2) /* Disable D cache */
   mcr p15, 0, \tmp, c1, c0, 0 /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -265,6 +267,7 @@
   mrc p15, 0, \tmp, c1, c0, 0  /* Read SCTLR */
   orr \tmp, \tmp, #(0x1 << 12) /* Enable I cache */
   mcr p15, 0, \tmp, c1, c0, 0  /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -285,6 +288,7 @@
   mrc p15, 0, \tmp, c1, c0, 0  /* Read SCTLR */
   bic \tmp, \tmp, #(0x1 << 12) /* Disable I cache */
   mcr p15, 0, \tmp, c1, c0, 0  /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -304,6 +308,7 @@
 .macro cp15_invalidate_icache_inner_sharable, tmp
   mov \tmp, #0
   mrc p15, 0, \tmp, c7, c1, 0 /* ICIALLUIS */
+  isb
 .endm
 
 /****************************************************************************
@@ -323,6 +328,7 @@
 .macro cp15_invalidate_btb_inner_sharable, tmp
   mov \tmp, #0
   mrc p15, 0, \tmp, c7, c1, 6 /* BPIALLIS */
+  isb
 .endm
 
 /****************************************************************************
@@ -362,6 +368,7 @@
 
 .macro cp15_invalidate_icache_bymva, va
   mrc p15, 0, \va, c7, c5, 1 /* ICIMVAU */
+  isb
 .endm
 
 /****************************************************************************
@@ -381,6 +388,7 @@
 .macro cp15_flush_btb, tmp
   mov \tmp, #0
   mrc p15, 0, \tmp, c7, c5, 6 /* BPIALL */
+  isb
 .endm
 
 /****************************************************************************
@@ -399,6 +407,7 @@
 
 .macro cp15_flush_btb_bymva, va
   mrc p15, 0, \va, c7, c5, 7 /* BPIMVA */
+  isb
 .endm
 
 /****************************************************************************
@@ -417,6 +426,7 @@
 
 .macro cp15_invalidate_dcacheline_bymva, va
   mrc p15, 0, \va, c7, c6, 1 /* DCIMVAC */
+  isb
 .endm
 
 /****************************************************************************
@@ -435,6 +445,7 @@
 
 .macro cp15_invalidate_dcacheline_bysetway, setway
   mrc p15, 0, \setway, c7, c6, 2 /* DCISW */
+  isb
 .endm
 
 /****************************************************************************
@@ -453,6 +464,7 @@
 
 .macro cp15_clean_dcache_bymva, va
   mrc p15, 0, \va, c7, c10, 1 /* DCCMVAC */
+  isb
 .endm
 
 /****************************************************************************
@@ -471,6 +483,7 @@
 
 .macro cp15_clean_dcache_bysetway, setway
   mrc p15, 0, \setway, c7, c10, 2 /* DCCSW */
+  isb
 .endm
 
 /****************************************************************************
@@ -489,6 +502,7 @@
 
 .macro cp15_clean_ucache_bymva, va
   mrc p15, 0, \va, c7, c11, 1 /* DCCMVAU */
+  isb
 .endm
 
 /****************************************************************************
@@ -507,6 +521,7 @@
 
 .macro cp15_cleaninvalidate_dcacheline_bymva, va
   mrc p15, 0, \va, c7, c14, 1 /* DCCIMVAC */
+  isb
 .endm
 
 /****************************************************************************
@@ -525,6 +540,7 @@
 
 .macro cp15_cleaninvalidate_dcacheline, setway
   mrc p15, 0, \setway, c7, c14, 2 /* DCCISW */
+  isb
 .endm
 
 #endif /* __ASSEMBLY__ */
@@ -556,6 +572,7 @@ static inline void cp15_enable_dcache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr |= SCTLR_C;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -579,6 +596,7 @@ static inline void cp15_disable_dcache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr &= ~SCTLR_C;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -602,6 +620,7 @@ static inline void cp15_enable_icache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr |= SCTLR_I;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -625,6 +644,7 @@ static inline void cp15_disable_icache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr &= ~SCTLR_I;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -644,6 +664,7 @@ static inline void cp15_disable_icache(void)
 static inline void cp15_invalidate_icache_inner_sharable(void)
 {
   CP15_SET(ICIALLUIS, 0);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -663,6 +684,7 @@ static inline void cp15_invalidate_icache_inner_sharable(void)
 static inline void cp15_invalidate_btb_inner_sharable(void)
 {
   CP15_SET(BPIALLIS, 0);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -703,6 +725,7 @@ static inline void cp15_invalidate_icache_all(void)
 static inline void cp15_invalidate_icache_bymva(unsigned int va)
 {
   CP15_SET(ICIMVAU, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -722,6 +745,7 @@ static inline void cp15_invalidate_icache_bymva(unsigned int va)
 static inline void cp15_flush_btb(void)
 {
   CP15_SET(BPIALL, 0);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -741,6 +765,7 @@ static inline void cp15_flush_btb(void)
 static inline void cp15_flush_btb_bymva(unsigned int va)
 {
   CP15_SET(BPIMVA, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -762,6 +787,7 @@ static inline void cp15_flush_btb_bymva(unsigned int va)
 static inline void cp15_invalidate_dcacheline_bymva(unsigned int va)
 {
   CP15_SET(DCIMVAC, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -783,6 +809,7 @@ static inline void cp15_invalidate_dcacheline_bymva(unsigned int va)
 static inline void cp15_invalidate_dcacheline_bysetway(unsigned int setway)
 {
   CP15_SET(DCISW, setway);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -804,6 +831,7 @@ static inline void cp15_invalidate_dcacheline_bysetway(unsigned int setway)
 static inline void cp15_clean_dcache_bymva(unsigned int va)
 {
   CP15_SET(DCCMVAC, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -823,6 +851,7 @@ static inline void cp15_clean_dcache_bymva(unsigned int va)
 static inline void cp15_clean_dcache_bysetway(unsigned int setway)
 {
   CP15_SET(DCCSW, setway);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -842,6 +871,7 @@ static inline void cp15_clean_dcache_bysetway(unsigned int setway)
 static inline void cp15_clean_ucache_bymva(unsigned int va)
 {
   CP15_SET(DCCMVAU, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -861,6 +891,7 @@ static inline void cp15_clean_ucache_bymva(unsigned int va)
 static inline void cp15_cleaninvalidate_dcacheline_bymva(unsigned int va)
 {
   CP15_SET(DCCIMVAC, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -880,6 +911,7 @@ static inline void cp15_cleaninvalidate_dcacheline_bymva(unsigned int va)
 static inline void cp15_cleaninvalidate_dcacheline(unsigned int setway)
 {
   CP15_SET(DCCISW, setway);
+  ARM_ISB();
 }
 
 #endif /* __ASSEMBLY__ */

arch/arm/src/armv7-r/arm_l2cc_pl310.c

@@ -37,6 +37,7 @@
 #include <nuttx/irq.h>
 
 #include "arm_internal.h"
+#include "barriers.h"
 #include "l2cc.h"
 #include "l2cc_pl310.h"
 
@@ -235,10 +236,6 @@
 #  define OK                       0
 #endif
 
-/* Data synchronization barrier */
-
-#define dsb(a) __asm__ __volatile__ ("dsb " #a : : : "memory")
-
 /****************************************************************************
  * Private Functions
  ****************************************************************************/
@@ -404,6 +401,8 @@ void arm_l2ccinitialize(void)
 
       l2cc_invalidate_all();
       putreg32(L2CC_CR_L2CEN, L2CC_CR);
+      ARM_DSB();
+      ARM_ISB();
     }
 
   sinfo("(%d ways) * (%d bytes/way) = %d bytes\n",
@@ -434,6 +433,8 @@ void l2cc_enable(void)
   flags = enter_critical_section();
   l2cc_invalidate_all();
   putreg32(L2CC_CR_L2CEN, L2CC_CR);
+  ARM_DSB();
+  ARM_ISB();
   leave_critical_section(flags);
 }
 
@@ -463,7 +464,8 @@ void l2cc_disable(void)
   /* Disable the L2CC-P310 L2 cache by clearing the Control Register (CR) */
 
   putreg32(0, L2CC_CR);
-  dsb();
+  ARM_DSB();
+  ARM_ISB();
   leave_critical_section(flags);
 }
 

arch/arm/src/armv7-r/cp15_cacheops.h

@@ -232,6 +232,7 @@
   mrc p15, 0, \tmp, c1, c0, 0 /* Read SCTLR */
   orr \tmp, \tmp, #(0x1 << 2) /* Enable D cache */
   mcr p15, 0, \tmp, c1, c0, 0 /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -252,6 +253,7 @@
   mrc p15, 0, \tmp, c1, c0, 0 /* Read SCTLR */
   bic \tmp, \tmp, #(0x1 << 2) /* Disable D cache */
   mcr p15, 0, \tmp, c1, c0, 0 /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -272,6 +274,7 @@
   mrc p15, 0, \tmp, c1, c0, 0  /* Read SCTLR */
   orr \tmp, \tmp, #(0x1 << 12) /* Enable I cache */
   mcr p15, 0, \tmp, c1, c0, 0  /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -292,6 +295,7 @@
   mrc p15, 0, \tmp, c1, c0, 0  /* Read SCTLR */
   bic \tmp, \tmp, #(0x1 << 12) /* Disable I cache */
   mcr p15, 0, \tmp, c1, c0, 0  /* Update the SCTLR */
+  isb
 .endm
 
 /****************************************************************************
@@ -311,6 +315,7 @@
 .macro cp15_invalidate_icache_inner_sharable, tmp
   mov \tmp, #0
   mrc p15, 0, \tmp, c7, c1, 0 /* ICIALLUIS */
+  isb
 .endm
 
 /****************************************************************************
@@ -330,6 +335,7 @@
 .macro cp15_invalidate_btb_inner_sharable, tmp
   mov \tmp, #0
   mrc p15, 0, \tmp, c7, c1, 6 /* BPIALLIS */
+  isb
 .endm
 
 /****************************************************************************
@@ -369,6 +375,7 @@
 
 .macro cp15_invalidate_icache_bymva, va
   mrc p15, 0, \va, c7, c5, 1 /* ICIMVAU */
+  isb
 .endm
 
 /****************************************************************************
@@ -388,6 +395,7 @@
 .macro cp15_flush_btb, tmp
   mov \tmp, #0
   mrc p15, 0, \tmp, c7, c5, 6 /* BPIALL */
+  isb
 .endm
 
 /****************************************************************************
@@ -406,6 +414,7 @@
 
 .macro cp15_flush_btb_bymva, va
   mrc p15, 0, \va, c7, c5, 7 /* BPIMVA */
+  isb
 .endm
 
 /****************************************************************************
@@ -424,6 +433,7 @@
 
 .macro cp15_invalidate_dcacheline_bymva, va
   mrc p15, 0, \va, c7, c6, 1 /* DCIMVAC */
+  isb
 .endm
 
 /****************************************************************************
@@ -442,6 +452,7 @@
 
 .macro cp15_invalidate_dcacheline_bysetway, setway
   mrc p15, 0, \setway, c7, c6, 2 /* DCISW */
+  isb
 .endm
 
 /****************************************************************************
@@ -460,6 +471,7 @@
 
 .macro cp15_clean_dcache_bymva, va
   mrc p15, 0, \va, c7, c10, 1 /* DCCMVAC */
+  isb
 .endm
 
 /****************************************************************************
@@ -478,6 +490,7 @@
 
 .macro cp15_clean_dcache_bysetway, setway
   mrc p15, 0, \setway, c7, c10, 2 /* DCCSW */
+  isb
 .endm
 
 /****************************************************************************
@@ -496,6 +509,7 @@
 
 .macro cp15_clean_ucache_bymva, va
   mrc p15, 0, \va, c7, c11, 1 /* DCCMVAU */
+  isb
 .endm
 
 /****************************************************************************
@@ -514,6 +528,7 @@
 
 .macro cp15_cleaninvalidate_dcacheline_bymva, va
   mrc p15, 0, \va, c7, c14, 1 /* DCCIMVAC */
+  isb
 .endm
 
 /****************************************************************************
@@ -532,6 +547,7 @@
 
 .macro cp15_cleaninvalidate_dcacheline, setway
   mrc p15, 0, \setway, c7, c14, 2 /* DCCISW */
+  isb
 .endm
 
 #endif /* __ASSEMBLY__ */
@@ -563,6 +579,7 @@ static inline void cp15_enable_dcache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr |= SCTLR_C;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -586,6 +603,7 @@ static inline void cp15_disable_dcache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr &= ~SCTLR_C;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -609,6 +627,7 @@ static inline void cp15_enable_icache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr |= SCTLR_I;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -632,6 +651,7 @@ static inline void cp15_disable_icache(void)
   sctlr = CP15_GET(SCTLR);
   sctlr &= ~SCTLR_I;
   CP15_SET(SCTLR, sctlr);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -651,6 +671,7 @@ static inline void cp15_disable_icache(void)
 static inline void cp15_invalidate_icache_inner_sharable(void)
 {
   CP15_SET(ICIALLUIS, 0);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -670,6 +691,7 @@ static inline void cp15_invalidate_icache_inner_sharable(void)
 static inline void cp15_invalidate_btb_inner_sharable(void)
 {
   CP15_SET(BPIALLIS, 0);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -710,6 +732,7 @@ static inline void cp15_invalidate_icache_all(void)
 static inline void cp15_invalidate_icache_bymva(unsigned int va)
 {
   CP15_SET(ICIMVAU, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -729,6 +752,7 @@ static inline void cp15_invalidate_icache_bymva(unsigned int va)
 static inline void cp15_flush_btb(void)
 {
   CP15_SET(BPIALL, 0);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -748,6 +772,7 @@ static inline void cp15_flush_btb(void)
 static inline void cp15_flush_btb_bymva(unsigned int va)
 {
   CP15_SET(BPIMVA, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -769,6 +794,7 @@ static inline void cp15_flush_btb_bymva(unsigned int va)
 static inline void cp15_invalidate_dcacheline_bymva(unsigned int va)
 {
   CP15_SET(DCIMVAC, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -790,6 +816,7 @@ static inline void cp15_invalidate_dcacheline_bymva(unsigned int va)
 static inline void cp15_invalidate_dcacheline_bysetway(unsigned int setway)
 {
   CP15_SET(DCISW, setway);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -811,6 +838,7 @@ static inline void cp15_invalidate_dcacheline_bysetway(unsigned int setway)
 static inline void cp15_clean_dcache_bymva(unsigned int va)
 {
   CP15_SET(DCCMVAC, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -830,6 +858,7 @@ static inline void cp15_clean_dcache_bymva(unsigned int va)
 static inline void cp15_clean_dcache_bysetway(unsigned int setway)
 {
   CP15_SET(DCCSW, setway);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -849,6 +878,7 @@ static inline void cp15_clean_dcache_bysetway(unsigned int setway)
 static inline void cp15_clean_ucache_bymva(unsigned int va)
 {
   CP15_SET(DCCMVAU, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -868,6 +898,7 @@ static inline void cp15_clean_ucache_bymva(unsigned int va)
 static inline void cp15_cleaninvalidate_dcacheline_bymva(unsigned int va)
 {
   CP15_SET(DCCIMVAC, va);
+  ARM_ISB();
 }
 
 /****************************************************************************
@@ -887,6 +918,7 @@ static inline void cp15_cleaninvalidate_dcacheline_bymva(unsigned int va)
 static inline void cp15_cleaninvalidate_dcacheline(unsigned int setway)
 {
   CP15_SET(DCCISW, setway);
+  ARM_ISB();
 }
 
 #endif /* __ASSEMBLY__ */