diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig index c2da844c5c7..d67475dffd6 100644 --- a/arch/x86_64/Kconfig +++ b/arch/x86_64/Kconfig @@ -30,6 +30,8 @@ config ARCH_INTEL64 select ARCH_HAVE_SSE41 select ARCH_HAVE_SSE42 select ARCH_HAVE_SSE4A + select ARCH_ICACHE + select ARCH_DCACHE ---help--- Intel x86_64 architecture diff --git a/arch/x86_64/include/intel64/arch.h b/arch/x86_64/include/intel64/arch.h index 1fd022d9a64..fd85fcc4377 100644 --- a/arch/x86_64/include/intel64/arch.h +++ b/arch/x86_64/include/intel64/arch.h @@ -135,14 +135,31 @@ /* CPUID Leaf Definitions */ -#define X86_64_CPUID_CAP 0x01 -# define X86_64_CPUID_01_SSE3 (1 << 0) -# define X86_64_CPUID_01_PCID (1 << 17) -# define X86_64_CPUID_01_X2APIC (1 << 21) -# define X86_64_CPUID_01_TSCDEA (1 << 24) -# define X86_64_CPUID_01_XSAVE (1 << 26) -# define X86_64_CPUID_01_RDRAND (1 << 30) -#define X86_64_CPUID_TSC 0x15 +#define X86_64_CPUID_VENDOR 0x00 +#define X86_64_CPUID_CAP 0x01 +# define X86_64_CPUID_01_SSE3 (1 << 0) +# define X86_64_CPUID_01_SSSE3 (1 << 9) +# define X86_64_CPUID_01_PCID (1 << 17) +# define X86_64_CPUID_01_SSE41 (1 << 19) +# define X86_64_CPUID_01_SSE42 (1 << 20) +# define X86_64_CPUID_01_X2APIC (1 << 21) +# define X86_64_CPUID_01_TSCDEA (1 << 24) +# define X86_64_CPUID_01_XSAVE (1 << 26) +# define X86_64_CPUID_01_RDRAND (1 << 30) +# define X86_64_CPUID_01_APICID(ebx) ((ebx) >> 24) +#define X86_64_CPUID_EXTCAP 0x07 +# define X86_64_CPUID_07_AVX2 (1 << 5) +# define X86_64_CPUID_07_AVX512F (1 << 16) +# define X86_64_CPUID_07_AVX512DQ (1 << 17) +# define X86_64_CPUID_07_SMAP (1 << 20) +# define X86_64_CPUID_07_AVX512IFMA (1 << 21) +# define X86_64_CPUID_07_CLWB (1 << 24) +# define X86_64_CPUID_07_AVX512PF (1 << 26) +# define X86_64_CPUID_07_AVX512ER (1 << 27) +# define X86_64_CPUID_07_AVX512CD (1 << 28) +# define X86_64_CPUID_07_AVX512BW (1 << 30) +# define X86_64_CPUID_07_AVX512VL (1 << 31) +#define X86_64_CPUID_TSC 0x15 /* MSR Definitions */ diff --git a/arch/x86_64/src/intel64/CMakeLists.txt b/arch/x86_64/src/intel64/CMakeLists.txt index f36f46f06e0..4525a4d7853 100644 --- a/arch/x86_64/src/intel64/CMakeLists.txt +++ b/arch/x86_64/src/intel64/CMakeLists.txt @@ -38,6 +38,7 @@ set(SRCS intel64_usestack.c intel64_systemreset.c intel64_freq.c + intel64_cache.c intel64_start.c intel64_handlers.c intel64_idle.c diff --git a/arch/x86_64/src/intel64/Kconfig b/arch/x86_64/src/intel64/Kconfig index d8cafd91896..f0b5722ec3c 100644 --- a/arch/x86_64/src/intel64/Kconfig +++ b/arch/x86_64/src/intel64/Kconfig @@ -21,6 +21,34 @@ endchoice # "Intel64 Chip Selection" config ARCH_INTEL64_HAVE_TSC bool +config ARCH_INTEL64_CACHE_LINESIZE + int "Cache line size (hardcoded)" + depends on ARCH_DCACHE || ARCH_ICACHE + default 64 + ---help--- + Cache line size. If set to 0, we read the value from CPUID, + (Probably) all new Intel CPUs have this value equal to 64. + +config ARCH_INTEL64_DCACHE_SIZE + int "Data cache line size (L1d)" + depends on ARCH_DCACHE + default 0 + ---help--- + Data cache line size (L1d). If set to 0, we read the value from CPUID. + +config ARCH_INTEL64_ICACHE_SIZE + int "Instruction cache line size (L1i)" + depends on ARCH_ICACHE + default 0 + ---help--- + Instruction cache line size (L1i). If set to 0, we read the value from CPUID. + +config ARCH_INTEL64_HAVE_CLWB + bool "CLWB support" + default n + ---help--- + Select to enable the use of CLWB to write back cache line + choice prompt "System Timer Source" default ARCH_INTEL64_TSC_DEADLINE diff --git a/arch/x86_64/src/intel64/Make.defs b/arch/x86_64/src/intel64/Make.defs index 8d6fe6383ec..082812f1f67 100644 --- a/arch/x86_64/src/intel64/Make.defs +++ b/arch/x86_64/src/intel64/Make.defs @@ -25,7 +25,7 @@ CMN_CSRCS += intel64_map_region.c intel64_regdump.c intel64_releasestack.c CMN_CSRCS += intel64_rtc.c intel64_restore_auxstate.c intel64_savestate.c CMN_CSRCS += intel64_stackframe.c intel64_schedulesigaction.c CMN_CSRCS += intel64_sigdeliver.c intel64_usestack.c x86_64_tcbinfo.c -CMN_CSRCS += intel64_systemreset.c intel64_freq.c +CMN_CSRCS += intel64_systemreset.c intel64_freq.c intel64_cache.c # Required Intel64 files diff --git a/arch/x86_64/src/intel64/intel64_cache.c b/arch/x86_64/src/intel64/intel64_cache.c new file mode 100644 index 00000000000..924afcd4aa0 --- /dev/null +++ b/arch/x86_64/src/intel64/intel64_cache.c @@ -0,0 +1,591 @@ +/**************************************************************************** + * arch/x86_64/src/intel64/intel64_cache.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include +#include + +#include + +#include "x86_64_internal.h" + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/**************************************************************************** + * Inline Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: x86_64_wbindv + ****************************************************************************/ + +static inline void x86_64_wbindv(void) +{ + asm volatile("wbinvd" : : : "memory"); +} + +/**************************************************************************** + * Name: x86_64_wbnoinvd + ****************************************************************************/ + +static inline void x86_64_wbnoinvd(void) +{ + asm volatile("wbnoinvd" : : : "memory"); +} + +/**************************************************************************** + * Name: x86_64_invd + ****************************************************************************/ + +static inline void x86_64_invd(void) +{ + asm volatile("invd" : : : "memory"); +} + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: x86_64_cache_linesize + ****************************************************************************/ + +static size_t x86_64_cache_linesize(void) +{ +#if CONFIG_ARCH_INTEL64_CACHE_LINESIZE == 0 + unsigned long eax = 0; + unsigned long ebx = 0; + + eax = 1; + asm volatile("cpuid\n\t" + : "=b" (ebx) + : "a" (eax)); + + return ((ebx >> 8) & 0xff) * 8; +#else + return CONFIG_ARCH_INTEL64_CACHE_LINESIZE; +#endif +} + +/**************************************************************************** + * Name: x86_64_cache_size + ****************************************************************************/ + +static size_t x86_64_cache_size(int leaf) +{ + unsigned long eax; + unsigned long ebx; + unsigned long ecx; + unsigned long edx; + + /* The leaf 0 is Data cache */ + + eax = 4; + ecx = leaf; + asm volatile("cpuid" + : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) + : "a"(eax), "c"(ecx)); + + /* (Ways + 1) * (Partitions + 1) * (Line_Size + 1) * (Sets + 1) */ + + return ((((ebx >> 22) & 0x3ff) + 1) * + (((ebx >> 12) & 0x3ff) + 1) * + ((ebx & 0x7ff) + 1) * + (ecx + 1)); +} + +/**************************************************************************** + * Name: x86_64_cache_enable + ****************************************************************************/ + +static void x86_64_cache_enable(void) +{ + /* Clear "Not-write through" (NW) and "Cache disable" (CD) bits */ + + asm volatile("\t mov %%cr0, %%rax\n" + "\t mov $0x9fffffff, %%rbx\n" + "\t and %%rbx, %%rax\n" + "\t mov %%rax, %%cr0\n" + ::: "memory", "rax", "rbx"); +} + +/**************************************************************************** + * Name: x86_64_cache_disable + ****************************************************************************/ + +static void x86_64_cache_disable(void) +{ + /* Set "Not-write through" (NW) and "Cache disable" (CD) bits */ + + asm volatile("\t mov %%cr0, %%rax\n" + "\t mov $0x9fffffff, %%rbx \n" + "\t and %%rbx, %%rax \n" + "\t mov $0x60000000, %%rbx\n" + "\t or %%rbx, %%rax\n" + "\t mov %%rax, %%cr0\n" + :::"memory", "rax", "rbx"); + + /* And flush all caches */ + + up_flush_dcache_all(); + + /* Disable MTRR */ + + write_msr(MSR_MTRR_DEF_TYPE, 0); + + /* And flush once again */ + + up_flush_dcache_all(); +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: up_get_icache_linesize + * + * Description: + * Get icache linesize + * + * Input Parameters: + * None + * + * Returned Value: + * Cache line size + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ICACHE +size_t up_get_icache_linesize(void) +{ + return x86_64_cache_linesize(); +} + +/**************************************************************************** + * Name: up_get_icache_size + * + * Description: + * Get icache size + * + * Input Parameters: + * None + * + * Returned Value: + * Cache size + * + ****************************************************************************/ + +size_t up_get_icache_size(void) +{ +#if CONFIG_ARCH_INTEL64_ICACHE_SIZE == 0 + return x86_64_cache_size(1); +#else + return CONFIG_ARCH_INTEL64_ICACHE_SIZE; +#endif +} +#endif + +/**************************************************************************** + * Name: up_enable_icache + * + * Description: + * Enable the I-Cache + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ICACHE +void up_enable_icache(void) +{ + x86_64_cache_enable(); +} +#endif + +/**************************************************************************** + * Name: up_disable_icache + * + * Description: + * Disable the I-Cache + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ICACHE +void up_disable_icache(void) +{ + x86_64_cache_disable(); +} +#endif + +/**************************************************************************** + * Name: up_invalidate_icache + * + * Description: + * Invalidate the instruction cache within the specified region. + * + * Input Parameters: + * start - virtual start address of region + * end - virtual end address of region + 1 + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ICACHE +void up_invalidate_icache(uintptr_t start, uintptr_t end) +{ + /* NOTE: x86 doesn't have separate instructions for I-cache */ + + up_invalidate_dcache(start, end); +} +#endif /* CONFIG_ARCH_ICACHE */ + +/**************************************************************************** + * Name: up_invalidate_icache_all + * + * Description: + * Invalidate the entire contents of I cache. + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ICACHE +void up_invalidate_icache_all(void) +{ + /* NOTE: x86 doesn't have separate instructions for I-cache */ + + up_invalidate_dcache_all(); +} +#endif + +/**************************************************************************** + * Name: up_get_dcache_linesize + * + * Description: + * Get dcache linesize + * + * Input Parameters: + * None + * + * Returned Value: + * Cache line size + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +size_t up_get_dcache_linesize(void) +{ + return x86_64_cache_linesize(); +} + +/**************************************************************************** + * Name: up_get_dcache_size + * + * Description: + * Get icache size + * + * Input Parameters: + * None + * + * Returned Value: + * Cache size + * + ****************************************************************************/ + +size_t up_get_dcache_size(void) +{ +#if CONFIG_ARCH_INTEL64_DCACHE_SIZE == 0 + return x86_64_cache_size(0); +#else + return CONFIG_ARCH_INTEL64_DCACHE_SIZE; +#endif +} +#endif + +/**************************************************************************** + * Name: up_enable_dcache + * + * Description: + * Enable the D-Cache + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_enable_dcache(void) +{ + x86_64_cache_enable(); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_disable_dcache + * + * Description: + * Disable the D-Cache + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_disable_dcache(void) +{ + x86_64_cache_disable(); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_invalidae_dcache + * + * Description: + * Invalidate the data cache within the specified region; we will be + * performing a DMA operation in this region and we want to purge old data + * in the cache. + * + * Input Parameters: + * start - virtual start address of region + * end - virtual end address of region + 1 + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_invalidate_dcache(uintptr_t start, uintptr_t end) +{ + up_flush_dcache(start, end); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_invalidate_dcache_all + * + * Description: + * Invalidate the entire contents of D cache. + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_invalidate_dcache_all(void) +{ + x86_64_invd(); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_clean_dcache + * + * Description: + * Clean the data cache within the specified region by flushing the + * contents of the data cache to memory. + * + * Input Parameters: + * start - virtual start address of region + * end - virtual end address of region + 1 + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_clean_dcache(uintptr_t start, uintptr_t end) +{ +#ifdef CONFIG_ARCH_INTEL64_HAVE_CLWB + size_t lsize = up_get_dcache_linesize(); + + start &= ~(lsize - 1); + + asm volatile("mfence" : : : "memory"); + + do + { + asm volatile("\tclwb %0;\n" : "+m" (start)); + + /* Increment the address by the size of one cache line. */ + + start += lsize; + } + while (start < end); + + asm volatile("mfence" : : : "memory"); +#else + x86_64_wbnoinvd(); +#endif +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_clean_dcache_all + * + * Description: + * Clean the entire data cache within the specified region by flushing the + * contents of the data cache to memory. + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_clean_dcache_all(void) +{ + x86_64_wbnoinvd(); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_flush_dcache + * + * Description: + * Flush the data cache within the specified region by cleaning and + * invalidating the D cache. + * + * Input Parameters: + * start - virtual start address of region + * end - virtual end address of region + 1 + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_flush_dcache(uintptr_t start, uintptr_t end) +{ + size_t lsize = up_get_dcache_linesize(); + + start &= ~(lsize - 1); + + asm volatile("mfence" : : : "memory"); + + do + { + asm volatile("\tclflush %0;\n" : "+m" (start)); + + /* Increment the address by the size of one cache line. */ + + start += lsize; + } + while (start < end); + + asm volatile("mfence" : : : "memory"); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_flush_dcache_all + * + * Description: + * Flush the entire data cache by cleaning and invalidating the D cache. + * + * Input Parameters: + * None + * + * Returned Value: + * None + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_DCACHE +void up_flush_dcache_all(void) +{ + x86_64_wbindv(); +} +#endif /* CONFIG_ARCH_DCACHE */ + +/**************************************************************************** + * Name: up_coherent_dcache + * + * Description: + * Ensure that the I and D caches are coherent within specified region + * by cleaning the D cache (i.e., flushing the D cache contents to memory + * and invalidating the I cache. This is typically used when code has been + * written to a memory region, and will be executed. + * + * Input Parameters: + * addr - virtual start address of region + * len - Size of the address region in bytes + * + * Returned Value: + * None + * + ****************************************************************************/ + +#if defined(CONFIG_ARCH_ICACHE) && defined(CONFIG_ARCH_DCACHE) +void up_coherent_dcache(uintptr_t addr, size_t len) +{ + uintptr_t end; + + if (len > 0) + { + /* Flush any dirtcy D-Cache lines to memory */ + + end = addr + len; + up_clean_dcache(addr, end); + + /* Invalidate I-Cache lines */ + + up_invalidate_icache(addr, end); + } +} +#endif diff --git a/arch/x86_64/src/intel64/intel64_check_capability.c b/arch/x86_64/src/intel64/intel64_check_capability.c index f5c8d2805b0..e777b8554b2 100644 --- a/arch/x86_64/src/intel64/intel64_check_capability.c +++ b/arch/x86_64/src/intel64/intel64_check_capability.c @@ -56,6 +56,7 @@ void x86_64_check_and_enable_capability(void) { + unsigned long ebx; unsigned long ecx; unsigned long require; @@ -80,15 +81,33 @@ void x86_64_check_and_enable_capability(void) #endif asm volatile("cpuid" : "=c" (ecx) : "a" (X86_64_CPUID_CAP) - : "rbx", "rdx", "memory"); + : "rdx", "memory"); - /* Check x2APIC availability */ + /* Check features availability from ECX */ if ((ecx & require) != require) { goto err; } + /* Extended features */ + + require = 0; + +#ifdef CONFIG_ARCH_INTEL64_HAVE_CLWB + require |= X86_64_CPUID_07_CLWB; +#endif + + asm volatile("cpuid" : "=b" (ebx) : "a" (X86_64_CPUID_EXTCAP), "c" (0) + : "rdx", "memory"); + + /* Check features availability */ + + if ((ebx & require) != require) + { + goto err; + } + #ifdef CONFIG_ARCH_INTEL64_HAVE_XSAVE __enable_sse_avx(); #endif @@ -97,6 +116,11 @@ void x86_64_check_and_enable_capability(void) __enable_pcid(); #endif + /* Enable I- and D-Caches */ + + up_enable_icache(); + up_enable_dcache(); + return; err: