diff --git a/arch/x86_64/include/intel64/irq.h b/arch/x86_64/include/intel64/irq.h index c37ecceee68..af852b14122 100644 --- a/arch/x86_64/include/intel64/irq.h +++ b/arch/x86_64/include/intel64/irq.h @@ -418,9 +418,9 @@ /* Align registers to 64-bytes */ #ifdef CONFIG_ARCH_X86_64_AVX512 -# define XMMAREA_REG_ALIGN (13) +# define XMMAREA_REG_ALIGN (12) #else -# define XMMAREA_REG_ALIGN (7) +# define XMMAREA_REG_ALIGN (6) #endif /* Register offset in XMMAREA */ @@ -434,42 +434,43 @@ #define REG_GS (1 + XMMAREA_REG_OFFSET) #define REG_ES (2 + XMMAREA_REG_OFFSET) #define REG_DS (3 + XMMAREA_REG_OFFSET) /* Data segment selector */ +#define REG_FSBASE (4 + XMMAREA_REG_OFFSET) /* FS base */ /* Remaining regs */ -#define REG_RAX (4 + XMMAREA_REG_OFFSET) -#define REG_RBX (5 + XMMAREA_REG_OFFSET) -#define REG_RBP (6 + XMMAREA_REG_OFFSET) -#define REG_R10 (7 + XMMAREA_REG_OFFSET) -#define REG_R11 (8 + XMMAREA_REG_OFFSET) -#define REG_R12 (9 + XMMAREA_REG_OFFSET) -#define REG_R13 (10 + XMMAREA_REG_OFFSET) -#define REG_R14 (11 + XMMAREA_REG_OFFSET) -#define REG_R15 (12 + XMMAREA_REG_OFFSET) +#define REG_RAX (5 + XMMAREA_REG_OFFSET) +#define REG_RBX (6 + XMMAREA_REG_OFFSET) +#define REG_RBP (7 + XMMAREA_REG_OFFSET) +#define REG_R10 (8 + XMMAREA_REG_OFFSET) +#define REG_R11 (9 + XMMAREA_REG_OFFSET) +#define REG_R12 (10 + XMMAREA_REG_OFFSET) +#define REG_R13 (11 + XMMAREA_REG_OFFSET) +#define REG_R14 (12 + XMMAREA_REG_OFFSET) +#define REG_R15 (13 + XMMAREA_REG_OFFSET) /* ABI calling convention */ -#define REG_R9 (13 + XMMAREA_REG_OFFSET) -#define REG_R8 (14 + XMMAREA_REG_OFFSET) -#define REG_RCX (15 + XMMAREA_REG_OFFSET) -#define REG_RDX (16 + XMMAREA_REG_OFFSET) -#define REG_RSI (17 + XMMAREA_REG_OFFSET) -#define REG_RDI (18 + XMMAREA_REG_OFFSET) +#define REG_R9 (14 + XMMAREA_REG_OFFSET) +#define REG_R8 (15 + XMMAREA_REG_OFFSET) +#define REG_RCX (16 + XMMAREA_REG_OFFSET) +#define REG_RDX (17 + XMMAREA_REG_OFFSET) +#define REG_RSI (18 + XMMAREA_REG_OFFSET) +#define REG_RDI (19 + XMMAREA_REG_OFFSET) /* IRQ saved */ -#define REG_ERRCODE (19 + XMMAREA_REG_OFFSET) /* Error code */ -#define REG_RIP (20 + XMMAREA_REG_OFFSET) /* Pushed by process on interrupt processing */ -#define REG_CS (21 + XMMAREA_REG_OFFSET) -#define REG_RFLAGS (22 + XMMAREA_REG_OFFSET) -#define REG_RSP (23 + XMMAREA_REG_OFFSET) -#define REG_SS (24 + XMMAREA_REG_OFFSET) +#define REG_ERRCODE (20 + XMMAREA_REG_OFFSET) /* Error code */ +#define REG_RIP (21 + XMMAREA_REG_OFFSET) /* Pushed by process on interrupt processing */ +#define REG_CS (22 + XMMAREA_REG_OFFSET) +#define REG_RFLAGS (23 + XMMAREA_REG_OFFSET) +#define REG_RSP (24 + XMMAREA_REG_OFFSET) +#define REG_SS (25 + XMMAREA_REG_OFFSET) -#define XMMAREA_REGS (25) +#define XMMAREA_REGS (26) /* Aux register used by implementation */ -#define REG_AUX (26 + XMMAREA_REG_OFFSET) +#define REG_AUX (27 + XMMAREA_REG_OFFSET) /* NOTE 2: This is not really state data. Rather, this is just a convenient * way to pass parameters from the interrupt handler to C code. @@ -666,7 +667,8 @@ static inline_function uint64_t read_fsbase(void) return val; } -static inline_function void write_fsbase(unsigned long val) +nostackprotect_function +static inline void write_fsbase(unsigned long val) { __asm__ volatile("wrfsbase %0" : /* no output */ diff --git a/arch/x86_64/src/intel64/intel64_checkstack.c b/arch/x86_64/src/intel64/intel64_checkstack.c index 32bc5968f14..6b212d380b2 100644 --- a/arch/x86_64/src/intel64/intel64_checkstack.c +++ b/arch/x86_64/src/intel64/intel64_checkstack.c @@ -25,19 +25,22 @@ ****************************************************************************/ #include + #include "sched/sched.h" #include "x86_64_internal.h" -#ifdef CONFIG_STACK_COLORATION - /**************************************************************************** - * Private Functions + * Pre-processor Definitions ****************************************************************************/ +#define X86_64_RED_ZONE 256 + /**************************************************************************** * Public Functions ****************************************************************************/ +#ifdef CONFIG_STACK_COLORATION + /**************************************************************************** * Name: x86_64_stack_check * @@ -108,7 +111,8 @@ void x86_64_stack_color(void *stackbase, size_t nbytes) /* Take extra care that we do not write outside the stack boundaries */ - stkptr = (uint32_t *)STACK_ALIGN_UP((uintptr_t)stackbase); + stkptr = (uint32_t *)STACK_ALIGN_UP( + (uintptr_t)(stackbase + X86_64_RED_ZONE)); if (nbytes == 0) /* 0: colorize the running stack */ { diff --git a/arch/x86_64/src/intel64/intel64_cpustart.c b/arch/x86_64/src/intel64/intel64_cpustart.c index 81ce6af6e43..466cc771081 100644 --- a/arch/x86_64/src/intel64/intel64_cpustart.c +++ b/arch/x86_64/src/intel64/intel64_cpustart.c @@ -37,6 +37,8 @@ #include "sched/sched.h" #include "init/init.h" +#include "x86_64_internal.h" + #include "intel64_lowsetup.h" #include "intel64_cpu.h" #include "x86_64_hwdebug.h" @@ -152,6 +154,14 @@ void x86_64_ap_boot(void) tcb = current_task(cpu); UNUSED(tcb); +#ifdef CONFIG_SCHED_THREAD_LOCAL + /* Make sure that FS_BASE is not null */ + + write_fsbase((uintptr_t)tcb->stack_alloc_ptr + + sizeof(struct tls_info_s) + + (_END_TBSS - _START_TDATA)); +#endif + /* Configure interrupts */ up_irqinitialize(); diff --git a/arch/x86_64/src/intel64/intel64_fullcontextrestore.S b/arch/x86_64/src/intel64/intel64_fullcontextrestore.S index 8d18c166716..4b5427d8c52 100644 --- a/arch/x86_64/src/intel64/intel64_fullcontextrestore.S +++ b/arch/x86_64/src/intel64/intel64_fullcontextrestore.S @@ -126,7 +126,7 @@ x86_64_fullcontextrestore: */ #ifdef CONFIG_SCHED_THREAD_LOCAL - mov (8*REG_FS)(%rdi), %rax + mov (8*REG_FSBASE)(%rdi), %rax wrfsbase %rax #endif diff --git a/arch/x86_64/src/intel64/intel64_initialstate.c b/arch/x86_64/src/intel64/intel64_initialstate.c index f9cc6f21cd7..7ebb6788a87 100644 --- a/arch/x86_64/src/intel64/intel64_initialstate.c +++ b/arch/x86_64/src/intel64/intel64_initialstate.c @@ -47,6 +47,10 @@ # error XCPTCONTEXT_SIZE must be aligned to XCPTCONTEXT_ALIGN ! #endif +#if defined(CONFIG_STACK_CANARIES) && !defined(CONFIG_SCHED_THREAD_LOCAL) +# error x86_64 stack canaries requires TLS support ! +#endif + /* Aligned size of the kernel stack */ #ifdef CONFIG_ARCH_KERNEL_STACK @@ -177,17 +181,20 @@ void up_initial_state(struct tcb_s *tcb) */ #ifdef CONFIG_SCHED_THREAD_LOCAL - xcp->regs[REG_FS] = (uintptr_t)tcb->stack_alloc_ptr - + sizeof(struct tls_info_s) - + (_END_TBSS - _START_TDATA); + xcp->regs[REG_FSBASE] = ((uintptr_t)tcb->stack_alloc_ptr + + sizeof(struct tls_info_s) + + (_END_TBSS - _START_TDATA)); - *(uint64_t *)(xcp->regs[REG_FS]) = xcp->regs[REG_FS]; + *(uint64_t *)(xcp->regs[REG_FSBASE]) = xcp->regs[REG_FSBASE]; + + /* Do not write FSBASE now. This task is not running yet! */ - write_fsbase(xcp->regs[REG_FS]); #else - xcp->regs[REG_FS] = 0; + xcp->regs[REG_FSBASE] = 0; #endif + xcp->regs[REG_FS] = 0; + /* GS used for CPU private data */ xcp->regs[REG_GS] = 0; diff --git a/arch/x86_64/src/intel64/intel64_start.c b/arch/x86_64/src/intel64/intel64_start.c index e9471d47d05..1787925001a 100644 --- a/arch/x86_64/src/intel64/intel64_start.c +++ b/arch/x86_64/src/intel64/intel64_start.c @@ -37,8 +37,8 @@ #include "x86_64_internal.h" -#include "intel64_cpu.h" #include "intel64_lowsetup.h" +#include "intel64_cpu.h" /**************************************************************************** * Public Data @@ -160,6 +160,15 @@ void __nxstart(void) *dest++ = 0; } +#ifdef CONFIG_SCHED_THREAD_LOCAL + /* Make sure that FS_BASE is not null */ + + write_fsbase((uintptr_t)(g_idle_topstack[0] - + CONFIG_IDLETHREAD_STACKSIZE + + sizeof(struct tls_info_s) + + (_END_TBSS - _START_TDATA))); +#endif + /* Low-level, pre-OS initialization */ intel64_lowsetup(); diff --git a/arch/x86_64/src/intel64/intel64_vectors.S b/arch/x86_64/src/intel64/intel64_vectors.S index 56c019524a7..c74264990ff 100644 --- a/arch/x86_64/src/intel64/intel64_vectors.S +++ b/arch/x86_64/src/intel64/intel64_vectors.S @@ -746,6 +746,11 @@ irq_common: mov %fs, %ax /* Lower 16-bits of rax. */ movq %rax, (8*REG_FS)(%rdi) /* Save the data segment descriptor */ +#ifdef CONFIG_SCHED_THREAD_LOCAL + rdfsbase %rax + movq %rax, (8*REG_FSBASE)(%rdi) +#endif + /* Save registers from stack */ movq 0(%rsp), %rcx