diff --git a/bsps/x86_64/amd64/start/ap_trampoline.S b/bsps/x86_64/amd64/start/ap_trampoline.S index e1268d5f86..6f3eea0828 100644 --- a/bsps/x86_64/amd64/start/ap_trampoline.S +++ b/bsps/x86_64/amd64/start/ap_trampoline.S @@ -113,6 +113,7 @@ SYM(_Trampoline_start): movw %ax, %ds movw %ax, %es movw %ax, %ss + movw %ax, %fs # Acquire the processor's stack GET_SELF_CPU_CONTROL_RAX diff --git a/bsps/x86_64/amd64/start/start.S b/bsps/x86_64/amd64/start/start.S index b876b7be7e..2cd57752d5 100644 --- a/bsps/x86_64/amd64/start/start.S +++ b/bsps/x86_64/amd64/start/start.S @@ -60,6 +60,7 @@ _start: movw %ax, %ds movw %ax, %es movw %ax, %ss + movw %ax, %fs /* Load code segment register */ pushq $GDT_CODE_SEG_OFFSET diff --git a/cpukit/include/rtems/score/tls.h b/cpukit/include/rtems/score/tls.h index 8716c5230c..508fa52beb 100644 --- a/cpukit/include/rtems/score/tls.h +++ b/cpukit/include/rtems/score/tls.h @@ -116,9 +116,9 @@ typedef struct { } TLS_Dynamic_thread_vector; typedef struct TLS_Thread_control_block { -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) struct TLS_Thread_control_block *tcb; -#else /* !__i386__ */ +#else /* !(__i386__ || __x86_64__) */ TLS_Dynamic_thread_vector *dtv; /* * GCC under AArch64/LP64 expects a 16 byte TCB at the beginning of the TLS @@ -127,7 +127,7 @@ typedef struct TLS_Thread_control_block { #if CPU_SIZEOF_POINTER == 4 || defined(AARCH64_MULTILIB_ARCH_V8) uintptr_t reserved; #endif -#endif /* __i386__ */ +#endif /* __i386__ || __x86_64__ */ } TLS_Thread_control_block; typedef struct { @@ -204,7 +204,7 @@ static inline void _TLS_Initialize_TCB_and_DTV( TLS_Dynamic_thread_vector *dtv ) { -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) (void) dtv; tcb->tcb = tcb; #else @@ -242,7 +242,7 @@ static inline void *_TLS_Initialize_area( void *tls_area ) config = &_TLS_Configuration; alignment = (uintptr_t) config->alignment; -#ifdef __i386__ +#if defined(__i386__) || defined(__x86_64__) dtv = NULL; #else dtv = (TLS_Dynamic_thread_vector *) tls_area; diff --git a/cpukit/score/cpu/x86_64/include/rtems/asm.h b/cpukit/score/cpu/x86_64/include/rtems/asm.h index 26470e9d44..d4e11622ac 100644 --- a/cpukit/score/cpu/x86_64/include/rtems/asm.h +++ b/cpukit/score/cpu/x86_64/include/rtems/asm.h @@ -199,6 +199,30 @@ .macro GET_SELF_CPU_CONTROL_RSI GET_SELF_CPU_CONTROL rsi,%esi .endm +.macro GET_SELF_CPU_CONTROL_R8 + GET_SELF_CPU_CONTROL r8,%r8d +.endm +.macro GET_SELF_CPU_CONTROL_R9 + GET_SELF_CPU_CONTROL r9,%r9d +.endm +.macro GET_SELF_CPU_CONTROL_R10 + GET_SELF_CPU_CONTROL r10,%r10d +.endm +.macro GET_SELF_CPU_CONTROL_R11 + GET_SELF_CPU_CONTROL r11,%r11d +.endm +.macro GET_SELF_CPU_CONTROL_R12 + GET_SELF_CPU_CONTROL r12,%r12d +.endm +.macro GET_SELF_CPU_CONTROL_R13 + GET_SELF_CPU_CONTROL r13,%r13d +.endm +.macro GET_SELF_CPU_CONTROL_R14 + GET_SELF_CPU_CONTROL r14,%r14d +.endm +.macro GET_SELF_CPU_CONTROL_R15 + GET_SELF_CPU_CONTROL r15,%r15d +.endm #endif // _RTEMS_ASM_H diff --git a/cpukit/score/cpu/x86_64/include/rtems/score/cpu.h b/cpukit/score/cpu/x86_64/include/rtems/score/cpu.h index 5db67dcb18..5b4b0395d7 100644 --- a/cpukit/score/cpu/x86_64/include/rtems/score/cpu.h +++ b/cpukit/score/cpu/x86_64/include/rtems/score/cpu.h @@ -73,7 +73,9 @@ extern "C" { #define CPU_CONTEXT_CONTROL_R14 CPU_CONTEXT_CONTROL_R13 + 8 #define CPU_CONTEXT_CONTROL_R15 CPU_CONTEXT_CONTROL_R14 + 8 -#define CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE CPU_CONTEXT_CONTROL_R15 + 8 +#define CPU_CONTEXT_CONTROL_FS CPU_CONTEXT_CONTROL_R15 + 8 + +#define CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE CPU_CONTEXT_CONTROL_FS + 8 #define CPU_CONTEXT_CONTROL_IS_EXECUTING CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE + 4 @@ -94,9 +96,10 @@ typedef struct { uint64_t r14; uint64_t r15; - uint32_t isr_dispatch_disable; + /* Thread pointer */ + uint64_t fs; - // XXX: FS segment descriptor for TLS + uint32_t isr_dispatch_disable; #ifdef RTEMS_SMP volatile uint16_t is_executing; diff --git a/cpukit/score/cpu/x86_64/include/rtems/score/cpuimpl.h b/cpukit/score/cpu/x86_64/include/rtems/score/cpuimpl.h index 4529a323da..ce4b362b35 100644 --- a/cpukit/score/cpu/x86_64/include/rtems/score/cpuimpl.h +++ b/cpukit/score/cpu/x86_64/include/rtems/score/cpuimpl.h @@ -86,15 +86,16 @@ static inline void _CPU_Use_thread_local_storage( const Context_Control *context ) { - (void) context; + uint32_t high = (uint32_t) (context->fs >> 32); + uint32_t low = (uint32_t) context->fs; + wrmsr(FSBASE_MSR, low, high); } static inline void *_CPU_Get_TLS_thread_pointer( const Context_Control *context ) { - (void) context; - return NULL; + return (void*) context->fs; } #ifdef __cplusplus diff --git a/cpukit/score/cpu/x86_64/include/rtems/score/x86_64.h b/cpukit/score/cpu/x86_64/include/rtems/score/x86_64.h index 9070d9c3e8..6633575e03 100644 --- a/cpukit/score/cpu/x86_64/include/rtems/score/x86_64.h +++ b/cpukit/score/cpu/x86_64/include/rtems/score/x86_64.h @@ -38,6 +38,8 @@ extern "C" { #define EFLAGS_INTR_ENABLE 0x200 +#define FSBASE_MSR 0xC0000100 + #if DEBUG #define DBG_PRINTF(format, args...) \ printf(format, ## args) diff --git a/cpukit/score/cpu/x86_64/x86_64-context-initialize.c b/cpukit/score/cpu/x86_64/x86_64-context-initialize.c index b6bb37cd78..1edc2bb9b9 100644 --- a/cpukit/score/cpu/x86_64/x86_64-context-initialize.c +++ b/cpukit/score/cpu/x86_64/x86_64-context-initialize.c @@ -70,13 +70,11 @@ void _CPU_Context_Initialize( ) { uintptr_t _stack; + uintptr_t tcb; /* avoid warning for being unused */ (void) is_fp; - // XXX: Should be used in the future - (void) tls_area; - if ( new_level ) { the_context->rflags = CPU_EFLAGS_INTERRUPTS_OFF; } @@ -93,5 +91,10 @@ void _CPU_Context_Initialize( the_context->rbp = (void *) 0; the_context->rsp = (void *) _stack; - // XXX: Initialize thread-local storage area (TLS / TCB) + if (tls_area != NULL) { + tcb = (uintptr_t) _TLS_Initialize_area(tls_area); + } else { + tcb = 0; + } + the_context->fs = tcb; } diff --git a/cpukit/score/cpu/x86_64/x86_64-context-switch.S b/cpukit/score/cpu/x86_64/x86_64-context-switch.S index 6f2c66193a..c27cdee08d 100644 --- a/cpukit/score/cpu/x86_64/x86_64-context-switch.S +++ b/cpukit/score/cpu/x86_64/x86_64-context-switch.S @@ -56,25 +56,25 @@ PUBLIC(_CPU_Context_switch_no_return) SYM(_CPU_Context_switch): SYM(_CPU_Context_switch_no_return): - movq RUNCONTEXT_ARG, rax /* rax = running threads context */ - GET_SELF_CPU_CONTROL_RCX /* rcx = per CPU information */ + movq RUNCONTEXT_ARG, r10 /* r10 = running threads context */ + GET_SELF_CPU_CONTROL_R11 /* r11 = per CPU information */ /* Fill up Context_Control struct */ pushf - popq CPU_CONTEXT_CONTROL_EFLAGS(rax) /* pop rflags into context */ - movq rbx, CPU_CONTEXT_CONTROL_RBX(rax) - movq rsp, CPU_CONTEXT_CONTROL_RSP(rax) - movq rbp, CPU_CONTEXT_CONTROL_RBP(rax) - movq r12, CPU_CONTEXT_CONTROL_R12(rax) - movq r13, CPU_CONTEXT_CONTROL_R13(rax) - movq r14, CPU_CONTEXT_CONTROL_R14(rax) - movq r15, CPU_CONTEXT_CONTROL_R15(rax) + popq CPU_CONTEXT_CONTROL_EFLAGS(r10) /* pop rflags into context */ + movq rbx, CPU_CONTEXT_CONTROL_RBX(r10) + movq rsp, CPU_CONTEXT_CONTROL_RSP(r10) + movq rbp, CPU_CONTEXT_CONTROL_RBP(r10) + movq r12, CPU_CONTEXT_CONTROL_R12(r10) + movq r13, CPU_CONTEXT_CONTROL_R13(r10) + movq r14, CPU_CONTEXT_CONTROL_R14(r10) + movq r15, CPU_CONTEXT_CONTROL_R15(r10) - movl PER_CPU_ISR_DISPATCH_DISABLE(rcx), %edx - movl %edx, CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE(rax) + movl PER_CPU_ISR_DISPATCH_DISABLE(r11), %edx + movl %edx, CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE(r10) - movq rax, r8 /* r8 = running threads context */ - movq HEIRCONTEXT_ARG, rax /* rax = heir threads context */ + movq r10, r8 /* r8 = running threads context */ + movq HEIRCONTEXT_ARG, r10 /* r10 = heir threads context */ #ifdef RTEMS_SMP /* @@ -82,58 +82,63 @@ SYM(_CPU_Context_switch_no_return): * the stack to the temporary interrupt stack of this processor. Mark * the context of the executing thread as not executing. */ - leaq PER_CPU_INTERRUPT_FRAME_AREA + CPU_INTERRUPT_FRAME_SIZE(rcx), rsp + leaq PER_CPU_INTERRUPT_FRAME_AREA + CPU_INTERRUPT_FRAME_SIZE(r11), rsp movw $0, CPU_CONTEXT_CONTROL_IS_EXECUTING(r8) .check_is_executing: - lock btsw $0, CPU_CONTEXT_CONTROL_IS_EXECUTING(rax) /* Indicator in carry flag */ + lock btsw $0, CPU_CONTEXT_CONTROL_IS_EXECUTING(r10) /* Indicator in carry flag */ jnc .restore .get_potential_new_heir: /* We may have a new heir */ /* Read the executing and heir */ - movq PER_CPU_OFFSET_EXECUTING(rcx), r9 - movq PER_CPU_OFFSET_HEIR(rcx), r10 + movq PER_CPU_OFFSET_EXECUTING(r11), r8 + movq PER_CPU_OFFSET_HEIR(r11), r9 /* * Update the executing only if necessary to avoid cache line * monopolization. */ - cmpq r9, r10 + cmpq r8, r9 je .check_is_executing /* Calculate the heir context pointer */ - addq r10, rax - subq r9, rax + addq r9, r10 + subq r8, r10 /* Update the executing */ - movq r10, PER_CPU_OFFSET_EXECUTING(rcx) + movq r9, PER_CPU_OFFSET_EXECUTING(r11) jmp .check_is_executing #endif .restore: - movq CPU_CONTEXT_CONTROL_RBX(rax), rbx - movq CPU_CONTEXT_CONTROL_RSP(rax), rsp + movl CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE(r10), %edx + movl %edx, PER_CPU_ISR_DISPATCH_DISABLE(r11) + + movq CPU_CONTEXT_CONTROL_RBX(r10), rbx + movq CPU_CONTEXT_CONTROL_RSP(r10), rsp /* * We need to load rflags after rsp to avoid an interrupt while the ISR stack * is still being used during the initialization process */ - pushq CPU_CONTEXT_CONTROL_EFLAGS(rax) /* push rflags */ + pushq CPU_CONTEXT_CONTROL_EFLAGS(r10) /* push rflags */ popf /* restore rflags */ - movq CPU_CONTEXT_CONTROL_RBP(rax), rbp - movq CPU_CONTEXT_CONTROL_R12(rax), r12 - movq CPU_CONTEXT_CONTROL_R13(rax), r13 - movq CPU_CONTEXT_CONTROL_R14(rax), r14 - movq CPU_CONTEXT_CONTROL_R15(rax), r15 + movq CPU_CONTEXT_CONTROL_RBP(r10), rbp + movq CPU_CONTEXT_CONTROL_R12(r10), r12 + movq CPU_CONTEXT_CONTROL_R13(r10), r13 + movq CPU_CONTEXT_CONTROL_R14(r10), r14 + movq CPU_CONTEXT_CONTROL_R15(r10), r15 - movl CPU_CONTEXT_CONTROL_ISR_DISPATCH_DISABLE(rax), %edx - movl %edx, PER_CPU_ISR_DISPATCH_DISABLE(rcx) - - /* XXX: TLS - load GDT and refresh FS segment selector */ + movq CPU_CONTEXT_CONTROL_FS(r10), rax + /* High bits in %edx and low bits in %eax */ + movq rax, rdx + shrq $32, rdx + movl $FSBASE_MSR, %ecx + wrmsr ret @@ -148,8 +153,8 @@ PUBLIC(_CPU_Context_restore) .set NEWCONTEXT_ARG, REG_ARG0 /* context to restore argument */ SYM(_CPU_Context_restore): - movq NEWCONTEXT_ARG, rax /* rax = running threads context */ - GET_SELF_CPU_CONTROL_RCX /* rcx = per CPU information */ + movq NEWCONTEXT_ARG, r10 /* r10 = running threads context */ + GET_SELF_CPU_CONTROL_R11 /* r11 = per CPU information */ jmp .restore END_CODE diff --git a/cpukit/score/src/tlsallocsize.c b/cpukit/score/src/tlsallocsize.c index fa28391b83..3307f75da5 100644 --- a/cpukit/score/src/tlsallocsize.c +++ b/cpukit/score/src/tlsallocsize.c @@ -101,7 +101,7 @@ uintptr_t _TLS_Get_allocation_size( void ) stack_align = CPU_STACK_ALIGNMENT; tls_align = RTEMS_ALIGN_UP( (uintptr_t) config->alignment, stack_align ); -#ifndef __i386__ +#if !defined(__i386__) && !defined(__x86_64__) /* Reserve space for the dynamic thread vector */ allocation_size += RTEMS_ALIGN_UP( sizeof( TLS_Dynamic_thread_vector ), stack_align );