| From foo@baz Mon May 21 22:23:32 CEST 2018 |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Tue, 14 Feb 2017 00:11:04 -0800 |
| Subject: x86/process: Optimize TIF_NOTSC switch |
| |
| From: Thomas Gleixner <tglx@linutronix.de> |
| |
| commit 5a920155e388ec22a22e0532fb695b9215c9b34d upstream |
| |
| Provide and use a toggle helper instead of doing it with a branch. |
| |
| x86_64: arch/x86/kernel/process.o |
| text data bss dec hex |
| 3008 8577 16 11601 2d51 Before |
| 2976 8577 16 11569 2d31 After |
| |
| i386: arch/x86/kernel/process.o |
| text data bss dec hex |
| 2925 8673 8 11606 2d56 Before |
| 2893 8673 8 11574 2d36 After |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| Cc: Peter Zijlstra <peterz@infradead.org> |
| Cc: Andy Lutomirski <luto@kernel.org> |
| Link: http://lkml.kernel.org/r/20170214081104.9244-4-khuey@kylehuey.com |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| Signed-off-by: David Woodhouse <dwmw@amazon.co.uk> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| arch/x86/include/asm/tlbflush.h | 10 ++++++++++ |
| arch/x86/kernel/process.c | 22 ++++------------------ |
| 2 files changed, 14 insertions(+), 18 deletions(-) |
| |
| --- a/arch/x86/include/asm/tlbflush.h |
| +++ b/arch/x86/include/asm/tlbflush.h |
| @@ -111,6 +111,16 @@ static inline void cr4_clear_bits(unsign |
| } |
| } |
| |
| +static inline void cr4_toggle_bits(unsigned long mask) |
| +{ |
| + unsigned long cr4; |
| + |
| + cr4 = this_cpu_read(cpu_tlbstate.cr4); |
| + cr4 ^= mask; |
| + this_cpu_write(cpu_tlbstate.cr4, cr4); |
| + __write_cr4(cr4); |
| +} |
| + |
| /* Read the CR4 shadow. */ |
| static inline unsigned long cr4_read_shadow(void) |
| { |
| --- a/arch/x86/kernel/process.c |
| +++ b/arch/x86/kernel/process.c |
| @@ -134,11 +134,6 @@ void flush_thread(void) |
| fpu__clear(&tsk->thread.fpu); |
| } |
| |
| -static void hard_disable_TSC(void) |
| -{ |
| - cr4_set_bits(X86_CR4_TSD); |
| -} |
| - |
| void disable_TSC(void) |
| { |
| preempt_disable(); |
| @@ -147,15 +142,10 @@ void disable_TSC(void) |
| * Must flip the CPU state synchronously with |
| * TIF_NOTSC in the current running context. |
| */ |
| - hard_disable_TSC(); |
| + cr4_set_bits(X86_CR4_TSD); |
| preempt_enable(); |
| } |
| |
| -static void hard_enable_TSC(void) |
| -{ |
| - cr4_clear_bits(X86_CR4_TSD); |
| -} |
| - |
| static void enable_TSC(void) |
| { |
| preempt_disable(); |
| @@ -164,7 +154,7 @@ static void enable_TSC(void) |
| * Must flip the CPU state synchronously with |
| * TIF_NOTSC in the current running context. |
| */ |
| - hard_enable_TSC(); |
| + cr4_clear_bits(X86_CR4_TSD); |
| preempt_enable(); |
| } |
| |
| @@ -238,12 +228,8 @@ void __switch_to_xtra(struct task_struct |
| wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); |
| } |
| |
| - if ((tifp ^ tifn) & _TIF_NOTSC) { |
| - if (tifn & _TIF_NOTSC) |
| - hard_disable_TSC(); |
| - else |
| - hard_enable_TSC(); |
| - } |
| + if ((tifp ^ tifn) & _TIF_NOTSC) |
| + cr4_toggle_bits(X86_CR4_TSD); |
| } |
| |
| /* |