| From f8e617f4582995f7c25ef25b4167213120ad122b Mon Sep 17 00:00:00 2001 |
| From: Mike Galbraith <bitbucket@online.de> |
| Date: Sat, 18 Jan 2014 17:14:44 +0100 |
| Subject: sched/idle/x86: Optimize unnecessary mwait_idle() resched IPIs |
| |
| From: Mike Galbraith <bitbucket@online.de> |
| |
| commit f8e617f4582995f7c25ef25b4167213120ad122b upstream. |
| |
| To fully take advantage of MWAIT, apparently the CLFLUSH instruction needs |
| another quirk on certain CPUs: proper barriers around it on certain machines. |
| |
| On a Q6600 SMP system, pipe-test scheduling performance, cross core, |
| improves significantly: |
| |
| 3.8.13 487.2 KHz 1.000 |
| 3.13.0-master 415.5 KHz .852 |
| 3.13.0-master+ 415.2 KHz .852 + restore mwait_idle |
| 3.13.0-master++ 488.5 KHz 1.002 + restore mwait_idle + IPI fix |
| |
| Since X86_BUG_CLFLUSH_MONITOR is already a quirk, don't create a separate |
| quirk for the extra smp_mb()s. |
| |
| Signed-off-by: Mike Galbraith <bitbucket@online.de> |
| Cc: Borislav Petkov <bp@alien8.de> |
| Cc: H. Peter Anvin <hpa@zytor.com> |
| Cc: Ian Malone <ibmalone@gmail.com> |
| Cc: Josh Boyer <jwboyer@redhat.com> |
| Cc: Len Brown <len.brown@intel.com> |
| Cc: Len Brown <lenb@kernel.org> |
| Cc: Linus Torvalds <torvalds@linux-foundation.org> |
| Cc: Mike Galbraith <efault@gmx.de> |
| Cc: Peter Zijlstra <peterz@infradead.org> |
| Cc: Thomas Gleixner <tglx@linutronix.de> |
| Link: http://lkml.kernel.org/r/1390061684.5566.4.camel@marge.simpson.net |
| [ Ported to recent kernel, added comments about the quirk. ] |
| Signed-off-by: Ingo Molnar <mingo@kernel.org> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/x86/kernel/process.c | 12 ++++++++---- |
| 1 file changed, 8 insertions(+), 4 deletions(-) |
| |
| --- a/arch/x86/kernel/process.c |
| +++ b/arch/x86/kernel/process.c |
| @@ -429,18 +429,22 @@ static int prefer_mwait_c1_over_halt(con |
| |
| static void mwait_idle(void) |
| { |
| - if (!need_resched()) { |
| - if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) |
| + if (!current_set_polling_and_test()) { |
| + if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) { |
| + smp_mb(); /* quirk */ |
| clflush((void *)¤t_thread_info()->flags); |
| + smp_mb(); /* quirk */ |
| + } |
| |
| __monitor((void *)¤t_thread_info()->flags, 0, 0); |
| - smp_mb(); |
| if (!need_resched()) |
| __sti_mwait(0, 0); |
| else |
| local_irq_enable(); |
| - } else |
| + } else { |
| local_irq_enable(); |
| + } |
| + __current_clr_polling(); |
| } |
| |
| void select_idle_routine(const struct cpuinfo_x86 *c) |