| From dd84441a797150dcc49298ec95c459a8891d8bb1 Mon Sep 17 00:00:00 2001 |
| From: David Woodhouse <dwmw@amazon.co.uk> |
| Date: Mon, 19 Feb 2018 10:50:54 +0000 |
| Subject: x86/speculation: Use IBRS if available before calling into firmware |
| |
| From: David Woodhouse <dwmw@amazon.co.uk> |
| |
| commit dd84441a797150dcc49298ec95c459a8891d8bb1 upstream. |
| |
| Retpoline means the kernel is safe because it has no indirect branches. |
| But firmware isn't, so use IBRS for firmware calls if it's available. |
| |
| Block preemption while IBRS is set, although in practice the call sites |
| already had to be doing that. |
| |
| Ignore hpwdt.c for now. It's taking spinlocks and calling into firmware |
| code, from an NMI handler. I don't want to touch that with a bargepole. |
| |
| Signed-off-by: David Woodhouse <dwmw@amazon.co.uk> |
| Reviewed-by: Thomas Gleixner <tglx@linutronix.de> |
| Cc: Linus Torvalds <torvalds@linux-foundation.org> |
| Cc: Peter Zijlstra <peterz@infradead.org> |
| Cc: arjan.van.de.ven@intel.com |
| Cc: bp@alien8.de |
| Cc: dave.hansen@intel.com |
| Cc: jmattson@google.com |
| Cc: karahmed@amazon.de |
| Cc: kvm@vger.kernel.org |
| Cc: pbonzini@redhat.com |
| Cc: rkrcmar@redhat.com |
| Link: http://lkml.kernel.org/r/1519037457-7643-2-git-send-email-dwmw@amazon.co.uk |
| Signed-off-by: Ingo Molnar <mingo@kernel.org> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/x86/include/asm/apm.h | 6 +++++ |
| arch/x86/include/asm/cpufeatures.h | 1 |
| arch/x86/include/asm/efi.h | 17 +++++++++++++-- |
| arch/x86/include/asm/nospec-branch.h | 39 ++++++++++++++++++++++++++--------- |
| arch/x86/kernel/cpu/bugs.c | 12 +++++++++- |
| 5 files changed, 63 insertions(+), 12 deletions(-) |
| |
| --- a/arch/x86/include/asm/apm.h |
| +++ b/arch/x86/include/asm/apm.h |
| @@ -7,6 +7,8 @@ |
| #ifndef _ASM_X86_MACH_DEFAULT_APM_H |
| #define _ASM_X86_MACH_DEFAULT_APM_H |
| |
| +#include <asm/nospec-branch.h> |
| + |
| #ifdef APM_ZERO_SEGS |
| # define APM_DO_ZERO_SEGS \ |
| "pushl %%ds\n\t" \ |
| @@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 |
| * N.B. We do NOT need a cld after the BIOS call |
| * because we always save and restore the flags. |
| */ |
| + firmware_restrict_branch_speculation_start(); |
| __asm__ __volatile__(APM_DO_ZERO_SEGS |
| "pushl %%edi\n\t" |
| "pushl %%ebp\n\t" |
| @@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 |
| "=S" (*esi) |
| : "a" (func), "b" (ebx_in), "c" (ecx_in) |
| : "memory", "cc"); |
| + firmware_restrict_branch_speculation_end(); |
| } |
| |
| static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in, |
| @@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_ |
| * N.B. We do NOT need a cld after the BIOS call |
| * because we always save and restore the flags. |
| */ |
| + firmware_restrict_branch_speculation_start(); |
| __asm__ __volatile__(APM_DO_ZERO_SEGS |
| "pushl %%edi\n\t" |
| "pushl %%ebp\n\t" |
| @@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_ |
| "=S" (si) |
| : "a" (func), "b" (ebx_in), "c" (ecx_in) |
| : "memory", "cc"); |
| + firmware_restrict_branch_speculation_end(); |
| return error; |
| } |
| |
| --- a/arch/x86/include/asm/cpufeatures.h |
| +++ b/arch/x86/include/asm/cpufeatures.h |
| @@ -211,6 +211,7 @@ |
| #define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* "" Fill RSB on context switches */ |
| |
| #define X86_FEATURE_USE_IBPB ( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */ |
| +#define X86_FEATURE_USE_IBRS_FW ( 7*32+22) /* "" Use IBRS during runtime firmware calls */ |
| |
| /* Virtualization flags: Linux defined, word 8 */ |
| #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ |
| --- a/arch/x86/include/asm/efi.h |
| +++ b/arch/x86/include/asm/efi.h |
| @@ -6,6 +6,7 @@ |
| #include <asm/pgtable.h> |
| #include <asm/processor-flags.h> |
| #include <asm/tlb.h> |
| +#include <asm/nospec-branch.h> |
| |
| /* |
| * We map the EFI regions needed for runtime services non-contiguously, |
| @@ -36,8 +37,18 @@ |
| |
| extern asmlinkage unsigned long efi_call_phys(void *, ...); |
| |
| -#define arch_efi_call_virt_setup() kernel_fpu_begin() |
| -#define arch_efi_call_virt_teardown() kernel_fpu_end() |
| +#define arch_efi_call_virt_setup() \ |
| +({ \ |
| + kernel_fpu_begin(); \ |
| + firmware_restrict_branch_speculation_start(); \ |
| +}) |
| + |
| +#define arch_efi_call_virt_teardown() \ |
| +({ \ |
| + firmware_restrict_branch_speculation_end(); \ |
| + kernel_fpu_end(); \ |
| +}) |
| + |
| |
| /* |
| * Wrap all the virtual calls in a way that forces the parameters on the stack. |
| @@ -73,6 +84,7 @@ struct efi_scratch { |
| efi_sync_low_kernel_mappings(); \ |
| preempt_disable(); \ |
| __kernel_fpu_begin(); \ |
| + firmware_restrict_branch_speculation_start(); \ |
| \ |
| if (efi_scratch.use_pgd) { \ |
| efi_scratch.prev_cr3 = __read_cr3(); \ |
| @@ -91,6 +103,7 @@ struct efi_scratch { |
| __flush_tlb_all(); \ |
| } \ |
| \ |
| + firmware_restrict_branch_speculation_end(); \ |
| __kernel_fpu_end(); \ |
| preempt_enable(); \ |
| }) |
| --- a/arch/x86/include/asm/nospec-branch.h |
| +++ b/arch/x86/include/asm/nospec-branch.h |
| @@ -219,17 +219,38 @@ static inline void vmexit_fill_RSB(void) |
| #endif |
| } |
| |
| +#define alternative_msr_write(_msr, _val, _feature) \ |
| + asm volatile(ALTERNATIVE("", \ |
| + "movl %[msr], %%ecx\n\t" \ |
| + "movl %[val], %%eax\n\t" \ |
| + "movl $0, %%edx\n\t" \ |
| + "wrmsr", \ |
| + _feature) \ |
| + : : [msr] "i" (_msr), [val] "i" (_val) \ |
| + : "eax", "ecx", "edx", "memory") |
| + |
| static inline void indirect_branch_prediction_barrier(void) |
| { |
| - asm volatile(ALTERNATIVE("", |
| - "movl %[msr], %%ecx\n\t" |
| - "movl %[val], %%eax\n\t" |
| - "movl $0, %%edx\n\t" |
| - "wrmsr", |
| - X86_FEATURE_USE_IBPB) |
| - : : [msr] "i" (MSR_IA32_PRED_CMD), |
| - [val] "i" (PRED_CMD_IBPB) |
| - : "eax", "ecx", "edx", "memory"); |
| + alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, |
| + X86_FEATURE_USE_IBPB); |
| +} |
| + |
| +/* |
| + * With retpoline, we must use IBRS to restrict branch prediction |
| + * before calling into firmware. |
| + */ |
| +static inline void firmware_restrict_branch_speculation_start(void) |
| +{ |
| + preempt_disable(); |
| + alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS, |
| + X86_FEATURE_USE_IBRS_FW); |
| +} |
| + |
| +static inline void firmware_restrict_branch_speculation_end(void) |
| +{ |
| + alternative_msr_write(MSR_IA32_SPEC_CTRL, 0, |
| + X86_FEATURE_USE_IBRS_FW); |
| + preempt_enable(); |
| } |
| |
| #endif /* __ASSEMBLY__ */ |
| --- a/arch/x86/kernel/cpu/bugs.c |
| +++ b/arch/x86/kernel/cpu/bugs.c |
| @@ -300,6 +300,15 @@ retpoline_auto: |
| setup_force_cpu_cap(X86_FEATURE_USE_IBPB); |
| pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n"); |
| } |
| + |
| + /* |
| + * Retpoline means the kernel is safe because it has no indirect |
| + * branches. But firmware isn't, so use IBRS to protect that. |
| + */ |
| + if (boot_cpu_has(X86_FEATURE_IBRS)) { |
| + setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); |
| + pr_info("Enabling Restricted Speculation for firmware calls\n"); |
| + } |
| } |
| |
| #undef pr_fmt |
| @@ -326,8 +335,9 @@ ssize_t cpu_show_spectre_v2(struct devic |
| if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) |
| return sprintf(buf, "Not affected\n"); |
| |
| - return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], |
| + return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], |
| boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "", |
| + boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", |
| spectre_v2_module_string()); |
| } |
| #endif |