| From 76b043848fd22dbf7f8bf3a1452f8c70d557b860 Mon Sep 17 00:00:00 2001 |
| From: David Woodhouse <dwmw@amazon.co.uk> |
| Date: Thu, 11 Jan 2018 21:46:25 +0000 |
| Subject: x86/retpoline: Add initial retpoline support |
| |
| From: David Woodhouse <dwmw@amazon.co.uk> |
| |
| commit 76b043848fd22dbf7f8bf3a1452f8c70d557b860 upstream. |
| |
| Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide |
| the corresponding thunks. Provide assembler macros for invoking the thunks |
| in the same way that GCC does, from native and inline assembler. |
| |
| This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In |
| some circumstances, IBRS microcode features may be used instead, and the |
| retpoline can be disabled. |
| |
| On AMD CPUs if lfence is serialising, the retpoline can be dramatically |
| simplified to a simple "lfence; jmp *\reg". A future patch, after it has |
| been verified that lfence really is serialising in all circumstances, can |
| enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition |
| to X86_FEATURE_RETPOLINE. |
| |
| Do not align the retpoline in the altinstr section, because there is no |
| guarantee that it stays aligned when it's copied over the oldinstr during |
| alternative patching. |
| |
| [ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks] |
| [ tglx: Put actual function CALL/JMP in front of the macros, convert to |
| symbolic labels ] |
| [ dwmw2: Convert back to numeric labels, merge objtool fixes ] |
| |
| Signed-off-by: David Woodhouse <dwmw@amazon.co.uk> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| Acked-by: Arjan van de Ven <arjan@linux.intel.com> |
| Acked-by: Ingo Molnar <mingo@kernel.org> |
| Cc: gnomes@lxorguk.ukuu.org.uk |
| Cc: Rik van Riel <riel@redhat.com> |
| Cc: Andi Kleen <ak@linux.intel.com> |
| Cc: Josh Poimboeuf <jpoimboe@redhat.com> |
| Cc: thomas.lendacky@amd.com |
| Cc: Peter Zijlstra <peterz@infradead.org> |
| Cc: Linus Torvalds <torvalds@linux-foundation.org> |
| Cc: Jiri Kosina <jikos@kernel.org> |
| Cc: Andy Lutomirski <luto@amacapital.net> |
| Cc: Dave Hansen <dave.hansen@intel.com> |
| Cc: Kees Cook <keescook@google.com> |
| Cc: Tim Chen <tim.c.chen@linux.intel.com> |
| Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org> |
| Cc: Paul Turner <pjt@google.com> |
| Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk |
| Signed-off-by: David Woodhouse <dwmw@amazon.co.uk> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| arch/x86/Kconfig | 13 +++ |
| arch/x86/Makefile | 10 ++ |
| arch/x86/include/asm/asm-prototypes.h | 25 ++++++ |
| arch/x86/include/asm/cpufeatures.h | 3 |
| arch/x86/include/asm/nospec-branch.h | 128 ++++++++++++++++++++++++++++++++++ |
| arch/x86/kernel/cpu/common.c | 4 + |
| arch/x86/lib/Makefile | 1 |
| arch/x86/lib/retpoline.S | 48 ++++++++++++ |
| 8 files changed, 232 insertions(+) |
| create mode 100644 arch/x86/include/asm/nospec-branch.h |
| create mode 100644 arch/x86/lib/retpoline.S |
| |
| --- a/arch/x86/Kconfig |
| +++ b/arch/x86/Kconfig |
| @@ -408,6 +408,19 @@ config GOLDFISH |
| def_bool y |
| depends on X86_GOLDFISH |
| |
| +config RETPOLINE |
| + bool "Avoid speculative indirect branches in kernel" |
| + default y |
| + ---help--- |
| + Compile kernel with the retpoline compiler options to guard against |
| + kernel-to-user data leaks by avoiding speculative indirect |
| + branches. Requires a compiler with -mindirect-branch=thunk-extern |
| + support for full protection. The kernel may run slower. |
| + |
| + Without compiler support, at least indirect branches in assembler |
| + code are eliminated. Since this includes the syscall entry path, |
| + it is not entirely pointless. |
| + |
| if X86_32 |
| config X86_EXTENDED_PLATFORM |
| bool "Support for extended (non-PC) x86 platforms" |
| --- a/arch/x86/Makefile |
| +++ b/arch/x86/Makefile |
| @@ -182,6 +182,16 @@ KBUILD_CFLAGS += -fno-asynchronous-unwin |
| KBUILD_CFLAGS += $(mflags-y) |
| KBUILD_AFLAGS += $(mflags-y) |
| |
| +# Avoid indirect branches in kernel to deal with Spectre |
| +ifdef CONFIG_RETPOLINE |
| + RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register) |
| + ifneq ($(RETPOLINE_CFLAGS),) |
| + KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE |
| + else |
| + $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.) |
| + endif |
| +endif |
| + |
| archscripts: scripts_basic |
| $(Q)$(MAKE) $(build)=arch/x86/tools relocs |
| |
| --- a/arch/x86/include/asm/asm-prototypes.h |
| +++ b/arch/x86/include/asm/asm-prototypes.h |
| @@ -10,7 +10,32 @@ |
| #include <asm/pgtable.h> |
| #include <asm/special_insns.h> |
| #include <asm/preempt.h> |
| +#include <asm/asm.h> |
| |
| #ifndef CONFIG_X86_CMPXCHG64 |
| extern void cmpxchg8b_emu(void); |
| #endif |
| + |
| +#ifdef CONFIG_RETPOLINE |
| +#ifdef CONFIG_X86_32 |
| +#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void); |
| +#else |
| +#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void); |
| +INDIRECT_THUNK(8) |
| +INDIRECT_THUNK(9) |
| +INDIRECT_THUNK(10) |
| +INDIRECT_THUNK(11) |
| +INDIRECT_THUNK(12) |
| +INDIRECT_THUNK(13) |
| +INDIRECT_THUNK(14) |
| +INDIRECT_THUNK(15) |
| +#endif |
| +INDIRECT_THUNK(ax) |
| +INDIRECT_THUNK(bx) |
| +INDIRECT_THUNK(cx) |
| +INDIRECT_THUNK(dx) |
| +INDIRECT_THUNK(si) |
| +INDIRECT_THUNK(di) |
| +INDIRECT_THUNK(bp) |
| +INDIRECT_THUNK(sp) |
| +#endif /* CONFIG_RETPOLINE */ |
| --- a/arch/x86/include/asm/cpufeatures.h |
| +++ b/arch/x86/include/asm/cpufeatures.h |
| @@ -194,6 +194,9 @@ |
| #define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ |
| #define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ |
| |
| +#define X86_FEATURE_RETPOLINE ( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */ |
| +#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */ |
| + |
| #define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */ |
| #define X86_FEATURE_AVX512_4VNNIW (7*32+16) /* AVX-512 Neural Network Instructions */ |
| #define X86_FEATURE_AVX512_4FMAPS (7*32+17) /* AVX-512 Multiply Accumulation Single precision */ |
| --- /dev/null |
| +++ b/arch/x86/include/asm/nospec-branch.h |
| @@ -0,0 +1,128 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| + |
| +#ifndef __NOSPEC_BRANCH_H__ |
| +#define __NOSPEC_BRANCH_H__ |
| + |
| +#include <asm/alternative.h> |
| +#include <asm/alternative-asm.h> |
| +#include <asm/cpufeatures.h> |
| + |
| +#ifdef __ASSEMBLY__ |
| + |
| +/* |
| + * This should be used immediately before a retpoline alternative. It tells |
| + * objtool where the retpolines are so that it can make sense of the control |
| + * flow by just reading the original instruction(s) and ignoring the |
| + * alternatives. |
| + */ |
| +.macro ANNOTATE_NOSPEC_ALTERNATIVE |
| + .Lannotate_\@: |
| + .pushsection .discard.nospec |
| + .long .Lannotate_\@ - . |
| + .popsection |
| +.endm |
| + |
| +/* |
| + * These are the bare retpoline primitives for indirect jmp and call. |
| + * Do not use these directly; they only exist to make the ALTERNATIVE |
| + * invocation below less ugly. |
| + */ |
| +.macro RETPOLINE_JMP reg:req |
| + call .Ldo_rop_\@ |
| +.Lspec_trap_\@: |
| + pause |
| + jmp .Lspec_trap_\@ |
| +.Ldo_rop_\@: |
| + mov \reg, (%_ASM_SP) |
| + ret |
| +.endm |
| + |
| +/* |
| + * This is a wrapper around RETPOLINE_JMP so the called function in reg |
| + * returns to the instruction after the macro. |
| + */ |
| +.macro RETPOLINE_CALL reg:req |
| + jmp .Ldo_call_\@ |
| +.Ldo_retpoline_jmp_\@: |
| + RETPOLINE_JMP \reg |
| +.Ldo_call_\@: |
| + call .Ldo_retpoline_jmp_\@ |
| +.endm |
| + |
| +/* |
| + * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple |
| + * indirect jmp/call which may be susceptible to the Spectre variant 2 |
| + * attack. |
| + */ |
| +.macro JMP_NOSPEC reg:req |
| +#ifdef CONFIG_RETPOLINE |
| + ANNOTATE_NOSPEC_ALTERNATIVE |
| + ALTERNATIVE_2 __stringify(jmp *\reg), \ |
| + __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
| + __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
| +#else |
| + jmp *\reg |
| +#endif |
| +.endm |
| + |
| +.macro CALL_NOSPEC reg:req |
| +#ifdef CONFIG_RETPOLINE |
| + ANNOTATE_NOSPEC_ALTERNATIVE |
| + ALTERNATIVE_2 __stringify(call *\reg), \ |
| + __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
| + __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD |
| +#else |
| + call *\reg |
| +#endif |
| +.endm |
| + |
| +#else /* __ASSEMBLY__ */ |
| + |
| +#define ANNOTATE_NOSPEC_ALTERNATIVE \ |
| + "999:\n\t" \ |
| + ".pushsection .discard.nospec\n\t" \ |
| + ".long 999b - .\n\t" \ |
| + ".popsection\n\t" |
| + |
| +#if defined(CONFIG_X86_64) && defined(RETPOLINE) |
| + |
| +/* |
| + * Since the inline asm uses the %V modifier which is only in newer GCC, |
| + * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. |
| + */ |
| +# define CALL_NOSPEC \ |
| + ANNOTATE_NOSPEC_ALTERNATIVE \ |
| + ALTERNATIVE( \ |
| + "call *%[thunk_target]\n", \ |
| + "call __x86_indirect_thunk_%V[thunk_target]\n", \ |
| + X86_FEATURE_RETPOLINE) |
| +# define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
| + |
| +#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) |
| +/* |
| + * For i386 we use the original ret-equivalent retpoline, because |
| + * otherwise we'll run out of registers. We don't care about CET |
| + * here, anyway. |
| + */ |
| +# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ |
| + " jmp 904f;\n" \ |
| + " .align 16\n" \ |
| + "901: call 903f;\n" \ |
| + "902: pause;\n" \ |
| + " jmp 902b;\n" \ |
| + " .align 16\n" \ |
| + "903: addl $4, %%esp;\n" \ |
| + " pushl %[thunk_target];\n" \ |
| + " ret;\n" \ |
| + " .align 16\n" \ |
| + "904: call 901b;\n", \ |
| + X86_FEATURE_RETPOLINE) |
| + |
| +# define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
| +#else /* No retpoline */ |
| +# define CALL_NOSPEC "call *%[thunk_target]\n" |
| +# define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
| +#endif |
| + |
| +#endif /* __ASSEMBLY__ */ |
| +#endif /* __NOSPEC_BRANCH_H__ */ |
| --- a/arch/x86/kernel/cpu/common.c |
| +++ b/arch/x86/kernel/cpu/common.c |
| @@ -889,6 +889,10 @@ static void __init early_identify_cpu(st |
| setup_force_cpu_bug(X86_BUG_SPECTRE_V1); |
| setup_force_cpu_bug(X86_BUG_SPECTRE_V2); |
| |
| +#ifdef CONFIG_RETPOLINE |
| + setup_force_cpu_cap(X86_FEATURE_RETPOLINE); |
| +#endif |
| + |
| fpu__init_system(c); |
| |
| #ifdef CONFIG_X86_32 |
| --- a/arch/x86/lib/Makefile |
| +++ b/arch/x86/lib/Makefile |
| @@ -25,6 +25,7 @@ lib-y += memcpy_$(BITS).o |
| lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o |
| lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o |
| lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o |
| +lib-$(CONFIG_RETPOLINE) += retpoline.o |
| |
| obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o |
| |
| --- /dev/null |
| +++ b/arch/x86/lib/retpoline.S |
| @@ -0,0 +1,48 @@ |
| +/* SPDX-License-Identifier: GPL-2.0 */ |
| + |
| +#include <linux/stringify.h> |
| +#include <linux/linkage.h> |
| +#include <asm/dwarf2.h> |
| +#include <asm/cpufeatures.h> |
| +#include <asm/alternative-asm.h> |
| +#include <asm/export.h> |
| +#include <asm/nospec-branch.h> |
| + |
| +.macro THUNK reg |
| + .section .text.__x86.indirect_thunk.\reg |
| + |
| +ENTRY(__x86_indirect_thunk_\reg) |
| + CFI_STARTPROC |
| + JMP_NOSPEC %\reg |
| + CFI_ENDPROC |
| +ENDPROC(__x86_indirect_thunk_\reg) |
| +.endm |
| + |
| +/* |
| + * Despite being an assembler file we can't just use .irp here |
| + * because __KSYM_DEPS__ only uses the C preprocessor and would |
| + * only see one instance of "__x86_indirect_thunk_\reg" rather |
| + * than one per register with the correct names. So we do it |
| + * the simple and nasty way... |
| + */ |
| +#define EXPORT_THUNK(reg) EXPORT_SYMBOL(__x86_indirect_thunk_ ## reg) |
| +#define GENERATE_THUNK(reg) THUNK reg ; EXPORT_THUNK(reg) |
| + |
| +GENERATE_THUNK(_ASM_AX) |
| +GENERATE_THUNK(_ASM_BX) |
| +GENERATE_THUNK(_ASM_CX) |
| +GENERATE_THUNK(_ASM_DX) |
| +GENERATE_THUNK(_ASM_SI) |
| +GENERATE_THUNK(_ASM_DI) |
| +GENERATE_THUNK(_ASM_BP) |
| +GENERATE_THUNK(_ASM_SP) |
| +#ifdef CONFIG_64BIT |
| +GENERATE_THUNK(r8) |
| +GENERATE_THUNK(r9) |
| +GENERATE_THUNK(r10) |
| +GENERATE_THUNK(r11) |
| +GENERATE_THUNK(r12) |
| +GENERATE_THUNK(r13) |
| +GENERATE_THUNK(r14) |
| +GENERATE_THUNK(r15) |
| +#endif |
