| From foo@baz Tue Mar 8 08:40:58 PM CET 2022 |
| From: "Russell King (Oracle)" <rmk+kernel@armlinux.org.uk> |
| Date: Thu, 10 Feb 2022 16:05:45 +0000 |
| Subject: ARM: Spectre-BHB workaround |
| |
| From: "Russell King (Oracle)" <rmk+kernel@armlinux.org.uk> |
| |
| commit b9baf5c8c5c356757f4f9d8180b5e9d234065bc3 upstream. |
| |
| Workaround the Spectre BHB issues for Cortex-A15, Cortex-A57, |
| Cortex-A72, Cortex-A73 and Cortex-A75. We also include Brahma B15 as |
| well to be safe, which is affected by Spectre V2 in the same ways as |
| Cortex-A15. |
| |
| Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> |
| Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk> |
| [changes due to lack of SYSTEM_FREEING_INITMEM - gregkh] |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| arch/arm/include/asm/assembler.h | 10 ++++ |
| arch/arm/include/asm/spectre.h | 4 + |
| arch/arm/kernel/entry-armv.S | 79 +++++++++++++++++++++++++++++++++++--- |
| arch/arm/kernel/entry-common.S | 24 +++++++++++ |
| arch/arm/kernel/spectre.c | 4 + |
| arch/arm/kernel/traps.c | 38 ++++++++++++++++++ |
| arch/arm/kernel/vmlinux-xip.lds.S | 18 +++++++- |
| arch/arm/kernel/vmlinux.lds.S | 18 +++++++- |
| arch/arm/mm/Kconfig | 10 ++++ |
| arch/arm/mm/proc-v7-bugs.c | 76 ++++++++++++++++++++++++++++++++++++ |
| 10 files changed, 269 insertions(+), 12 deletions(-) |
| |
| --- a/arch/arm/include/asm/assembler.h |
| +++ b/arch/arm/include/asm/assembler.h |
| @@ -108,6 +108,16 @@ |
| .endm |
| #endif |
| |
| +#if __LINUX_ARM_ARCH__ < 7 |
| + .macro dsb, args |
| + mcr p15, 0, r0, c7, c10, 4 |
| + .endm |
| + |
| + .macro isb, args |
| + mcr p15, 0, r0, c7, r5, 4 |
| + .endm |
| +#endif |
| + |
| .macro asm_trace_hardirqs_off, save=1 |
| #if defined(CONFIG_TRACE_IRQFLAGS) |
| .if \save |
| --- a/arch/arm/include/asm/spectre.h |
| +++ b/arch/arm/include/asm/spectre.h |
| @@ -14,6 +14,7 @@ enum { |
| __SPECTRE_V2_METHOD_ICIALLU, |
| __SPECTRE_V2_METHOD_SMC, |
| __SPECTRE_V2_METHOD_HVC, |
| + __SPECTRE_V2_METHOD_LOOP8, |
| }; |
| |
| enum { |
| @@ -21,8 +22,11 @@ enum { |
| SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU), |
| SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC), |
| SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC), |
| + SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8), |
| }; |
| |
| void spectre_v2_update_state(unsigned int state, unsigned int methods); |
| |
| +int spectre_bhb_update_vectors(unsigned int method); |
| + |
| #endif |
| --- a/arch/arm/kernel/entry-armv.S |
| +++ b/arch/arm/kernel/entry-armv.S |
| @@ -1036,12 +1036,11 @@ vector_\name: |
| sub lr, lr, #\correction |
| .endif |
| |
| - @ |
| - @ Save r0, lr_<exception> (parent PC) and spsr_<exception> |
| - @ (parent CPSR) |
| - @ |
| + @ Save r0, lr_<exception> (parent PC) |
| stmia sp, {r0, lr} @ save r0, lr |
| - mrs lr, spsr |
| + |
| + @ Save spsr_<exception> (parent CPSR) |
| +2: mrs lr, spsr |
| str lr, [sp, #8] @ save spsr |
| |
| @ |
| @@ -1062,6 +1061,44 @@ vector_\name: |
| movs pc, lr @ branch to handler in SVC mode |
| ENDPROC(vector_\name) |
| |
| +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
| + .subsection 1 |
| + .align 5 |
| +vector_bhb_loop8_\name: |
| + .if \correction |
| + sub lr, lr, #\correction |
| + .endif |
| + |
| + @ Save r0, lr_<exception> (parent PC) |
| + stmia sp, {r0, lr} |
| + |
| + @ bhb workaround |
| + mov r0, #8 |
| +1: b . + 4 |
| + subs r0, r0, #1 |
| + bne 1b |
| + dsb |
| + isb |
| + b 2b |
| +ENDPROC(vector_bhb_loop8_\name) |
| + |
| +vector_bhb_bpiall_\name: |
| + .if \correction |
| + sub lr, lr, #\correction |
| + .endif |
| + |
| + @ Save r0, lr_<exception> (parent PC) |
| + stmia sp, {r0, lr} |
| + |
| + @ bhb workaround |
| + mcr p15, 0, r0, c7, c5, 6 @ BPIALL |
| + @ isb not needed due to "movs pc, lr" in the vector stub |
| + @ which gives a "context synchronisation". |
| + b 2b |
| +ENDPROC(vector_bhb_bpiall_\name) |
| + .previous |
| +#endif |
| + |
| .align 2 |
| @ handler addresses follow this label |
| 1: |
| @@ -1070,6 +1107,10 @@ ENDPROC(vector_\name) |
| .section .stubs, "ax", %progbits |
| @ This must be the first word |
| .word vector_swi |
| +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
| + .word vector_bhb_loop8_swi |
| + .word vector_bhb_bpiall_swi |
| +#endif |
| |
| vector_rst: |
| ARM( swi SYS_ERROR0 ) |
| @@ -1184,8 +1225,10 @@ vector_addrexcptn: |
| * FIQ "NMI" handler |
| *----------------------------------------------------------------------------- |
| * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86 |
| - * systems. |
| + * systems. This must be the last vector stub, so lets place it in its own |
| + * subsection. |
| */ |
| + .subsection 2 |
| vector_stub fiq, FIQ_MODE, 4 |
| |
| .long __fiq_usr @ 0 (USR_26 / USR_32) |
| @@ -1218,6 +1261,30 @@ vector_addrexcptn: |
| W(b) vector_irq |
| W(b) vector_fiq |
| |
| +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
| + .section .vectors.bhb.loop8, "ax", %progbits |
| +.L__vectors_bhb_loop8_start: |
| + W(b) vector_rst |
| + W(b) vector_bhb_loop8_und |
| + W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004 |
| + W(b) vector_bhb_loop8_pabt |
| + W(b) vector_bhb_loop8_dabt |
| + W(b) vector_addrexcptn |
| + W(b) vector_bhb_loop8_irq |
| + W(b) vector_bhb_loop8_fiq |
| + |
| + .section .vectors.bhb.bpiall, "ax", %progbits |
| +.L__vectors_bhb_bpiall_start: |
| + W(b) vector_rst |
| + W(b) vector_bhb_bpiall_und |
| + W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008 |
| + W(b) vector_bhb_bpiall_pabt |
| + W(b) vector_bhb_bpiall_dabt |
| + W(b) vector_addrexcptn |
| + W(b) vector_bhb_bpiall_irq |
| + W(b) vector_bhb_bpiall_fiq |
| +#endif |
| + |
| .data |
| |
| .globl cr_alignment |
| --- a/arch/arm/kernel/entry-common.S |
| +++ b/arch/arm/kernel/entry-common.S |
| @@ -143,12 +143,36 @@ ENDPROC(ret_from_fork) |
| */ |
| |
| .align 5 |
| +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
| +ENTRY(vector_bhb_loop8_swi) |
| + sub sp, sp, #PT_REGS_SIZE |
| + stmia sp, {r0 - r12} |
| + mov r8, #8 |
| +1: b 2f |
| +2: subs r8, r8, #1 |
| + bne 1b |
| + dsb |
| + isb |
| + b 3f |
| +ENDPROC(vector_bhb_loop8_swi) |
| + |
| + .align 5 |
| +ENTRY(vector_bhb_bpiall_swi) |
| + sub sp, sp, #PT_REGS_SIZE |
| + stmia sp, {r0 - r12} |
| + mcr p15, 0, r8, c7, c5, 6 @ BPIALL |
| + isb |
| + b 3f |
| +ENDPROC(vector_bhb_bpiall_swi) |
| +#endif |
| + .align 5 |
| ENTRY(vector_swi) |
| #ifdef CONFIG_CPU_V7M |
| v7m_exception_entry |
| #else |
| sub sp, sp, #PT_REGS_SIZE |
| stmia sp, {r0 - r12} @ Calling r0 - r12 |
| +3: |
| ARM( add r8, sp, #S_PC ) |
| ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr |
| THUMB( mov r8, sp ) |
| --- a/arch/arm/kernel/spectre.c |
| +++ b/arch/arm/kernel/spectre.c |
| @@ -45,6 +45,10 @@ ssize_t cpu_show_spectre_v2(struct devic |
| method = "Firmware call"; |
| break; |
| |
| + case SPECTRE_V2_METHOD_LOOP8: |
| + method = "History overwrite"; |
| + break; |
| + |
| default: |
| method = "Multiple mitigations"; |
| break; |
| --- a/arch/arm/kernel/traps.c |
| +++ b/arch/arm/kernel/traps.c |
| @@ -31,6 +31,7 @@ |
| #include <linux/atomic.h> |
| #include <asm/cacheflush.h> |
| #include <asm/exception.h> |
| +#include <asm/spectre.h> |
| #include <asm/unistd.h> |
| #include <asm/traps.h> |
| #include <asm/ptrace.h> |
| @@ -833,6 +834,43 @@ static void flush_vectors(void *vma, siz |
| flush_icache_range(start, end); |
| } |
| |
| +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
| +int spectre_bhb_update_vectors(unsigned int method) |
| +{ |
| + extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[]; |
| + extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[]; |
| + void *vec_start, *vec_end; |
| + |
| + if (system_state >= SYSTEM_RUNNING) { |
| + pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n", |
| + smp_processor_id()); |
| + return SPECTRE_VULNERABLE; |
| + } |
| + |
| + switch (method) { |
| + case SPECTRE_V2_METHOD_LOOP8: |
| + vec_start = __vectors_bhb_loop8_start; |
| + vec_end = __vectors_bhb_loop8_end; |
| + break; |
| + |
| + case SPECTRE_V2_METHOD_BPIALL: |
| + vec_start = __vectors_bhb_bpiall_start; |
| + vec_end = __vectors_bhb_bpiall_end; |
| + break; |
| + |
| + default: |
| + pr_err("CPU%u: unknown Spectre BHB state %d\n", |
| + smp_processor_id(), method); |
| + return SPECTRE_VULNERABLE; |
| + } |
| + |
| + copy_from_lma(vectors_page, vec_start, vec_end); |
| + flush_vectors(vectors_page, 0, vec_end - vec_start); |
| + |
| + return SPECTRE_MITIGATED; |
| +} |
| +#endif |
| + |
| void __init early_trap_init(void *vectors_base) |
| { |
| extern char __stubs_start[], __stubs_end[]; |
| --- a/arch/arm/kernel/vmlinux-xip.lds.S |
| +++ b/arch/arm/kernel/vmlinux-xip.lds.S |
| @@ -154,11 +154,23 @@ SECTIONS |
| * only thing that matters is their relative offsets |
| */ |
| __vectors_lma = .; |
| - .vectors 0xffff0000 : AT(__vectors_start) { |
| - *(.vectors) |
| + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { |
| + .vectors { |
| + *(.vectors) |
| + } |
| + .vectors.bhb.loop8 { |
| + *(.vectors.bhb.loop8) |
| + } |
| + .vectors.bhb.bpiall { |
| + *(.vectors.bhb.bpiall) |
| + } |
| } |
| ARM_LMA(__vectors, .vectors); |
| - . = __vectors_lma + SIZEOF(.vectors); |
| + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); |
| + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); |
| + . = __vectors_lma + SIZEOF(.vectors) + |
| + SIZEOF(.vectors.bhb.loop8) + |
| + SIZEOF(.vectors.bhb.bpiall); |
| |
| __stubs_lma = .; |
| .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { |
| --- a/arch/arm/kernel/vmlinux.lds.S |
| +++ b/arch/arm/kernel/vmlinux.lds.S |
| @@ -175,11 +175,23 @@ SECTIONS |
| * only thing that matters is their relative offsets |
| */ |
| __vectors_lma = .; |
| - .vectors 0xffff0000 : AT(__vectors_start) { |
| - *(.vectors) |
| + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { |
| + .vectors { |
| + *(.vectors) |
| + } |
| + .vectors.bhb.loop8 { |
| + *(.vectors.bhb.loop8) |
| + } |
| + .vectors.bhb.bpiall { |
| + *(.vectors.bhb.bpiall) |
| + } |
| } |
| ARM_LMA(__vectors, .vectors); |
| - . = __vectors_lma + SIZEOF(.vectors); |
| + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); |
| + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); |
| + . = __vectors_lma + SIZEOF(.vectors) + |
| + SIZEOF(.vectors.bhb.loop8) + |
| + SIZEOF(.vectors.bhb.bpiall); |
| |
| __stubs_lma = .; |
| .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { |
| --- a/arch/arm/mm/Kconfig |
| +++ b/arch/arm/mm/Kconfig |
| @@ -824,6 +824,16 @@ config HARDEN_BRANCH_PREDICTOR |
| |
| If unsure, say Y. |
| |
| +config HARDEN_BRANCH_HISTORY |
| + bool "Harden Spectre style attacks against branch history" if EXPERT |
| + depends on CPU_SPECTRE |
| + default y |
| + help |
| + Speculation attacks against some high-performance processors can |
| + make use of branch history to influence future speculation. When |
| + taking an exception, a sequence of branches overwrites the branch |
| + history, or branch history is invalidated. |
| + |
| config TLS_REG_EMUL |
| bool |
| select NEED_KUSER_HELPERS |
| --- a/arch/arm/mm/proc-v7-bugs.c |
| +++ b/arch/arm/mm/proc-v7-bugs.c |
| @@ -186,6 +186,81 @@ static void cpu_v7_spectre_v2_init(void) |
| spectre_v2_update_state(state, method); |
| } |
| |
| +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
| +static int spectre_bhb_method; |
| + |
| +static const char *spectre_bhb_method_name(int method) |
| +{ |
| + switch (method) { |
| + case SPECTRE_V2_METHOD_LOOP8: |
| + return "loop"; |
| + |
| + case SPECTRE_V2_METHOD_BPIALL: |
| + return "BPIALL"; |
| + |
| + default: |
| + return "unknown"; |
| + } |
| +} |
| + |
| +static int spectre_bhb_install_workaround(int method) |
| +{ |
| + if (spectre_bhb_method != method) { |
| + if (spectre_bhb_method) { |
| + pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", |
| + smp_processor_id()); |
| + |
| + return SPECTRE_VULNERABLE; |
| + } |
| + |
| + if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) |
| + return SPECTRE_VULNERABLE; |
| + |
| + spectre_bhb_method = method; |
| + } |
| + |
| + pr_info("CPU%u: Spectre BHB: using %s workaround\n", |
| + smp_processor_id(), spectre_bhb_method_name(method)); |
| + |
| + return SPECTRE_MITIGATED; |
| +} |
| +#else |
| +static int spectre_bhb_install_workaround(int method) |
| +{ |
| + return SPECTRE_VULNERABLE; |
| +} |
| +#endif |
| + |
| +static void cpu_v7_spectre_bhb_init(void) |
| +{ |
| + unsigned int state, method = 0; |
| + |
| + switch (read_cpuid_part()) { |
| + case ARM_CPU_PART_CORTEX_A15: |
| + case ARM_CPU_PART_BRAHMA_B15: |
| + case ARM_CPU_PART_CORTEX_A57: |
| + case ARM_CPU_PART_CORTEX_A72: |
| + state = SPECTRE_MITIGATED; |
| + method = SPECTRE_V2_METHOD_LOOP8; |
| + break; |
| + |
| + case ARM_CPU_PART_CORTEX_A73: |
| + case ARM_CPU_PART_CORTEX_A75: |
| + state = SPECTRE_MITIGATED; |
| + method = SPECTRE_V2_METHOD_BPIALL; |
| + break; |
| + |
| + default: |
| + state = SPECTRE_UNAFFECTED; |
| + break; |
| + } |
| + |
| + if (state == SPECTRE_MITIGATED) |
| + state = spectre_bhb_install_workaround(method); |
| + |
| + spectre_v2_update_state(state, method); |
| +} |
| + |
| static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, |
| u32 mask, const char *msg) |
| { |
| @@ -226,4 +301,5 @@ void cpu_v7_ca15_ibe(void) |
| void cpu_v7_bugs_init(void) |
| { |
| cpu_v7_spectre_v2_init(); |
| + cpu_v7_spectre_bhb_init(); |
| } |