| From foo@baz Sat Mar 19 01:51:18 PM CET 2022 |
| From: James Morse <james.morse@arm.com> |
| Date: Fri, 18 Mar 2022 17:48:34 +0000 |
| Subject: arm64: entry: Add vectors that have the bhb mitigation sequences |
| To: stable@vger.kernel.org |
| Cc: linux-kernel@vger.kernel.org, james.morse@arm.com, catalin.marinas@arm.com |
| Message-ID: <20220318174842.2321061-15-james.morse@arm.com> |
| |
| From: James Morse <james.morse@arm.com> |
| |
| commit ba2689234be92024e5635d30fe744f4853ad97db upstream. |
| |
| Some CPUs affected by Spectre-BHB need a sequence of branches, or a |
| firmware call to be run before any indirect branch. This needs to go |
| in the vectors. No CPU needs both. |
| |
| While this can be patched in, it would run on all CPUs as there is a |
| single set of vectors. If only one part of a big/little combination is |
| affected, the unaffected CPUs have to run the mitigation too. |
| |
| Create extra vectors that include the sequence. Subsequent patches will |
| allow affected CPUs to select this set of vectors. Later patches will |
| modify the loop count to match what the CPU requires. |
| |
| Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> |
| Signed-off-by: James Morse <james.morse@arm.com> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| arch/arm64/include/asm/assembler.h | 25 +++++++++++++++++ |
| arch/arm64/include/asm/vectors.h | 34 +++++++++++++++++++++++ |
| arch/arm64/kernel/entry.S | 53 ++++++++++++++++++++++++++++++------- |
| include/linux/arm-smccc.h | 7 ++++ |
| 4 files changed, 110 insertions(+), 9 deletions(-) |
| create mode 100644 arch/arm64/include/asm/vectors.h |
| |
| --- a/arch/arm64/include/asm/assembler.h |
| +++ b/arch/arm64/include/asm/assembler.h |
| @@ -711,4 +711,29 @@ USER(\label, ic ivau, \tmp2) // invali |
| .Lyield_out_\@ : |
| .endm |
| |
| + .macro __mitigate_spectre_bhb_loop tmp |
| +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| + mov \tmp, #32 |
| +.Lspectre_bhb_loop\@: |
| + b . + 4 |
| + subs \tmp, \tmp, #1 |
| + b.ne .Lspectre_bhb_loop\@ |
| + dsb nsh |
| + isb |
| +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
| + .endm |
| + |
| + /* Save/restores x0-x3 to the stack */ |
| + .macro __mitigate_spectre_bhb_fw |
| +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| + stp x0, x1, [sp, #-16]! |
| + stp x2, x3, [sp, #-16]! |
| + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 |
| +alternative_cb arm64_update_smccc_conduit |
| + nop // Patched to SMC/HVC #0 |
| +alternative_cb_end |
| + ldp x2, x3, [sp], #16 |
| + ldp x0, x1, [sp], #16 |
| +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
| + .endm |
| #endif /* __ASM_ASSEMBLER_H */ |
| --- /dev/null |
| +++ b/arch/arm64/include/asm/vectors.h |
| @@ -0,0 +1,34 @@ |
| +/* SPDX-License-Identifier: GPL-2.0-only */ |
| +/* |
| + * Copyright (C) 2022 ARM Ltd. |
| + */ |
| +#ifndef __ASM_VECTORS_H |
| +#define __ASM_VECTORS_H |
| + |
| +/* |
| + * Note: the order of this enum corresponds to two arrays in entry.S: |
| + * tramp_vecs and __bp_harden_el1_vectors. By default the canonical |
| + * 'full fat' vectors are used directly. |
| + */ |
| +enum arm64_bp_harden_el1_vectors { |
| +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| + /* |
| + * Perform the BHB loop mitigation, before branching to the canonical |
| + * vectors. |
| + */ |
| + EL1_VECTOR_BHB_LOOP, |
| + |
| + /* |
| + * Make the SMC call for firmware mitigation, before branching to the |
| + * canonical vectors. |
| + */ |
| + EL1_VECTOR_BHB_FW, |
| +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
| + |
| + /* |
| + * Remap the kernel before branching to the canonical vectors. |
| + */ |
| + EL1_VECTOR_KPTI, |
| +}; |
| + |
| +#endif /* __ASM_VECTORS_H */ |
| --- a/arch/arm64/kernel/entry.S |
| +++ b/arch/arm64/kernel/entry.S |
| @@ -970,13 +970,26 @@ alternative_else_nop_endif |
| sub \dst, \dst, PAGE_SIZE |
| .endm |
| |
| - .macro tramp_ventry, vector_start, regsize, kpti |
| + |
| +#define BHB_MITIGATION_NONE 0 |
| +#define BHB_MITIGATION_LOOP 1 |
| +#define BHB_MITIGATION_FW 2 |
| + |
| + .macro tramp_ventry, vector_start, regsize, kpti, bhb |
| .align 7 |
| 1: |
| .if \regsize == 64 |
| msr tpidrro_el0, x30 // Restored in kernel_ventry |
| .endif |
| |
| + .if \bhb == BHB_MITIGATION_LOOP |
| + /* |
| + * This sequence must appear before the first indirect branch. i.e. the |
| + * ret out of tramp_ventry. It appears here because x30 is free. |
| + */ |
| + __mitigate_spectre_bhb_loop x30 |
| + .endif // \bhb == BHB_MITIGATION_LOOP |
| + |
| .if \kpti == 1 |
| /* |
| * Defend against branch aliasing attacks by pushing a dummy |
| @@ -1001,6 +1014,15 @@ alternative_insn isb, nop, ARM64_WORKARO |
| ldr x30, =vectors |
| .endif // \kpti == 1 |
| |
| + .if \bhb == BHB_MITIGATION_FW |
| + /* |
| + * The firmware sequence must appear before the first indirect branch. |
| + * i.e. the ret out of tramp_ventry. But it also needs the stack to be |
| + * mapped to save/restore the registers the SMC clobbers. |
| + */ |
| + __mitigate_spectre_bhb_fw |
| + .endif // \bhb == BHB_MITIGATION_FW |
| + |
| add x30, x30, #(1b - \vector_start + 4) |
| ret |
| .org 1b + 128 // Did we overflow the ventry slot? |
| @@ -1008,6 +1030,9 @@ alternative_insn isb, nop, ARM64_WORKARO |
| |
| .macro tramp_exit, regsize = 64 |
| adr x30, tramp_vectors |
| +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| + add x30, x30, SZ_4K |
| +#endif |
| msr vbar_el1, x30 |
| ldr lr, [sp, #S_LR] |
| tramp_unmap_kernel x29 |
| @@ -1018,26 +1043,32 @@ alternative_insn isb, nop, ARM64_WORKARO |
| eret |
| .endm |
| |
| - .macro generate_tramp_vector, kpti |
| + .macro generate_tramp_vector, kpti, bhb |
| .Lvector_start\@: |
| .space 0x400 |
| |
| .rept 4 |
| - tramp_ventry .Lvector_start\@, 64, \kpti |
| + tramp_ventry .Lvector_start\@, 64, \kpti, \bhb |
| .endr |
| .rept 4 |
| - tramp_ventry .Lvector_start\@, 32, \kpti |
| + tramp_ventry .Lvector_start\@, 32, \kpti, \bhb |
| .endr |
| .endm |
| |
| #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
| /* |
| * Exception vectors trampoline. |
| + * The order must match __bp_harden_el1_vectors and the |
| + * arm64_bp_harden_el1_vectors enum. |
| */ |
| .pushsection ".entry.tramp.text", "ax" |
| .align 11 |
| ENTRY(tramp_vectors) |
| - generate_tramp_vector kpti=1 |
| +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP |
| + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW |
| +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
| + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE |
| END(tramp_vectors) |
| |
| ENTRY(tramp_exit_native) |
| @@ -1064,7 +1095,7 @@ __entry_tramp_data_start: |
| * Exception vectors for spectre mitigations on entry from EL1 when |
| * kpti is not in use. |
| */ |
| - .macro generate_el1_vector |
| + .macro generate_el1_vector, bhb |
| .Lvector_start\@: |
| kernel_ventry 1, sync_invalid // Synchronous EL1t |
| kernel_ventry 1, irq_invalid // IRQ EL1t |
| @@ -1077,17 +1108,21 @@ __entry_tramp_data_start: |
| kernel_ventry 1, error // Error EL1h |
| |
| .rept 4 |
| - tramp_ventry .Lvector_start\@, 64, kpti=0 |
| + tramp_ventry .Lvector_start\@, 64, 0, \bhb |
| .endr |
| .rept 4 |
| - tramp_ventry .Lvector_start\@, 32, kpti=0 |
| + tramp_ventry .Lvector_start\@, 32, 0, \bhb |
| .endr |
| .endm |
| |
| +/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */ |
| .pushsection ".entry.text", "ax" |
| .align 11 |
| ENTRY(__bp_harden_el1_vectors) |
| - generate_el1_vector |
| +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| + generate_el1_vector bhb=BHB_MITIGATION_LOOP |
| + generate_el1_vector bhb=BHB_MITIGATION_FW |
| +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
| END(__bp_harden_el1_vectors) |
| .popsection |
| |
| --- a/include/linux/arm-smccc.h |
| +++ b/include/linux/arm-smccc.h |
| @@ -85,6 +85,13 @@ |
| ARM_SMCCC_SMC_32, \ |
| 0, 0x7fff) |
| |
| +#define ARM_SMCCC_ARCH_WORKAROUND_3 \ |
| + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ |
| + ARM_SMCCC_SMC_32, \ |
| + 0, 0x3fff) |
| + |
| +#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1 |
| + |
| #ifndef __ASSEMBLY__ |
| |
| #include <linux/linkage.h> |