| From foo@baz Wed Mar 9 04:10:24 PM CET 2022 |
| From: Peter Zijlstra <peterz@infradead.org> |
| Date: Wed, 16 Feb 2022 20:57:01 +0100 |
| Subject: x86/speculation: Add eIBRS + Retpoline options |
| |
| From: Peter Zijlstra <peterz@infradead.org> |
| |
| commit 1e19da8522c81bf46b335f84137165741e0d82b7 upstream. |
| |
| Thanks to the chaps at VUsec it is now clear that eIBRS is not |
| sufficient, therefore allow enabling of retpolines along with eIBRS. |
| |
| Add spectre_v2=eibrs, spectre_v2=eibrs,lfence and |
| spectre_v2=eibrs,retpoline options to explicitly pick your preferred |
| means of mitigation. |
| |
| Since there's new mitigations there's also user visible changes in |
| /sys/devices/system/cpu/vulnerabilities/spectre_v2 to reflect these |
| new mitigations. |
| |
| [ bp: Massage commit message, trim error messages, |
| do more precise eIBRS mode checking. ] |
| |
| Co-developed-by: Josh Poimboeuf <jpoimboe@redhat.com> |
| Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com> |
| Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> |
| Signed-off-by: Borislav Petkov <bp@suse.de> |
| Reviewed-by: Patrick Colp <patrick.colp@oracle.com> |
| Reviewed-by: Thomas Gleixner <tglx@linutronix.de> |
| [fllinden@amazon.com: backported to 4.19 (no Hygon)] |
| Signed-off-by: Frank van der Linden <fllinden@amazon.com> |
| Signed-off-by: Ben Hutchings <ben@decadent.org.uk> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| arch/x86/include/asm/nospec-branch.h | 4 - |
| arch/x86/kernel/cpu/bugs.c | 131 +++++++++++++++++++++++++---------- |
| 2 files changed, 98 insertions(+), 37 deletions(-) |
| |
| --- a/arch/x86/include/asm/nospec-branch.h |
| +++ b/arch/x86/include/asm/nospec-branch.h |
| @@ -225,7 +225,9 @@ enum spectre_v2_mitigation { |
| SPECTRE_V2_NONE, |
| SPECTRE_V2_RETPOLINE, |
| SPECTRE_V2_LFENCE, |
| - SPECTRE_V2_IBRS_ENHANCED, |
| + SPECTRE_V2_EIBRS, |
| + SPECTRE_V2_EIBRS_RETPOLINE, |
| + SPECTRE_V2_EIBRS_LFENCE, |
| }; |
| |
| /* The indirect branch speculation control variants */ |
| --- a/arch/x86/kernel/cpu/bugs.c |
| +++ b/arch/x86/kernel/cpu/bugs.c |
| @@ -621,6 +621,9 @@ enum spectre_v2_mitigation_cmd { |
| SPECTRE_V2_CMD_RETPOLINE, |
| SPECTRE_V2_CMD_RETPOLINE_GENERIC, |
| SPECTRE_V2_CMD_RETPOLINE_LFENCE, |
| + SPECTRE_V2_CMD_EIBRS, |
| + SPECTRE_V2_CMD_EIBRS_RETPOLINE, |
| + SPECTRE_V2_CMD_EIBRS_LFENCE, |
| }; |
| |
| enum spectre_v2_user_cmd { |
| @@ -693,6 +696,13 @@ spectre_v2_parse_user_cmdline(enum spect |
| return SPECTRE_V2_USER_CMD_AUTO; |
| } |
| |
| +static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode) |
| +{ |
| + return (mode == SPECTRE_V2_EIBRS || |
| + mode == SPECTRE_V2_EIBRS_RETPOLINE || |
| + mode == SPECTRE_V2_EIBRS_LFENCE); |
| +} |
| + |
| static void __init |
| spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) |
| { |
| @@ -760,7 +770,7 @@ spectre_v2_user_select_mitigation(enum s |
| */ |
| if (!boot_cpu_has(X86_FEATURE_STIBP) || |
| !smt_possible || |
| - spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) |
| + spectre_v2_in_eibrs_mode(spectre_v2_enabled)) |
| return; |
| |
| /* |
| @@ -782,7 +792,9 @@ static const char * const spectre_v2_str |
| [SPECTRE_V2_NONE] = "Vulnerable", |
| [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", |
| [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", |
| - [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", |
| + [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", |
| + [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", |
| + [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", |
| }; |
| |
| static const struct { |
| @@ -796,6 +808,9 @@ static const struct { |
| { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, |
| { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, |
| { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, |
| + { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, |
| + { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, |
| + { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, |
| { "auto", SPECTRE_V2_CMD_AUTO, false }, |
| }; |
| |
| @@ -833,15 +848,29 @@ static enum spectre_v2_mitigation_cmd __ |
| |
| if ((cmd == SPECTRE_V2_CMD_RETPOLINE || |
| cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || |
| - cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && |
| + cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || |
| + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || |
| + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && |
| !IS_ENABLED(CONFIG_RETPOLINE)) { |
| - pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); |
| + pr_err("%s selected but not compiled in. Switching to AUTO select\n", |
| + mitigation_options[i].option); |
| + return SPECTRE_V2_CMD_AUTO; |
| + } |
| + |
| + if ((cmd == SPECTRE_V2_CMD_EIBRS || |
| + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || |
| + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && |
| + !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { |
| + pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", |
| + mitigation_options[i].option); |
| return SPECTRE_V2_CMD_AUTO; |
| } |
| |
| - if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE) && |
| + if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || |
| + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && |
| !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { |
| - pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", mitigation_options[i].option); |
| + pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", |
| + mitigation_options[i].option); |
| return SPECTRE_V2_CMD_AUTO; |
| } |
| |
| @@ -850,6 +879,24 @@ static enum spectre_v2_mitigation_cmd __ |
| return cmd; |
| } |
| |
| +static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) |
| +{ |
| + if (!IS_ENABLED(CONFIG_RETPOLINE)) { |
| + pr_err("Kernel not compiled with retpoline; no mitigation available!"); |
| + return SPECTRE_V2_NONE; |
| + } |
| + |
| + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { |
| + if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { |
| + pr_err("LFENCE not serializing, switching to generic retpoline\n"); |
| + return SPECTRE_V2_RETPOLINE; |
| + } |
| + return SPECTRE_V2_LFENCE; |
| + } |
| + |
| + return SPECTRE_V2_RETPOLINE; |
| +} |
| + |
| static void __init spectre_v2_select_mitigation(void) |
| { |
| enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); |
| @@ -870,48 +917,60 @@ static void __init spectre_v2_select_mit |
| case SPECTRE_V2_CMD_FORCE: |
| case SPECTRE_V2_CMD_AUTO: |
| if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { |
| - mode = SPECTRE_V2_IBRS_ENHANCED; |
| - /* Force it so VMEXIT will restore correctly */ |
| - x86_spec_ctrl_base |= SPEC_CTRL_IBRS; |
| - wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); |
| - goto specv2_set_mode; |
| + mode = SPECTRE_V2_EIBRS; |
| + break; |
| } |
| - if (IS_ENABLED(CONFIG_RETPOLINE)) |
| - goto retpoline_auto; |
| + |
| + mode = spectre_v2_select_retpoline(); |
| break; |
| + |
| case SPECTRE_V2_CMD_RETPOLINE_LFENCE: |
| - if (IS_ENABLED(CONFIG_RETPOLINE)) |
| - goto retpoline_lfence; |
| + mode = SPECTRE_V2_LFENCE; |
| break; |
| + |
| case SPECTRE_V2_CMD_RETPOLINE_GENERIC: |
| - if (IS_ENABLED(CONFIG_RETPOLINE)) |
| - goto retpoline_generic; |
| + mode = SPECTRE_V2_RETPOLINE; |
| break; |
| + |
| case SPECTRE_V2_CMD_RETPOLINE: |
| - if (IS_ENABLED(CONFIG_RETPOLINE)) |
| - goto retpoline_auto; |
| + mode = spectre_v2_select_retpoline(); |
| + break; |
| + |
| + case SPECTRE_V2_CMD_EIBRS: |
| + mode = SPECTRE_V2_EIBRS; |
| + break; |
| + |
| + case SPECTRE_V2_CMD_EIBRS_LFENCE: |
| + mode = SPECTRE_V2_EIBRS_LFENCE; |
| + break; |
| + |
| + case SPECTRE_V2_CMD_EIBRS_RETPOLINE: |
| + mode = SPECTRE_V2_EIBRS_RETPOLINE; |
| break; |
| } |
| - pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); |
| - return; |
| |
| -retpoline_auto: |
| - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { |
| - retpoline_lfence: |
| - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { |
| - pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); |
| - goto retpoline_generic; |
| - } |
| - mode = SPECTRE_V2_LFENCE; |
| + if (spectre_v2_in_eibrs_mode(mode)) { |
| + /* Force it so VMEXIT will restore correctly */ |
| + x86_spec_ctrl_base |= SPEC_CTRL_IBRS; |
| + wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); |
| + } |
| + |
| + switch (mode) { |
| + case SPECTRE_V2_NONE: |
| + case SPECTRE_V2_EIBRS: |
| + break; |
| + |
| + case SPECTRE_V2_LFENCE: |
| + case SPECTRE_V2_EIBRS_LFENCE: |
| setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); |
| + /* fallthrough */ |
| + |
| + case SPECTRE_V2_RETPOLINE: |
| + case SPECTRE_V2_EIBRS_RETPOLINE: |
| setup_force_cpu_cap(X86_FEATURE_RETPOLINE); |
| - } else { |
| - retpoline_generic: |
| - mode = SPECTRE_V2_RETPOLINE; |
| - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); |
| + break; |
| } |
| |
| -specv2_set_mode: |
| spectre_v2_enabled = mode; |
| pr_info("%s\n", spectre_v2_strings[mode]); |
| |
| @@ -937,7 +996,7 @@ specv2_set_mode: |
| * the CPU supports Enhanced IBRS, kernel might un-intentionally not |
| * enable IBRS around firmware calls. |
| */ |
| - if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { |
| + if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) { |
| setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); |
| pr_info("Enabling Restricted Speculation for firmware calls\n"); |
| } |
| @@ -1597,7 +1656,7 @@ static ssize_t tsx_async_abort_show_stat |
| |
| static char *stibp_state(void) |
| { |
| - if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) |
| + if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) |
| return ""; |
| |
| switch (spectre_v2_user_stibp) { |