| From 5b0a9db080db56e1ec19528ce98e67ea36519878 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Fri, 18 Jun 2021 13:54:08 +0200 |
| Subject: x86/sev: Make sure IRQs are disabled while GHCB is active |
| |
| From: Joerg Roedel <jroedel@suse.de> |
| |
| [ Upstream commit d187f217335dba2b49fc9002aab2004e04acddee ] |
| |
| The #VC handler only cares about IRQs being disabled while the GHCB is |
| active, as it must not be interrupted by something which could cause |
| another #VC while it holds the GHCB (NMI is the exception for which the |
| backup GHCB exits). |
| |
| Make sure nothing interrupts the code path while the GHCB is active |
| by making sure that callers of __sev_{get,put}_ghcb() have disabled |
| interrupts upfront. |
| |
| [ bp: Massage commit message. ] |
| |
| Signed-off-by: Joerg Roedel <jroedel@suse.de> |
| Signed-off-by: Borislav Petkov <bp@suse.de> |
| Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> |
| Link: https://lkml.kernel.org/r/20210618115409.22735-2-joro@8bytes.org |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| arch/x86/kernel/sev-es.c | 34 ++++++++++++++++++++++------------ |
| 1 file changed, 22 insertions(+), 12 deletions(-) |
| |
| diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c |
| index e0cdab7cb632..0b5e35a51804 100644 |
| --- a/arch/x86/kernel/sev-es.c |
| +++ b/arch/x86/kernel/sev-es.c |
| @@ -12,7 +12,6 @@ |
| #include <linux/sched/debug.h> /* For show_regs() */ |
| #include <linux/percpu-defs.h> |
| #include <linux/mem_encrypt.h> |
| -#include <linux/lockdep.h> |
| #include <linux/printk.h> |
| #include <linux/mm_types.h> |
| #include <linux/set_memory.h> |
| @@ -180,11 +179,19 @@ void noinstr __sev_es_ist_exit(void) |
| this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); |
| } |
| |
| -static __always_inline struct ghcb *sev_es_get_ghcb(struct ghcb_state *state) |
| +/* |
| + * Nothing shall interrupt this code path while holding the per-CPU |
| + * GHCB. The backup GHCB is only for NMIs interrupting this path. |
| + * |
| + * Callers must disable local interrupts around it. |
| + */ |
| +static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) |
| { |
| struct sev_es_runtime_data *data; |
| struct ghcb *ghcb; |
| |
| + WARN_ON(!irqs_disabled()); |
| + |
| data = this_cpu_read(runtime_data); |
| ghcb = &data->ghcb_page; |
| |
| @@ -201,7 +208,9 @@ static __always_inline struct ghcb *sev_es_get_ghcb(struct ghcb_state *state) |
| data->ghcb_active = false; |
| data->backup_ghcb_active = false; |
| |
| + instrumentation_begin(); |
| panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); |
| + instrumentation_end(); |
| } |
| |
| /* Mark backup_ghcb active before writing to it */ |
| @@ -452,11 +461,13 @@ static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt |
| /* Include code shared with pre-decompression boot stage */ |
| #include "sev-es-shared.c" |
| |
| -static __always_inline void sev_es_put_ghcb(struct ghcb_state *state) |
| +static noinstr void __sev_put_ghcb(struct ghcb_state *state) |
| { |
| struct sev_es_runtime_data *data; |
| struct ghcb *ghcb; |
| |
| + WARN_ON(!irqs_disabled()); |
| + |
| data = this_cpu_read(runtime_data); |
| ghcb = &data->ghcb_page; |
| |
| @@ -480,7 +491,7 @@ void noinstr __sev_es_nmi_complete(void) |
| struct ghcb_state state; |
| struct ghcb *ghcb; |
| |
| - ghcb = sev_es_get_ghcb(&state); |
| + ghcb = __sev_get_ghcb(&state); |
| |
| vc_ghcb_invalidate(ghcb); |
| ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); |
| @@ -490,7 +501,7 @@ void noinstr __sev_es_nmi_complete(void) |
| sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); |
| VMGEXIT(); |
| |
| - sev_es_put_ghcb(&state); |
| + __sev_put_ghcb(&state); |
| } |
| |
| static u64 get_jump_table_addr(void) |
| @@ -502,7 +513,7 @@ static u64 get_jump_table_addr(void) |
| |
| local_irq_save(flags); |
| |
| - ghcb = sev_es_get_ghcb(&state); |
| + ghcb = __sev_get_ghcb(&state); |
| |
| vc_ghcb_invalidate(ghcb); |
| ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE); |
| @@ -516,7 +527,7 @@ static u64 get_jump_table_addr(void) |
| ghcb_sw_exit_info_2_is_valid(ghcb)) |
| ret = ghcb->save.sw_exit_info_2; |
| |
| - sev_es_put_ghcb(&state); |
| + __sev_put_ghcb(&state); |
| |
| local_irq_restore(flags); |
| |
| @@ -641,7 +652,7 @@ static void sev_es_ap_hlt_loop(void) |
| struct ghcb_state state; |
| struct ghcb *ghcb; |
| |
| - ghcb = sev_es_get_ghcb(&state); |
| + ghcb = __sev_get_ghcb(&state); |
| |
| while (true) { |
| vc_ghcb_invalidate(ghcb); |
| @@ -658,7 +669,7 @@ static void sev_es_ap_hlt_loop(void) |
| break; |
| } |
| |
| - sev_es_put_ghcb(&state); |
| + __sev_put_ghcb(&state); |
| } |
| |
| /* |
| @@ -1317,7 +1328,6 @@ DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) |
| } |
| |
| irq_state = irqentry_nmi_enter(regs); |
| - lockdep_assert_irqs_disabled(); |
| instrumentation_begin(); |
| |
| /* |
| @@ -1326,7 +1336,7 @@ DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) |
| * keep the IRQs disabled to protect us against concurrent TLB flushes. |
| */ |
| |
| - ghcb = sev_es_get_ghcb(&state); |
| + ghcb = __sev_get_ghcb(&state); |
| |
| vc_ghcb_invalidate(ghcb); |
| result = vc_init_em_ctxt(&ctxt, regs, error_code); |
| @@ -1334,7 +1344,7 @@ DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication) |
| if (result == ES_OK) |
| result = vc_handle_exitcode(&ctxt, ghcb, error_code); |
| |
| - sev_es_put_ghcb(&state); |
| + __sev_put_ghcb(&state); |
| |
| /* Done - now check the result */ |
| switch (result) { |
| -- |
| 2.30.2 |
| |