| From 216401f82bef982978a472666f715ca2f4acc329 Mon Sep 17 00:00:00 2001 |
| From: Peter Zijlstra <peterz@infradead.org> |
| Date: Thu, 11 Jul 2019 13:40:55 +0200 |
| Subject: [PATCH] x86/paravirt: Make read_cr2() CALLEE_SAVE |
| |
| commit 55aedddb6149ab71bec9f050846855113977b033 upstream. |
| |
| The one paravirt read_cr2() implementation (Xen) is actually quite trivial |
| and doesn't need to clobber anything other than the return register. |
| |
| Making read_cr2() CALLEE_SAVE avoids all the PUSH/POP nonsense and allows |
| more convenient use from assembly. |
| |
| Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| Reviewed-by: Juergen Gross <jgross@suse.com> |
| Cc: bp@alien8.de |
| Cc: rostedt@goodmis.org |
| Cc: luto@kernel.org |
| Cc: torvalds@linux-foundation.org |
| Cc: hpa@zytor.com |
| Cc: dave.hansen@linux.intel.com |
| Cc: zhe.he@windriver.com |
| Cc: joel@joelfernandes.org |
| Cc: devel@etsukata.com |
| Link: https://lkml.kernel.org/r/20190711114335.887392493@infradead.org |
| Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com> |
| |
| diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h |
| index 7ce7ac9d9d3f..515c0ceeb4a3 100644 |
| --- a/arch/x86/entry/calling.h |
| +++ b/arch/x86/entry/calling.h |
| @@ -360,3 +360,9 @@ For 32-bit we have the following conventions - kernel is built with |
| .Lafter_call_\@: |
| #endif |
| .endm |
| + |
| +#ifdef CONFIG_PARAVIRT_XXL |
| +#define GET_CR2_INTO(reg) GET_CR2_INTO_AX ; _ASM_MOV %_ASM_AX, reg |
| +#else |
| +#define GET_CR2_INTO(reg) _ASM_MOV %cr2, reg |
| +#endif |
| diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h |
| index d6f5ae2c79ab..dce26f1d13e1 100644 |
| --- a/arch/x86/include/asm/paravirt.h |
| +++ b/arch/x86/include/asm/paravirt.h |
| @@ -116,7 +116,7 @@ static inline void write_cr0(unsigned long x) |
| |
| static inline unsigned long read_cr2(void) |
| { |
| - return PVOP_CALL0(unsigned long, mmu.read_cr2); |
| + return PVOP_CALLEE0(unsigned long, mmu.read_cr2); |
| } |
| |
| static inline void write_cr2(unsigned long x) |
| @@ -910,13 +910,7 @@ extern void default_banner(void); |
| ANNOTATE_RETPOLINE_SAFE; \ |
| call PARA_INDIRECT(pv_ops+PV_CPU_swapgs); \ |
| ) |
| -#endif |
| - |
| -#define GET_CR2_INTO_RAX \ |
| - ANNOTATE_RETPOLINE_SAFE; \ |
| - call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2); |
| |
| -#ifdef CONFIG_PARAVIRT_XXL |
| #define USERGS_SYSRET64 \ |
| PARA_SITE(PARA_PATCH(PV_CPU_usergs_sysret64), \ |
| ANNOTATE_RETPOLINE_SAFE; \ |
| @@ -930,9 +924,19 @@ extern void default_banner(void); |
| call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl); \ |
| PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) |
| #endif |
| -#endif |
| +#endif /* CONFIG_PARAVIRT_XXL */ |
| +#endif /* CONFIG_X86_64 */ |
| + |
| +#ifdef CONFIG_PARAVIRT_XXL |
| + |
| +#define GET_CR2_INTO_AX \ |
| + PARA_SITE(PARA_PATCH(PV_MMU_read_cr2), \ |
| + ANNOTATE_RETPOLINE_SAFE; \ |
| + call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2); \ |
| + ) |
| + |
| +#endif /* CONFIG_PARAVIRT_XXL */ |
| |
| -#endif /* CONFIG_X86_32 */ |
| |
| #endif /* __ASSEMBLY__ */ |
| #else /* CONFIG_PARAVIRT */ |
| diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h |
| index 2474e434a6f7..8e46f1b764a3 100644 |
| --- a/arch/x86/include/asm/paravirt_types.h |
| +++ b/arch/x86/include/asm/paravirt_types.h |
| @@ -220,7 +220,7 @@ struct pv_mmu_ops { |
| void (*exit_mmap)(struct mm_struct *mm); |
| |
| #ifdef CONFIG_PARAVIRT_XXL |
| - unsigned long (*read_cr2)(void); |
| + struct paravirt_callee_save read_cr2; |
| void (*write_cr2)(unsigned long); |
| |
| unsigned long (*read_cr3)(void); |
| diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c |
| index da64452584b0..5c7ee3df4d0b 100644 |
| --- a/arch/x86/kernel/asm-offsets.c |
| +++ b/arch/x86/kernel/asm-offsets.c |
| @@ -76,6 +76,7 @@ static void __used common(void) |
| BLANK(); |
| OFFSET(XEN_vcpu_info_mask, vcpu_info, evtchn_upcall_mask); |
| OFFSET(XEN_vcpu_info_pending, vcpu_info, evtchn_upcall_pending); |
| + OFFSET(XEN_vcpu_info_arch_cr2, vcpu_info, arch.cr2); |
| #endif |
| |
| BLANK(); |
| diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S |
| index bcd206c8ac90..0e2d72929a8c 100644 |
| --- a/arch/x86/kernel/head_64.S |
| +++ b/arch/x86/kernel/head_64.S |
| @@ -29,9 +29,7 @@ |
| #ifdef CONFIG_PARAVIRT_XXL |
| #include <asm/asm-offsets.h> |
| #include <asm/paravirt.h> |
| -#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg |
| #else |
| -#define GET_CR2_INTO(reg) movq %cr2, reg |
| #define INTERRUPT_RETURN iretq |
| #endif |
| |
| @@ -323,7 +321,7 @@ early_idt_handler_common: |
| |
| cmpq $14,%rsi /* Page fault? */ |
| jnz 10f |
| - GET_CR2_INTO(%rdi) /* Can clobber any volatile register if pv */ |
| + GET_CR2_INTO(%rdi) /* can clobber %rax if pv */ |
| call early_make_pgtable |
| andl %eax,%eax |
| jz 20f /* All good */ |
| diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c |
| index 06f6bb48d018..f8ea3a00bdd9 100644 |
| --- a/arch/x86/kernel/paravirt.c |
| +++ b/arch/x86/kernel/paravirt.c |
| @@ -370,7 +370,7 @@ struct paravirt_patch_template pv_ops = { |
| .mmu.exit_mmap = paravirt_nop, |
| |
| #ifdef CONFIG_PARAVIRT_XXL |
| - .mmu.read_cr2 = native_read_cr2, |
| + .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(native_read_cr2), |
| .mmu.write_cr2 = native_write_cr2, |
| .mmu.read_cr3 = __native_read_cr3, |
| .mmu.write_cr3 = native_write_cr3, |
| diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c |
| index 30c14cb343fc..896cfd22cfda 100644 |
| --- a/arch/x86/xen/enlighten_pv.c |
| +++ b/arch/x86/xen/enlighten_pv.c |
| @@ -998,7 +998,8 @@ void __init xen_setup_vcpu_info_placement(void) |
| __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); |
| pv_ops.irq.irq_enable = |
| __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); |
| - pv_ops.mmu.read_cr2 = xen_read_cr2_direct; |
| + pv_ops.mmu.read_cr2 = |
| + __PV_IS_CALLEE_SAVE(xen_read_cr2_direct); |
| } |
| } |
| |
| diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c |
| index beb44e22afdf..bda8700cd8c6 100644 |
| --- a/arch/x86/xen/mmu_pv.c |
| +++ b/arch/x86/xen/mmu_pv.c |
| @@ -1307,16 +1307,6 @@ static void xen_write_cr2(unsigned long cr2) |
| this_cpu_read(xen_vcpu)->arch.cr2 = cr2; |
| } |
| |
| -static unsigned long xen_read_cr2(void) |
| -{ |
| - return this_cpu_read(xen_vcpu)->arch.cr2; |
| -} |
| - |
| -unsigned long xen_read_cr2_direct(void) |
| -{ |
| - return this_cpu_read(xen_vcpu_info.arch.cr2); |
| -} |
| - |
| static noinline void xen_flush_tlb(void) |
| { |
| struct mmuext_op *op; |
| @@ -2397,7 +2387,7 @@ static void xen_leave_lazy_mmu(void) |
| } |
| |
| static const struct pv_mmu_ops xen_mmu_ops __initconst = { |
| - .read_cr2 = xen_read_cr2, |
| + .read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2), |
| .write_cr2 = xen_write_cr2, |
| |
| .read_cr3 = xen_read_cr3, |
| diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S |
| index 8019edd0125c..be104eef80be 100644 |
| --- a/arch/x86/xen/xen-asm.S |
| +++ b/arch/x86/xen/xen-asm.S |
| @@ -10,6 +10,7 @@ |
| #include <asm/percpu.h> |
| #include <asm/processor-flags.h> |
| #include <asm/frame.h> |
| +#include <asm/asm.h> |
| |
| #include <linux/linkage.h> |
| |
| @@ -135,3 +136,18 @@ ENTRY(check_events) |
| FRAME_END |
| ret |
| ENDPROC(check_events) |
| + |
| +ENTRY(xen_read_cr2) |
| + FRAME_BEGIN |
| + _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX |
| + _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX |
| + FRAME_END |
| + ret |
| + ENDPROC(xen_read_cr2); |
| + |
| +ENTRY(xen_read_cr2_direct) |
| + FRAME_BEGIN |
| + _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX |
| + FRAME_END |
| + ret |
| + ENDPROC(xen_read_cr2_direct); |
| diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h |
| index 2f111f47ba98..45a441c33d6d 100644 |
| --- a/arch/x86/xen/xen-ops.h |
| +++ b/arch/x86/xen/xen-ops.h |
| @@ -134,6 +134,9 @@ __visible void xen_irq_disable_direct(void); |
| __visible unsigned long xen_save_fl_direct(void); |
| __visible void xen_restore_fl_direct(unsigned long); |
| |
| +__visible unsigned long xen_read_cr2(void); |
| +__visible unsigned long xen_read_cr2_direct(void); |
| + |
| /* These are not functions, and cannot be called normally */ |
| __visible void xen_iret(void); |
| __visible void xen_sysret32(void); |
| -- |
| 2.7.4 |
| |