| From c9f2a9a65e4855b74d92cdad688f6ee4a1a323ff Mon Sep 17 00:00:00 2001 |
| From: Matt Fleming <matt@codeblueprint.co.uk> |
| Date: Fri, 27 Nov 2015 21:09:33 +0000 |
| Subject: x86/efi: Hoist page table switching code into efi_call_virt() |
| |
| From: Matt Fleming <matt@codeblueprint.co.uk> |
| |
| commit c9f2a9a65e4855b74d92cdad688f6ee4a1a323ff upstream. |
| |
| This change is a prerequisite for pending patches that switch to |
| a dedicated EFI page table, instead of using 'trampoline_pgd' |
| which shares PGD entries with 'swapper_pg_dir'. The pending |
| patches make it impossible to dereference the runtime service |
| function pointer without first switching %cr3. |
| |
| It's true that we now have duplicated switching code in |
| efi_call_virt() and efi_call_phys_{prolog,epilog}() but we are |
| sacrificing code duplication for a little more clarity and the |
| ease of writing the page table switching code in C instead of |
| asm. |
| |
| Signed-off-by: Matt Fleming <matt@codeblueprint.co.uk> |
| Reviewed-by: Borislav Petkov <bp@suse.de> |
| Acked-by: Borislav Petkov <bp@suse.de> |
| Cc: Andrew Morton <akpm@linux-foundation.org> |
| Cc: Andy Lutomirski <luto@amacapital.net> |
| Cc: Andy Lutomirski <luto@kernel.org> |
| Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org> |
| Cc: Borislav Petkov <bp@alien8.de> |
| Cc: Brian Gerst <brgerst@gmail.com> |
| Cc: Dave Jones <davej@codemonkey.org.uk> |
| Cc: Denys Vlasenko <dvlasenk@redhat.com> |
| Cc: H. Peter Anvin <hpa@zytor.com> |
| Cc: Linus Torvalds <torvalds@linux-foundation.org> |
| Cc: Peter Zijlstra <peterz@infradead.org> |
| Cc: Sai Praneeth Prakhya <sai.praneeth.prakhya@intel.com> |
| Cc: Stephen Smalley <sds@tycho.nsa.gov> |
| Cc: Thomas Gleixner <tglx@linutronix.de> |
| Cc: Toshi Kani <toshi.kani@hp.com> |
| Cc: linux-efi@vger.kernel.org |
| Link: http://lkml.kernel.org/r/1448658575-17029-5-git-send-email-matt@codeblueprint.co.uk |
| Signed-off-by: Ingo Molnar <mingo@kernel.org> |
| Cc: "Ghannam, Yazen" <Yazen.Ghannam@amd.com> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/x86/include/asm/efi.h | 25 ++++++++++++++++++++ |
| arch/x86/platform/efi/efi_64.c | 24 +++++++++----------- |
| arch/x86/platform/efi/efi_stub_64.S | 43 ------------------------------------ |
| 3 files changed, 36 insertions(+), 56 deletions(-) |
| |
| --- a/arch/x86/include/asm/efi.h |
| +++ b/arch/x86/include/asm/efi.h |
| @@ -3,6 +3,7 @@ |
| |
| #include <asm/fpu/api.h> |
| #include <asm/pgtable.h> |
| +#include <asm/tlb.h> |
| |
| /* |
| * We map the EFI regions needed for runtime services non-contiguously, |
| @@ -64,6 +65,17 @@ extern u64 asmlinkage efi_call(void *fp, |
| |
| #define efi_call_phys(f, args...) efi_call((f), args) |
| |
| +/* |
| + * Scratch space used for switching the pagetable in the EFI stub |
| + */ |
| +struct efi_scratch { |
| + u64 r15; |
| + u64 prev_cr3; |
| + pgd_t *efi_pgt; |
| + bool use_pgd; |
| + u64 phys_stack; |
| +} __packed; |
| + |
| #define efi_call_virt(f, ...) \ |
| ({ \ |
| efi_status_t __s; \ |
| @@ -71,7 +83,20 @@ extern u64 asmlinkage efi_call(void *fp, |
| efi_sync_low_kernel_mappings(); \ |
| preempt_disable(); \ |
| __kernel_fpu_begin(); \ |
| + \ |
| + if (efi_scratch.use_pgd) { \ |
| + efi_scratch.prev_cr3 = read_cr3(); \ |
| + write_cr3((unsigned long)efi_scratch.efi_pgt); \ |
| + __flush_tlb_all(); \ |
| + } \ |
| + \ |
| __s = efi_call((void *)efi.systab->runtime->f, __VA_ARGS__); \ |
| + \ |
| + if (efi_scratch.use_pgd) { \ |
| + write_cr3(efi_scratch.prev_cr3); \ |
| + __flush_tlb_all(); \ |
| + } \ |
| + \ |
| __kernel_fpu_end(); \ |
| preempt_enable(); \ |
| __s; \ |
| --- a/arch/x86/platform/efi/efi_64.c |
| +++ b/arch/x86/platform/efi/efi_64.c |
| @@ -47,16 +47,7 @@ |
| */ |
| static u64 efi_va = EFI_VA_START; |
| |
| -/* |
| - * Scratch space used for switching the pagetable in the EFI stub |
| - */ |
| -struct efi_scratch { |
| - u64 r15; |
| - u64 prev_cr3; |
| - pgd_t *efi_pgt; |
| - bool use_pgd; |
| - u64 phys_stack; |
| -} __packed; |
| +struct efi_scratch efi_scratch; |
| |
| static void __init early_code_mapping_set_exec(int executable) |
| { |
| @@ -83,8 +74,11 @@ pgd_t * __init efi_call_phys_prolog(void |
| int pgd; |
| int n_pgds; |
| |
| - if (!efi_enabled(EFI_OLD_MEMMAP)) |
| - return NULL; |
| + if (!efi_enabled(EFI_OLD_MEMMAP)) { |
| + save_pgd = (pgd_t *)read_cr3(); |
| + write_cr3((unsigned long)efi_scratch.efi_pgt); |
| + goto out; |
| + } |
| |
| early_code_mapping_set_exec(1); |
| |
| @@ -96,6 +90,7 @@ pgd_t * __init efi_call_phys_prolog(void |
| vaddress = (unsigned long)__va(pgd * PGDIR_SIZE); |
| set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress)); |
| } |
| +out: |
| __flush_tlb_all(); |
| |
| return save_pgd; |
| @@ -109,8 +104,11 @@ void __init efi_call_phys_epilog(pgd_t * |
| int pgd_idx; |
| int nr_pgds; |
| |
| - if (!save_pgd) |
| + if (!efi_enabled(EFI_OLD_MEMMAP)) { |
| + write_cr3((unsigned long)save_pgd); |
| + __flush_tlb_all(); |
| return; |
| + } |
| |
| nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); |
| |
| --- a/arch/x86/platform/efi/efi_stub_64.S |
| +++ b/arch/x86/platform/efi/efi_stub_64.S |
| @@ -38,41 +38,6 @@ |
| mov %rsi, %cr0; \ |
| mov (%rsp), %rsp |
| |
| - /* stolen from gcc */ |
| - .macro FLUSH_TLB_ALL |
| - movq %r15, efi_scratch(%rip) |
| - movq %r14, efi_scratch+8(%rip) |
| - movq %cr4, %r15 |
| - movq %r15, %r14 |
| - andb $0x7f, %r14b |
| - movq %r14, %cr4 |
| - movq %r15, %cr4 |
| - movq efi_scratch+8(%rip), %r14 |
| - movq efi_scratch(%rip), %r15 |
| - .endm |
| - |
| - .macro SWITCH_PGT |
| - cmpb $0, efi_scratch+24(%rip) |
| - je 1f |
| - movq %r15, efi_scratch(%rip) # r15 |
| - # save previous CR3 |
| - movq %cr3, %r15 |
| - movq %r15, efi_scratch+8(%rip) # prev_cr3 |
| - movq efi_scratch+16(%rip), %r15 # EFI pgt |
| - movq %r15, %cr3 |
| - 1: |
| - .endm |
| - |
| - .macro RESTORE_PGT |
| - cmpb $0, efi_scratch+24(%rip) |
| - je 2f |
| - movq efi_scratch+8(%rip), %r15 |
| - movq %r15, %cr3 |
| - movq efi_scratch(%rip), %r15 |
| - FLUSH_TLB_ALL |
| - 2: |
| - .endm |
| - |
| ENTRY(efi_call) |
| SAVE_XMM |
| mov (%rsp), %rax |
| @@ -83,16 +48,8 @@ ENTRY(efi_call) |
| mov %r8, %r9 |
| mov %rcx, %r8 |
| mov %rsi, %rcx |
| - SWITCH_PGT |
| call *%rdi |
| - RESTORE_PGT |
| addq $48, %rsp |
| RESTORE_XMM |
| ret |
| ENDPROC(efi_call) |
| - |
| - .data |
| -ENTRY(efi_scratch) |
| - .fill 3,8,0 |
| - .byte 0 |
| - .quad 0 |