| From foo@baz Wed Jan 3 18:58:12 CET 2018 |
| From: Hugh Dickins <hughd@google.com> |
| Date: Sun, 27 Aug 2017 16:24:27 -0700 |
| Subject: kaiser: x86_cr3_pcid_noflush and x86_cr3_pcid_user |
| |
| From: Hugh Dickins <hughd@google.com> |
| |
| |
| Mostly this commit is just unshouting X86_CR3_PCID_KERN_VAR and |
| X86_CR3_PCID_USER_VAR: we usually name variables in lower-case. |
| |
| But why does x86_cr3_pcid_noflush need to be __aligned(PAGE_SIZE)? |
| Ah, it's a leftover from when kaiser_add_user_map() once complained |
| about mapping the same page twice. Make it __read_mostly instead. |
| (I'm a little uneasy about all the unrelated data which shares its |
| page getting user-mapped too, but that was so before, and not a big |
| deal: though we call it user-mapped, it's not mapped with _PAGE_USER.) |
| |
| And there is a little change around the two calls to do_nmi(). |
| Previously they set the NOFLUSH bit (if PCID supported) when |
| forcing to kernel context before do_nmi(); now they also have the |
| NOFLUSH bit set (if PCID supported) when restoring context after: |
| nothing done in do_nmi() should require a TLB to be flushed here. |
| |
| Signed-off-by: Hugh Dickins <hughd@google.com> |
| Acked-by: Jiri Kosina <jkosina@suse.cz> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| arch/x86/entry/entry_64.S | 8 ++++---- |
| arch/x86/include/asm/kaiser.h | 11 +++++------ |
| arch/x86/mm/kaiser.c | 13 +++++++------ |
| 3 files changed, 16 insertions(+), 16 deletions(-) |
| |
| --- a/arch/x86/entry/entry_64.S |
| +++ b/arch/x86/entry/entry_64.S |
| @@ -1290,11 +1290,11 @@ ENTRY(nmi) |
| /* Unconditionally use kernel CR3 for do_nmi() */ |
| /* %rax is saved above, so OK to clobber here */ |
| movq %cr3, %rax |
| + /* If PCID enabled, NOFLUSH now and NOFLUSH on return */ |
| + orq x86_cr3_pcid_noflush, %rax |
| pushq %rax |
| /* mask off "user" bit of pgd address and 12 PCID bits: */ |
| andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax |
| - /* Add back kernel PCID and "no flush" bit */ |
| - orq X86_CR3_PCID_KERN_VAR, %rax |
| movq %rax, %cr3 |
| #endif |
| call do_nmi |
| @@ -1534,11 +1534,11 @@ end_repeat_nmi: |
| /* Unconditionally use kernel CR3 for do_nmi() */ |
| /* %rax is saved above, so OK to clobber here */ |
| movq %cr3, %rax |
| + /* If PCID enabled, NOFLUSH now and NOFLUSH on return */ |
| + orq x86_cr3_pcid_noflush, %rax |
| pushq %rax |
| /* mask off "user" bit of pgd address and 12 PCID bits: */ |
| andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax |
| - /* Add back kernel PCID and "no flush" bit */ |
| - orq X86_CR3_PCID_KERN_VAR, %rax |
| movq %rax, %cr3 |
| #endif |
| |
| --- a/arch/x86/include/asm/kaiser.h |
| +++ b/arch/x86/include/asm/kaiser.h |
| @@ -25,7 +25,7 @@ |
| .macro _SWITCH_TO_KERNEL_CR3 reg |
| movq %cr3, \reg |
| andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg |
| -orq X86_CR3_PCID_KERN_VAR, \reg |
| +orq x86_cr3_pcid_noflush, \reg |
| movq \reg, %cr3 |
| .endm |
| |
| @@ -37,11 +37,10 @@ movq \reg, %cr3 |
| * not enabled): so that the one register can update both memory and cr3. |
| */ |
| movq %cr3, \reg |
| -andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg |
| -orq PER_CPU_VAR(X86_CR3_PCID_USER_VAR), \reg |
| +orq PER_CPU_VAR(x86_cr3_pcid_user), \reg |
| js 9f |
| /* FLUSH this time, reset to NOFLUSH for next time (if PCID enabled) */ |
| -movb \regb, PER_CPU_VAR(X86_CR3_PCID_USER_VAR+7) |
| +movb \regb, PER_CPU_VAR(x86_cr3_pcid_user+7) |
| 9: |
| movq \reg, %cr3 |
| .endm |
| @@ -94,8 +93,8 @@ movq PER_CPU_VAR(unsafe_stack_register_b |
| */ |
| DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup); |
| |
| -extern unsigned long X86_CR3_PCID_KERN_VAR; |
| -DECLARE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR); |
| +extern unsigned long x86_cr3_pcid_noflush; |
| +DECLARE_PER_CPU(unsigned long, x86_cr3_pcid_user); |
| |
| extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[]; |
| |
| --- a/arch/x86/mm/kaiser.c |
| +++ b/arch/x86/mm/kaiser.c |
| @@ -29,8 +29,8 @@ DEFINE_PER_CPU_USER_MAPPED(unsigned long |
| * This is also handy because systems that do not support PCIDs |
| * just end up or'ing a 0 into their CR3, which does no harm. |
| */ |
| -__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_KERN_VAR; |
| -DEFINE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR); |
| +unsigned long x86_cr3_pcid_noflush __read_mostly; |
| +DEFINE_PER_CPU(unsigned long, x86_cr3_pcid_user); |
| |
| /* |
| * At runtime, the only things we map are some things for CPU |
| @@ -304,7 +304,8 @@ void __init kaiser_init(void) |
| sizeof(gate_desc) * NR_VECTORS, |
| __PAGE_KERNEL); |
| |
| - kaiser_add_user_map_early(&X86_CR3_PCID_KERN_VAR, PAGE_SIZE, |
| + kaiser_add_user_map_early(&x86_cr3_pcid_noflush, |
| + sizeof(x86_cr3_pcid_noflush), |
| __PAGE_KERNEL); |
| } |
| |
| @@ -384,8 +385,8 @@ void kaiser_setup_pcid(void) |
| * These variables are used by the entry/exit |
| * code to change PCID and pgd and TLB flushing. |
| */ |
| - X86_CR3_PCID_KERN_VAR = kern_cr3; |
| - this_cpu_write(X86_CR3_PCID_USER_VAR, user_cr3); |
| + x86_cr3_pcid_noflush = kern_cr3; |
| + this_cpu_write(x86_cr3_pcid_user, user_cr3); |
| } |
| |
| /* |
| @@ -395,7 +396,7 @@ void kaiser_setup_pcid(void) |
| */ |
| void kaiser_flush_tlb_on_return_to_user(void) |
| { |
| - this_cpu_write(X86_CR3_PCID_USER_VAR, |
| + this_cpu_write(x86_cr3_pcid_user, |
| X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET); |
| } |
| EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user); |