| From: Hugh Dickins <hughd@google.com> |
| Date: Sat, 4 Nov 2017 18:43:06 -0700 |
| Subject: kaiser: kaiser_flush_tlb_on_return_to_user() check PCID |
| |
| Let kaiser_flush_tlb_on_return_to_user() do the X86_FEATURE_PCID |
| check, instead of each caller doing it inline first: nobody needs |
| to optimize for the noPCID case, it's clearer this way, and better |
| suits later changes. Replace those no-op X86_CR3_PCID_KERN_FLUSH lines |
| by a BUILD_BUG_ON() in load_new_mm_cr3(), in case something changes. |
| |
| (cherry picked from Change-Id: I9b528ed9d7c1ae4a3b4738c2894ee1740b6fb0b9) |
| |
| Signed-off-by: Hugh Dickins <hughd@google.com> |
| Signed-off-by: Ben Hutchings <ben@decadent.org.uk> |
| --- |
| arch/x86/include/asm/tlbflush.h | 4 ++-- |
| arch/x86/mm/kaiser.c | 6 +++--- |
| arch/x86/mm/tlb.c | 8 ++++---- |
| 3 files changed, 9 insertions(+), 9 deletions(-) |
| |
| diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h |
| index 63a101f3379c..88d849e51d72 100644 |
| --- a/arch/x86/include/asm/tlbflush.h |
| +++ b/arch/x86/include/asm/tlbflush.h |
| @@ -90,7 +90,7 @@ static inline void __native_flush_tlb(void) |
| * back: |
| */ |
| preempt_disable(); |
| - if (kaiser_enabled && this_cpu_has(X86_FEATURE_PCID)) |
| + if (kaiser_enabled) |
| kaiser_flush_tlb_on_return_to_user(); |
| native_write_cr3(native_read_cr3()); |
| preempt_enable(); |
| @@ -145,7 +145,7 @@ static inline void __native_flush_tlb_single(unsigned long addr) |
| */ |
| |
| if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) { |
| - if (kaiser_enabled && this_cpu_has(X86_FEATURE_PCID)) |
| + if (kaiser_enabled) |
| kaiser_flush_tlb_on_return_to_user(); |
| asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); |
| return; |
| diff --git a/arch/x86/mm/kaiser.c b/arch/x86/mm/kaiser.c |
| index d149557bcbe8..7e8c59825f6d 100644 |
| --- a/arch/x86/mm/kaiser.c |
| +++ b/arch/x86/mm/kaiser.c |
| @@ -429,12 +429,12 @@ void kaiser_setup_pcid(void) |
| |
| /* |
| * Make a note that this cpu will need to flush USER tlb on return to user. |
| - * Caller checks whether this_cpu_has(X86_FEATURE_PCID) before calling: |
| - * if cpu does not, then the NOFLUSH bit will never have been set. |
| + * If cpu does not have PCID, then the NOFLUSH bit will never have been set. |
| */ |
| void kaiser_flush_tlb_on_return_to_user(void) |
| { |
| - this_cpu_write(x86_cr3_pcid_user, |
| + if (this_cpu_has(X86_FEATURE_PCID)) |
| + this_cpu_write(x86_cr3_pcid_user, |
| X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET); |
| } |
| EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user); |
| diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c |
| index 34515ec300b8..b011d5d44c68 100644 |
| --- a/arch/x86/mm/tlb.c |
| +++ b/arch/x86/mm/tlb.c |
| @@ -21,7 +21,7 @@ static void load_new_mm_cr3(pgd_t *pgdir) |
| { |
| unsigned long new_mm_cr3 = __pa(pgdir); |
| |
| - if (kaiser_enabled && this_cpu_has(X86_FEATURE_PCID)) { |
| + if (kaiser_enabled) { |
| /* |
| * We reuse the same PCID for different tasks, so we must |
| * flush all the entries for the PCID out when we change tasks. |
| @@ -32,10 +32,10 @@ static void load_new_mm_cr3(pgd_t *pgdir) |
| * do it here, but can only be used if X86_FEATURE_INVPCID is |
| * available - and many machines support pcid without invpcid. |
| * |
| - * The line below is a no-op: X86_CR3_PCID_KERN_FLUSH is now 0; |
| - * but keep that line in there in case something changes. |
| + * If X86_CR3_PCID_KERN_FLUSH actually added something, then it |
| + * would be needed in the write_cr3() below - if PCIDs enabled. |
| */ |
| - new_mm_cr3 |= X86_CR3_PCID_KERN_FLUSH; |
| + BUILD_BUG_ON(X86_CR3_PCID_KERN_FLUSH); |
| kaiser_flush_tlb_on_return_to_user(); |
| } |
| |
| |