| From b644bb0c3a9f5b9c93b70387b1977ef9eace2500 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Mon, 14 Sep 2020 14:52:19 +1000 |
| Subject: powerpc/64s/radix: Fix mm_cpumask trimming race vs kthread_use_mm |
| |
| From: Nicholas Piggin <npiggin@gmail.com> |
| |
| [ Upstream commit a665eec0a22e11cdde708c1c256a465ebe768047 ] |
| |
| Commit 0cef77c7798a7 ("powerpc/64s/radix: flush remote CPUs out of |
| single-threaded mm_cpumask") added a mechanism to trim the mm_cpumask of |
| a process under certain conditions. One of the assumptions is that |
| mm_users would not be incremented via a reference outside the process |
| context with mmget_not_zero() then go on to kthread_use_mm() via that |
| reference. |
| |
| That invariant was broken by io_uring code (see previous sparc64 fix), |
| but I'll point Fixes: to the original powerpc commit because we are |
| changing that assumption going forward, so this will make backports |
| match up. |
| |
| Fix this by no longer relying on that assumption, but by having each CPU |
| check the mm is not being used, and clearing their own bit from the mask |
| only if it hasn't been switched-to by the time the IPI is processed. |
| |
| This relies on commit 38cf307c1f20 ("mm: fix kthread_use_mm() vs TLB |
| invalidate") and ARCH_WANT_IRQS_OFF_ACTIVATE_MM to disable irqs over mm |
| switch sequences. |
| |
| Fixes: 0cef77c7798a7 ("powerpc/64s/radix: flush remote CPUs out of single-threaded mm_cpumask") |
| Signed-off-by: Nicholas Piggin <npiggin@gmail.com> |
| Reviewed-by: Michael Ellerman <mpe@ellerman.id.au> |
| Depends-on: 38cf307c1f20 ("mm: fix kthread_use_mm() vs TLB invalidate") |
| Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> |
| Link: https://lore.kernel.org/r/20200914045219.3736466-5-npiggin@gmail.com |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| arch/powerpc/include/asm/tlb.h | 13 ------------- |
| arch/powerpc/mm/tlb-radix.c | 23 ++++++++++++++++------- |
| 2 files changed, 16 insertions(+), 20 deletions(-) |
| |
| diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h |
| index f0e571b2dc7c8..a6073fecdacd3 100644 |
| --- a/arch/powerpc/include/asm/tlb.h |
| +++ b/arch/powerpc/include/asm/tlb.h |
| @@ -76,19 +76,6 @@ static inline int mm_is_thread_local(struct mm_struct *mm) |
| return false; |
| return cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm)); |
| } |
| -static inline void mm_reset_thread_local(struct mm_struct *mm) |
| -{ |
| - WARN_ON(atomic_read(&mm->context.copros) > 0); |
| - /* |
| - * It's possible for mm_access to take a reference on mm_users to |
| - * access the remote mm from another thread, but it's not allowed |
| - * to set mm_cpumask, so mm_users may be > 1 here. |
| - */ |
| - WARN_ON(current->mm != mm); |
| - atomic_set(&mm->context.active_cpus, 1); |
| - cpumask_clear(mm_cpumask(mm)); |
| - cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); |
| -} |
| #else /* CONFIG_PPC_BOOK3S_64 */ |
| static inline int mm_is_thread_local(struct mm_struct *mm) |
| { |
| diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c |
| index 1749f15fc0705..80b8fc4173de6 100644 |
| --- a/arch/powerpc/mm/tlb-radix.c |
| +++ b/arch/powerpc/mm/tlb-radix.c |
| @@ -598,19 +598,29 @@ static void do_exit_flush_lazy_tlb(void *arg) |
| struct mm_struct *mm = arg; |
| unsigned long pid = mm->context.id; |
| |
| + /* |
| + * A kthread could have done a mmget_not_zero() after the flushing CPU |
| + * checked mm_is_singlethreaded, and be in the process of |
| + * kthread_use_mm when interrupted here. In that case, current->mm will |
| + * be set to mm, because kthread_use_mm() setting ->mm and switching to |
| + * the mm is done with interrupts off. |
| + */ |
| if (current->mm == mm) |
| - return; /* Local CPU */ |
| + goto out_flush; |
| |
| if (current->active_mm == mm) { |
| - /* |
| - * Must be a kernel thread because sender is single-threaded. |
| - */ |
| - BUG_ON(current->mm); |
| + WARN_ON_ONCE(current->mm != NULL); |
| + /* Is a kernel thread and is using mm as the lazy tlb */ |
| mmgrab(&init_mm); |
| - switch_mm(mm, &init_mm, current); |
| current->active_mm = &init_mm; |
| + switch_mm_irqs_off(mm, &init_mm, current); |
| mmdrop(mm); |
| } |
| + |
| + atomic_dec(&mm->context.active_cpus); |
| + cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm)); |
| + |
| +out_flush: |
| _tlbiel_pid(pid, RIC_FLUSH_ALL); |
| } |
| |
| @@ -625,7 +635,6 @@ static void exit_flush_lazy_tlbs(struct mm_struct *mm) |
| */ |
| smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb, |
| (void *)mm, 1); |
| - mm_reset_thread_local(mm); |
| } |
| |
| void radix__flush_tlb_mm(struct mm_struct *mm) |
| -- |
| 2.25.1 |
| |