| From 7a9cdebdcc17e426fb5287e4a82db1dfe86339b2 Mon Sep 17 00:00:00 2001 |
| From: Linus Torvalds <torvalds@linux-foundation.org> |
| Date: Wed, 12 Sep 2018 23:57:48 -1000 |
| Subject: mm: get rid of vmacache_flush_all() entirely |
| |
| From: Linus Torvalds <torvalds@linux-foundation.org> |
| |
| commit 7a9cdebdcc17e426fb5287e4a82db1dfe86339b2 upstream. |
| |
| Jann Horn points out that the vmacache_flush_all() function is not only |
| potentially expensive, it's buggy too. It also happens to be entirely |
| unnecessary, because the sequence number overflow case can be avoided by |
| simply making the sequence number be 64-bit. That doesn't even grow the |
| data structures in question, because the other adjacent fields are |
| already 64-bit. |
| |
| So simplify the whole thing by just making the sequence number overflow |
| case go away entirely, which gets rid of all the complications and makes |
| the code faster too. Win-win. |
| |
| [ Oleg Nesterov points out that the VMACACHE_FULL_FLUSHES statistics |
| also just goes away entirely with this ] |
| |
| Reported-by: Jann Horn <jannh@google.com> |
| Suggested-by: Will Deacon <will.deacon@arm.com> |
| Acked-by: Davidlohr Bueso <dave@stgolabs.net> |
| Cc: Oleg Nesterov <oleg@redhat.com> |
| Cc: stable@kernel.org |
| Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| include/linux/mm_types.h | 2 +- |
| include/linux/mm_types_task.h | 2 +- |
| include/linux/vm_event_item.h | 1 - |
| include/linux/vmacache.h | 5 ----- |
| mm/debug.c | 4 ++-- |
| mm/vmacache.c | 38 -------------------------------------- |
| 6 files changed, 4 insertions(+), 48 deletions(-) |
| |
| --- a/include/linux/mm_types.h |
| +++ b/include/linux/mm_types.h |
| @@ -354,7 +354,7 @@ struct kioctx_table; |
| struct mm_struct { |
| struct vm_area_struct *mmap; /* list of VMAs */ |
| struct rb_root mm_rb; |
| - u32 vmacache_seqnum; /* per-thread vmacache */ |
| + u64 vmacache_seqnum; /* per-thread vmacache */ |
| #ifdef CONFIG_MMU |
| unsigned long (*get_unmapped_area) (struct file *filp, |
| unsigned long addr, unsigned long len, |
| --- a/include/linux/mm_types_task.h |
| +++ b/include/linux/mm_types_task.h |
| @@ -32,7 +32,7 @@ |
| #define VMACACHE_MASK (VMACACHE_SIZE - 1) |
| |
| struct vmacache { |
| - u32 seqnum; |
| + u64 seqnum; |
| struct vm_area_struct *vmas[VMACACHE_SIZE]; |
| }; |
| |
| --- a/include/linux/vm_event_item.h |
| +++ b/include/linux/vm_event_item.h |
| @@ -105,7 +105,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PS |
| #ifdef CONFIG_DEBUG_VM_VMACACHE |
| VMACACHE_FIND_CALLS, |
| VMACACHE_FIND_HITS, |
| - VMACACHE_FULL_FLUSHES, |
| #endif |
| #ifdef CONFIG_SWAP |
| SWAP_RA, |
| --- a/include/linux/vmacache.h |
| +++ b/include/linux/vmacache.h |
| @@ -16,7 +16,6 @@ static inline void vmacache_flush(struct |
| memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas)); |
| } |
| |
| -extern void vmacache_flush_all(struct mm_struct *mm); |
| extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma); |
| extern struct vm_area_struct *vmacache_find(struct mm_struct *mm, |
| unsigned long addr); |
| @@ -30,10 +29,6 @@ extern struct vm_area_struct *vmacache_f |
| static inline void vmacache_invalidate(struct mm_struct *mm) |
| { |
| mm->vmacache_seqnum++; |
| - |
| - /* deal with overflows */ |
| - if (unlikely(mm->vmacache_seqnum == 0)) |
| - vmacache_flush_all(mm); |
| } |
| |
| #endif /* __LINUX_VMACACHE_H */ |
| --- a/mm/debug.c |
| +++ b/mm/debug.c |
| @@ -100,7 +100,7 @@ EXPORT_SYMBOL(dump_vma); |
| |
| void dump_mm(const struct mm_struct *mm) |
| { |
| - pr_emerg("mm %p mmap %p seqnum %d task_size %lu\n" |
| + pr_emerg("mm %p mmap %p seqnum %llu task_size %lu\n" |
| #ifdef CONFIG_MMU |
| "get_unmapped_area %p\n" |
| #endif |
| @@ -128,7 +128,7 @@ void dump_mm(const struct mm_struct *mm) |
| "tlb_flush_pending %d\n" |
| "def_flags: %#lx(%pGv)\n", |
| |
| - mm, mm->mmap, mm->vmacache_seqnum, mm->task_size, |
| + mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size, |
| #ifdef CONFIG_MMU |
| mm->get_unmapped_area, |
| #endif |
| --- a/mm/vmacache.c |
| +++ b/mm/vmacache.c |
| @@ -8,44 +8,6 @@ |
| #include <linux/vmacache.h> |
| |
| /* |
| - * Flush vma caches for threads that share a given mm. |
| - * |
| - * The operation is safe because the caller holds the mmap_sem |
| - * exclusively and other threads accessing the vma cache will |
| - * have mmap_sem held at least for read, so no extra locking |
| - * is required to maintain the vma cache. |
| - */ |
| -void vmacache_flush_all(struct mm_struct *mm) |
| -{ |
| - struct task_struct *g, *p; |
| - |
| - count_vm_vmacache_event(VMACACHE_FULL_FLUSHES); |
| - |
| - /* |
| - * Single threaded tasks need not iterate the entire |
| - * list of process. We can avoid the flushing as well |
| - * since the mm's seqnum was increased and don't have |
| - * to worry about other threads' seqnum. Current's |
| - * flush will occur upon the next lookup. |
| - */ |
| - if (atomic_read(&mm->mm_users) == 1) |
| - return; |
| - |
| - rcu_read_lock(); |
| - for_each_process_thread(g, p) { |
| - /* |
| - * Only flush the vmacache pointers as the |
| - * mm seqnum is already set and curr's will |
| - * be set upon invalidation when the next |
| - * lookup is done. |
| - */ |
| - if (mm == p->mm) |
| - vmacache_flush(p); |
| - } |
| - rcu_read_unlock(); |
| -} |
| - |
| -/* |
| * This task may be accessing a foreign mm via (for example) |
| * get_user_pages()->find_vma(). The vmacache is task-local and this |
| * task's vmacache pertains to a different mm (ie, its own). There is |