| From ba925fa35057a062ac98c3e8138b013ce4ce351c Mon Sep 17 00:00:00 2001 |
| From: Gerald Schaefer <gerald.schaefer@linux.ibm.com> |
| Date: Wed, 29 Jul 2020 22:22:34 +0200 |
| Subject: s390/gmap: improve THP splitting |
| |
| From: Gerald Schaefer <gerald.schaefer@linux.ibm.com> |
| |
| commit ba925fa35057a062ac98c3e8138b013ce4ce351c upstream. |
| |
| During s390_enable_sie(), we need to take care of splitting all qemu user |
| process THP mappings. This is currently done with follow_page(FOLL_SPLIT), |
| by simply iterating over all vma ranges, with PAGE_SIZE increment. |
| |
| This logic is sub-optimal and can result in a lot of unnecessary overhead, |
| especially when using qemu and ASAN with large shadow map. Ilya reported |
| significant system slow-down with one CPU busy for a long time and overall |
| unresponsiveness. |
| |
| Fix this by using walk_page_vma() and directly calling split_huge_pmd() |
| only for present pmds, which greatly reduces overhead. |
| |
| Cc: <stable@vger.kernel.org> # v5.4+ |
| Reported-by: Ilya Leoshkevich <iii@linux.ibm.com> |
| Tested-by: Ilya Leoshkevich <iii@linux.ibm.com> |
| Acked-by: Christian Borntraeger <borntraeger@de.ibm.com> |
| Signed-off-by: Gerald Schaefer <gerald.schaefer@linux.ibm.com> |
| Signed-off-by: Heiko Carstens <hca@linux.ibm.com> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/s390/mm/gmap.c | 27 ++++++++++++++++++++------- |
| 1 file changed, 20 insertions(+), 7 deletions(-) |
| |
| --- a/arch/s390/mm/gmap.c |
| +++ b/arch/s390/mm/gmap.c |
| @@ -2485,23 +2485,36 @@ void gmap_sync_dirty_log_pmd(struct gmap |
| } |
| EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd); |
| |
| +#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| +static int thp_split_walk_pmd_entry(pmd_t *pmd, unsigned long addr, |
| + unsigned long end, struct mm_walk *walk) |
| +{ |
| + struct vm_area_struct *vma = walk->vma; |
| + |
| + split_huge_pmd(vma, pmd, addr); |
| + return 0; |
| +} |
| + |
| +static const struct mm_walk_ops thp_split_walk_ops = { |
| + .pmd_entry = thp_split_walk_pmd_entry, |
| +}; |
| + |
| static inline void thp_split_mm(struct mm_struct *mm) |
| { |
| -#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| struct vm_area_struct *vma; |
| - unsigned long addr; |
| |
| for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { |
| - for (addr = vma->vm_start; |
| - addr < vma->vm_end; |
| - addr += PAGE_SIZE) |
| - follow_page(vma, addr, FOLL_SPLIT); |
| vma->vm_flags &= ~VM_HUGEPAGE; |
| vma->vm_flags |= VM_NOHUGEPAGE; |
| + walk_page_vma(vma, &thp_split_walk_ops, NULL); |
| } |
| mm->def_flags |= VM_NOHUGEPAGE; |
| -#endif |
| } |
| +#else |
| +static inline void thp_split_mm(struct mm_struct *mm) |
| +{ |
| +} |
| +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| |
| /* |
| * Remove all empty zero pages from the mapping for lazy refaulting |