| From c81c8a1eeede61e92a15103748c23d100880cc8a Mon Sep 17 00:00:00 2001 |
| From: Roland Dreier <roland@purestorage.com> |
| Date: Fri, 2 May 2014 11:18:41 -0700 |
| Subject: x86, ioremap: Speed up check for RAM pages |
| |
| From: Roland Dreier <roland@purestorage.com> |
| |
| commit c81c8a1eeede61e92a15103748c23d100880cc8a upstream. |
| |
| In __ioremap_caller() (the guts of ioremap), we loop over the range of |
| pfns being remapped and checks each one individually with page_is_ram(). |
| For large ioremaps, this can be very slow. For example, we have a |
| device with a 256 GiB PCI BAR, and ioremapping this BAR can take 20+ |
| seconds -- sometimes long enough to trigger the soft lockup detector! |
| |
| Internally, page_is_ram() calls walk_system_ram_range() on a single |
| page. Instead, we can make a single call to walk_system_ram_range() |
| from __ioremap_caller(), and do our further checks only for any RAM |
| pages that we find. For the common case of MMIO, this saves an enormous |
| amount of work, since the range being ioremapped doesn't intersect |
| system RAM at all. |
| |
| With this change, ioremap on our 256 GiB BAR takes less than 1 second. |
| |
| Signed-off-by: Roland Dreier <roland@purestorage.com> |
| Link: http://lkml.kernel.org/r/1399054721-1331-1-git-send-email-roland@kernel.org |
| Signed-off-by: H. Peter Anvin <hpa@linux.intel.com> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/x86/mm/ioremap.c | 26 +++++++++++++++++++------- |
| 1 file changed, 19 insertions(+), 7 deletions(-) |
| |
| --- a/arch/x86/mm/ioremap.c |
| +++ b/arch/x86/mm/ioremap.c |
| @@ -50,6 +50,21 @@ int ioremap_change_attr(unsigned long va |
| return err; |
| } |
| |
| +static int __ioremap_check_ram(unsigned long start_pfn, unsigned long nr_pages, |
| + void *arg) |
| +{ |
| + unsigned long i; |
| + |
| + for (i = 0; i < nr_pages; ++i) |
| + if (pfn_valid(start_pfn + i) && |
| + !PageReserved(pfn_to_page(start_pfn + i))) |
| + return 1; |
| + |
| + WARN_ONCE(1, "ioremap on RAM pfn 0x%lx\n", start_pfn); |
| + |
| + return 0; |
| +} |
| + |
| /* |
| * Remap an arbitrary physical address space into the kernel virtual |
| * address space. Needed when the kernel wants to access high addresses |
| @@ -93,14 +108,11 @@ static void __iomem *__ioremap_caller(re |
| /* |
| * Don't allow anybody to remap normal RAM that we're using.. |
| */ |
| + pfn = phys_addr >> PAGE_SHIFT; |
| last_pfn = last_addr >> PAGE_SHIFT; |
| - for (pfn = phys_addr >> PAGE_SHIFT; pfn <= last_pfn; pfn++) { |
| - int is_ram = page_is_ram(pfn); |
| - |
| - if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn))) |
| - return NULL; |
| - WARN_ON_ONCE(is_ram); |
| - } |
| + if (walk_system_ram_range(pfn, last_pfn - pfn + 1, NULL, |
| + __ioremap_check_ram) == 1) |
| + return NULL; |
| |
| /* |
| * Mappings have to be page-aligned |