| From dhobsong@igel.co.jp Mon Oct 29 00:52:00 2012 |
| From: Damian Hobson-Garcia <dhobsong@igel.co.jp> |
| Date: Mon, 29 Oct 2012 16:50:23 +0900 |
| Subject: [PATCH v2 06/58] mm: page_alloc: introduce alloc_contig_range() |
| To: greg@kroah.com, laurent.pinchart@ideasonboard.com, horms@verge.net.au |
| Cc: ltsi-dev@lists.linuxfoundation.org, dhobsong@igel.co.jp |
| Message-ID: <1351497075-32717-7-git-send-email-dhobsong@igel.co.jp> |
| |
| |
| From: Michal Nazarewicz <mina86@mina86.com> |
| |
| This commit adds the alloc_contig_range() function which tries |
| to allocate given range of pages. It tries to migrate all |
| already allocated pages that fall in the range thus freeing them. |
| Once all pages in the range are freed they are removed from the |
| buddy system thus allocated for the caller to use. |
| |
| Signed-off-by: Michal Nazarewicz <mina86@mina86.com> |
| Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com> |
| Acked-by: Mel Gorman <mel@csn.ul.ie> |
| Reviewed-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> |
| Tested-by: Rob Clark <rob.clark@linaro.org> |
| Tested-by: Ohad Ben-Cohen <ohad@wizery.com> |
| Tested-by: Benjamin Gaignard <benjamin.gaignard@linaro.org> |
| Tested-by: Robert Nelson <robertcnelson@gmail.com> |
| Tested-by: Barry Song <Baohua.Song@csr.com> |
| (cherry picked from commit 041d3a8cdc18dc375a128d90bbb753949a81b1fb) |
| |
| Signed-off-by: Damian Hobson-Garcia <dhobsong@igel.co.jp> |
| Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> |
| --- |
| include/linux/gfp.h | 8 ++ |
| mm/page_alloc.c | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| 2 files changed, 196 insertions(+) |
| |
| --- a/include/linux/gfp.h |
| +++ b/include/linux/gfp.h |
| @@ -391,4 +391,12 @@ static inline bool pm_suspended_storage( |
| } |
| #endif /* CONFIG_PM_SLEEP */ |
| |
| +#ifdef CONFIG_CMA |
| + |
| +/* The below functions must be run on a range from a single zone. */ |
| +extern int alloc_contig_range(unsigned long start, unsigned long end); |
| +extern void free_contig_range(unsigned long pfn, unsigned nr_pages); |
| + |
| +#endif |
| + |
| #endif /* __LINUX_GFP_H */ |
| --- a/mm/page_alloc.c |
| +++ b/mm/page_alloc.c |
| @@ -57,6 +57,7 @@ |
| #include <linux/ftrace_event.h> |
| #include <linux/memcontrol.h> |
| #include <linux/prefetch.h> |
| +#include <linux/migrate.h> |
| #include <linux/page-debug-flags.h> |
| |
| #include <asm/tlbflush.h> |
| @@ -5557,6 +5558,193 @@ out: |
| spin_unlock_irqrestore(&zone->lock, flags); |
| } |
| |
| +#ifdef CONFIG_CMA |
| + |
| +static unsigned long pfn_max_align_down(unsigned long pfn) |
| +{ |
| + return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, |
| + pageblock_nr_pages) - 1); |
| +} |
| + |
| +static unsigned long pfn_max_align_up(unsigned long pfn) |
| +{ |
| + return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, |
| + pageblock_nr_pages)); |
| +} |
| + |
| +static struct page * |
| +__alloc_contig_migrate_alloc(struct page *page, unsigned long private, |
| + int **resultp) |
| +{ |
| + return alloc_page(GFP_HIGHUSER_MOVABLE); |
| +} |
| + |
| +/* [start, end) must belong to a single zone. */ |
| +static int __alloc_contig_migrate_range(unsigned long start, unsigned long end) |
| +{ |
| + /* This function is based on compact_zone() from compaction.c. */ |
| + |
| + unsigned long pfn = start; |
| + unsigned int tries = 0; |
| + int ret = 0; |
| + |
| + struct compact_control cc = { |
| + .nr_migratepages = 0, |
| + .order = -1, |
| + .zone = page_zone(pfn_to_page(start)), |
| + .sync = true, |
| + }; |
| + INIT_LIST_HEAD(&cc.migratepages); |
| + |
| + migrate_prep_local(); |
| + |
| + while (pfn < end || !list_empty(&cc.migratepages)) { |
| + if (fatal_signal_pending(current)) { |
| + ret = -EINTR; |
| + break; |
| + } |
| + |
| + if (list_empty(&cc.migratepages)) { |
| + cc.nr_migratepages = 0; |
| + pfn = isolate_migratepages_range(cc.zone, &cc, |
| + pfn, end); |
| + if (!pfn) { |
| + ret = -EINTR; |
| + break; |
| + } |
| + tries = 0; |
| + } else if (++tries == 5) { |
| + ret = ret < 0 ? ret : -EBUSY; |
| + break; |
| + } |
| + |
| + ret = migrate_pages(&cc.migratepages, |
| + __alloc_contig_migrate_alloc, |
| + 0, false, true); |
| + } |
| + |
| + putback_lru_pages(&cc.migratepages); |
| + return ret > 0 ? 0 : ret; |
| +} |
| + |
| +/** |
| + * alloc_contig_range() -- tries to allocate given range of pages |
| + * @start: start PFN to allocate |
| + * @end: one-past-the-last PFN to allocate |
| + * |
| + * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES |
| + * aligned, however it's the caller's responsibility to guarantee that |
| + * we are the only thread that changes migrate type of pageblocks the |
| + * pages fall in. |
| + * |
| + * The PFN range must belong to a single zone. |
| + * |
| + * Returns zero on success or negative error code. On success all |
| + * pages which PFN is in [start, end) are allocated for the caller and |
| + * need to be freed with free_contig_range(). |
| + */ |
| +int alloc_contig_range(unsigned long start, unsigned long end) |
| +{ |
| + struct zone *zone = page_zone(pfn_to_page(start)); |
| + unsigned long outer_start, outer_end; |
| + int ret = 0, order; |
| + |
| + /* |
| + * What we do here is we mark all pageblocks in range as |
| + * MIGRATE_ISOLATE. Because pageblock and max order pages may |
| + * have different sizes, and due to the way page allocator |
| + * work, we align the range to biggest of the two pages so |
| + * that page allocator won't try to merge buddies from |
| + * different pageblocks and change MIGRATE_ISOLATE to some |
| + * other migration type. |
| + * |
| + * Once the pageblocks are marked as MIGRATE_ISOLATE, we |
| + * migrate the pages from an unaligned range (ie. pages that |
| + * we are interested in). This will put all the pages in |
| + * range back to page allocator as MIGRATE_ISOLATE. |
| + * |
| + * When this is done, we take the pages in range from page |
| + * allocator removing them from the buddy system. This way |
| + * page allocator will never consider using them. |
| + * |
| + * This lets us mark the pageblocks back as |
| + * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the |
| + * aligned range but not in the unaligned, original range are |
| + * put back to page allocator so that buddy can use them. |
| + */ |
| + |
| + ret = start_isolate_page_range(pfn_max_align_down(start), |
| + pfn_max_align_up(end)); |
| + if (ret) |
| + goto done; |
| + |
| + ret = __alloc_contig_migrate_range(start, end); |
| + if (ret) |
| + goto done; |
| + |
| + /* |
| + * Pages from [start, end) are within a MAX_ORDER_NR_PAGES |
| + * aligned blocks that are marked as MIGRATE_ISOLATE. What's |
| + * more, all pages in [start, end) are free in page allocator. |
| + * What we are going to do is to allocate all pages from |
| + * [start, end) (that is remove them from page allocator). |
| + * |
| + * The only problem is that pages at the beginning and at the |
| + * end of interesting range may be not aligned with pages that |
| + * page allocator holds, ie. they can be part of higher order |
| + * pages. Because of this, we reserve the bigger range and |
| + * once this is done free the pages we are not interested in. |
| + * |
| + * We don't have to hold zone->lock here because the pages are |
| + * isolated thus they won't get removed from buddy. |
| + */ |
| + |
| + lru_add_drain_all(); |
| + drain_all_pages(); |
| + |
| + order = 0; |
| + outer_start = start; |
| + while (!PageBuddy(pfn_to_page(outer_start))) { |
| + if (++order >= MAX_ORDER) { |
| + ret = -EBUSY; |
| + goto done; |
| + } |
| + outer_start &= ~0UL << order; |
| + } |
| + |
| + /* Make sure the range is really isolated. */ |
| + if (test_pages_isolated(outer_start, end)) { |
| + pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n", |
| + outer_start, end); |
| + ret = -EBUSY; |
| + goto done; |
| + } |
| + |
| + outer_end = isolate_freepages_range(outer_start, end); |
| + if (!outer_end) { |
| + ret = -EBUSY; |
| + goto done; |
| + } |
| + |
| + /* Free head and tail (if any) */ |
| + if (start != outer_start) |
| + free_contig_range(outer_start, start - outer_start); |
| + if (end != outer_end) |
| + free_contig_range(end, outer_end - end); |
| + |
| +done: |
| + undo_isolate_page_range(pfn_max_align_down(start), |
| + pfn_max_align_up(end)); |
| + return ret; |
| +} |
| + |
| +void free_contig_range(unsigned long pfn, unsigned nr_pages) |
| +{ |
| + for (; nr_pages--; ++pfn) |
| + __free_page(pfn_to_page(pfn)); |
| +} |
| +#endif |
| + |
| #ifdef CONFIG_MEMORY_HOTREMOVE |
| /* |
| * All pages in the range must be isolated before calling this. |