| From 91bfcdc157db256c0ac752cb3d51d2bb6a54d1fb Mon Sep 17 00:00:00 2001 |
| From: Paul Gortmaker <paul.gortmaker@windriver.com> |
| Date: Sun, 23 Jan 2011 19:17:40 -0500 |
| Subject: [PATCH] page_alloc: augment percpu pages support for newer kernels |
| |
| This is a follow-on to: |
| |
| mm-page_alloc-rt-friendly-per-cpu-pages.patch |
| (tip ff3fd6afd788760c846a2f4449487debb6c4b0ac) |
| |
| and: |
| |
| mm-page_alloc-reduce-lock-sections-further.patch |
| (tip 46167aec68f48cbbeff23cae9173bc4d19a7bcda) |
| |
| which is embedded in the big merge up to 33rt (5f854cfc024) |
| |
| At some point it probably makes sense to munge all three of |
| these together -- whee. |
| |
| See the constituent bits spread around in a git tip repo with: |
| |
| git diff 5f854cfc024622e4aae14d7cf422f6ff86278688^2 \ |
| 5f854cfc024622e4aae14d7cf422f6ff86278688 mm/page_alloc.c |
| |
| (the above merge changeset is a superset of this delta). |
| |
| You can find the origin of this change in the tip merge commit: |
| |
| commit 5f854cfc024622e4aae14d7cf422f6ff86278688 |
| Merge: cc24da0 4ec62b2 |
| Author: Thomas Gleixner <tglx@linutronix.de> |
| Date: Sun Feb 21 20:17:22 2010 +0100 |
| |
| Forward to 2.6.33-rc8 |
| |
| Merge branch 'linus' into rt/head with a pile of conflicts. |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| |
| Normally there are not significant changes/additions in a merge commit that |
| are not from any other "normal" commit. But in this case there are, so |
| break them out into separate explicit commits. |
| |
| Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com> |
| |
| diff --git a/mm/page_alloc.c b/mm/page_alloc.c |
| index 777ac72..ab0695e 100644 |
| --- a/mm/page_alloc.c |
| +++ b/mm/page_alloc.c |
| @@ -195,7 +195,7 @@ static inline void __lock_cpu_pcp(unsigned long *flags, int cpu) |
| { |
| #ifdef CONFIG_PREEMPT_RT |
| spin_lock(&__get_cpu_lock(pcp_locks, cpu)); |
| - flags = 0; |
| + *flags = 0; |
| #else |
| local_irq_save(*flags); |
| #endif |
| @@ -597,10 +597,9 @@ static inline int free_pages_check(struct page *page) |
| * pinned" detection logic. |
| */ |
| static void free_pcppages_bulk(struct zone *zone, int count, |
| - struct per_cpu_pages *pcp) |
| + struct per_cpu_pages *pcp) |
| { |
| int migratetype = 0; |
| - int batch_free = 0; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&zone->lock, flags); |
| @@ -608,6 +607,38 @@ static void free_pcppages_bulk(struct zone *zone, int count, |
| zone->pages_scanned = 0; |
| |
| __mod_zone_page_state(zone, NR_FREE_PAGES, count); |
| + |
| + for (migratetype =0; migratetype < MIGRATE_PCPTYPES; migratetype++) { |
| + struct list_head *list = &pcp->lists[migratetype]; |
| + |
| + while (!list_empty(list)) { |
| + struct page *page; |
| + |
| + page = list_first_entry(list, struct page, lru); |
| + /* must delete as __free_one_page list manipulates */ |
| + list_del(&page->lru); |
| + /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ |
| + __free_one_page(page, zone, 0, page_private(page)); |
| + trace_mm_page_pcpu_drain(page, 0, page_private(page)); |
| +#ifdef CONFIG_PREEMPT_RT |
| + cond_resched_lock(&zone->lock); |
| +#endif |
| + count--; |
| + } |
| + } |
| + WARN_ON(count != 0); |
| + spin_unlock_irqrestore(&zone->lock, flags); |
| +} |
| + |
| +static void isolate_pcp_pages(int count, struct per_cpu_pages *src, |
| + struct per_cpu_pages *dst) |
| +{ |
| + int migratetype, batch_free = 0; |
| + |
| + for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) |
| + INIT_LIST_HEAD(&dst->lists[migratetype]); |
| + migratetype = 0; |
| + |
| while (count) { |
| struct page *page; |
| struct list_head *list; |
| @@ -623,22 +654,16 @@ static void free_pcppages_bulk(struct zone *zone, int count, |
| batch_free++; |
| if (++migratetype == MIGRATE_PCPTYPES) |
| migratetype = 0; |
| - list = &pcp->lists[migratetype]; |
| + list = &src->lists[migratetype]; |
| } while (list_empty(list)); |
| |
| do { |
| - page = list_entry(list->prev, struct page, lru); |
| + page = list_last_entry(list, struct page, lru); |
| /* must delete as __free_one_page list manipulates */ |
| list_del(&page->lru); |
| - /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ |
| - __free_one_page(page, zone, 0, page_private(page)); |
| - trace_mm_page_pcpu_drain(page, 0, page_private(page)); |
| -#ifdef CONFIG_PREEMPT_RT |
| - cond_resched_lock(&zone->lock); |
| -#endif |
| + list_add(&page->lru, &dst->lists[migratetype]); |
| } while (--count && --batch_free && !list_empty(list)); |
| } |
| - spin_unlock_irqrestore(&zone->lock, flags); |
| } |
| |
| static void free_one_page(struct zone *zone, struct page *page, int order, |
| @@ -1047,16 +1072,6 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, |
| return i; |
| } |
| |
| -static void |
| -isolate_pcp_pages(int count, struct list_head *src, struct list_head *dst) |
| -{ |
| - while (count--) { |
| - struct page *page = list_last_entry(src, struct page, lru); |
| - list_move(&page->lru, dst); |
| - } |
| -} |
| - |
| - |
| #ifdef CONFIG_NUMA |
| /* |
| * Called from the vmstat counter updater to drain pagesets of this |
| @@ -1068,20 +1083,19 @@ isolate_pcp_pages(int count, struct list_head *src, struct list_head *dst) |
| */ |
| void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) |
| { |
| - LIST_HEAD(free_list); |
| + struct per_cpu_pages dst; |
| unsigned long flags; |
| - int to_drain; |
| - int this_cpu; |
| + int to_drain, this_cpu; |
| |
| lock_cpu_pcp(&flags, &this_cpu); |
| if (pcp->count >= pcp->batch) |
| to_drain = pcp->batch; |
| else |
| to_drain = pcp->count; |
| - isolate_pcp_pages(to_drain, &pcp->list, &free_list); |
| + isolate_pcp_pages(to_drain, pcp, &dst); |
| pcp->count -= to_drain; |
| unlock_cpu_pcp(flags, this_cpu); |
| - free_pages_bulk(zone, to_drain, &free_list, 0); |
| + free_pcppages_bulk(zone, to_drain, &dst); |
| } |
| #endif |
| |
| @@ -1099,8 +1113,7 @@ static void drain_pages(unsigned int cpu) |
| |
| for_each_populated_zone(zone) { |
| struct per_cpu_pageset *pset; |
| - struct per_cpu_pages *pcp; |
| - LIST_HEAD(free_list); |
| + struct per_cpu_pages *pcp, dst; |
| int count; |
| |
| __lock_cpu_pcp(&flags, cpu); |
| @@ -1111,12 +1124,13 @@ static void drain_pages(unsigned int cpu) |
| WARN_ON(1); |
| continue; |
| } |
| + |
| pcp = &pset->pcp; |
| - isolate_pcp_pages(pcp->count, &pcp->list, &free_list); |
| + isolate_pcp_pages(pcp->count, pcp, &dst); |
| count = pcp->count; |
| pcp->count = 0; |
| unlock_cpu_pcp(flags, cpu); |
| - free_pages_bulk(zone, count, &free_list, 0); |
| + free_pcppages_bulk(zone, count, &dst); |
| } |
| } |
| |
| @@ -1222,8 +1236,8 @@ void free_hot_cold_page(struct page *page, int cold) |
| struct per_cpu_pageset *pset; |
| struct per_cpu_pages *pcp; |
| unsigned long flags; |
| - int migratetype; |
| - int count, this_cpu, wasMlocked = __TestClearPageMlocked(page); |
| + int migratetype, this_cpu, count; |
| + int wasMlocked = __TestClearPageMlocked(page); |
| |
| trace_mm_page_free_direct(page, 0); |
| kmemcheck_free_shadow(page, 0); |
| @@ -1269,16 +1283,15 @@ void free_hot_cold_page(struct page *page, int cold) |
| list_add(&page->lru, &pcp->lists[migratetype]); |
| pcp->count++; |
| if (pcp->count >= pcp->high) { |
| - LIST_HEAD(free_list); |
| + struct per_cpu_pages dst; |
| |
| - isolate_pcp_pages(pcp->batch, &pcp->list, &free_list); |
| + isolate_pcp_pages(pcp->batch, pcp, &dst); |
| pcp->count -= pcp->batch; |
| count = pcp->batch; |
| put_zone_pcp(zone, flags, this_cpu); |
| - free_pages_bulk(zone, count, &free_list, 0); |
| + free_pcppages_bulk(zone, count, &dst); |
| return; |
| } |
| - |
| out: |
| put_zone_pcp(zone, flags, this_cpu); |
| } |
| @@ -1331,11 +1344,10 @@ again: |
| pset = get_zone_pcp(zone, &flags, &this_cpu); |
| |
| if (likely(order == 0)) { |
| - struct list_head *list; |
| struct per_cpu_pages *pcp = &pset->pcp; |
| + struct list_head *list; |
| |
| list = &pcp->lists[migratetype]; |
| - |
| if (list_empty(list)) { |
| pcp->count += rmqueue_bulk(zone, 0, |
| pcp->batch, list, |
| -- |
| 1.7.1.1 |
| |