| From 9289fc10a9175dda1affb671347aeb0491047a63 Mon Sep 17 00:00:00 2001 |
| From: Ingo Molnar <mingo@elte.hu> |
| Date: Fri, 3 Jul 2009 08:29:37 -0500 |
| Subject: [PATCH 093/274] mm: page_alloc: rt-friendly per-cpu pages |
| |
| rt-friendly per-cpu pages: convert the irqs-off per-cpu locking |
| method into a preemptible, explicit-per-cpu-locks method. |
| |
| Contains fixes from: |
| Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Thomas Gleixner <tglx@linutronix.de> |
| |
| Signed-off-by: Ingo Molnar <mingo@elte.hu> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| mm/page_alloc.c | 53 +++++++++++++++++++++++++++++++++++++---------------- |
| 1 file changed, 37 insertions(+), 16 deletions(-) |
| |
| diff --git a/mm/page_alloc.c b/mm/page_alloc.c |
| index bace229..b0f5314 100644 |
| --- a/mm/page_alloc.c |
| +++ b/mm/page_alloc.c |
| @@ -57,6 +57,7 @@ |
| #include <linux/ftrace_event.h> |
| #include <linux/memcontrol.h> |
| #include <linux/prefetch.h> |
| +#include <linux/locallock.h> |
| #include <linux/page-debug-flags.h> |
| |
| #include <asm/tlbflush.h> |
| @@ -216,6 +217,18 @@ EXPORT_SYMBOL(nr_node_ids); |
| EXPORT_SYMBOL(nr_online_nodes); |
| #endif |
| |
| +static DEFINE_LOCAL_IRQ_LOCK(pa_lock); |
| + |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| +# define cpu_lock_irqsave(cpu, flags) \ |
| + spin_lock_irqsave(&per_cpu(pa_lock, cpu).lock, flags) |
| +# define cpu_unlock_irqrestore(cpu, flags) \ |
| + spin_unlock_irqrestore(&per_cpu(pa_lock, cpu).lock, flags) |
| +#else |
| +# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags) |
| +# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags) |
| +#endif |
| + |
| int page_group_by_mobility_disabled __read_mostly; |
| |
| static void set_pageblock_migratetype(struct page *page, int migratetype) |
| @@ -721,13 +734,13 @@ static void __free_pages_ok(struct page *page, unsigned int order) |
| if (!free_pages_prepare(page, order)) |
| return; |
| |
| - local_irq_save(flags); |
| + local_lock_irqsave(pa_lock, flags); |
| if (unlikely(wasMlocked)) |
| free_page_mlock(page); |
| __count_vm_events(PGFREE, 1 << order); |
| free_one_page(page_zone(page), page, order, |
| get_pageblock_migratetype(page)); |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pa_lock, flags); |
| } |
| |
| void __meminit __free_pages_bootmem(struct page *page, unsigned int order) |
| @@ -1113,14 +1126,14 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) |
| unsigned long flags; |
| int to_drain; |
| |
| - local_irq_save(flags); |
| + local_lock_irqsave(pa_lock, flags); |
| if (pcp->count >= pcp->batch) |
| to_drain = pcp->batch; |
| else |
| to_drain = pcp->count; |
| free_pcppages_bulk(zone, to_drain, pcp); |
| pcp->count -= to_drain; |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pa_lock, flags); |
| } |
| #endif |
| |
| @@ -1140,7 +1153,7 @@ static void drain_pages(unsigned int cpu) |
| struct per_cpu_pageset *pset; |
| struct per_cpu_pages *pcp; |
| |
| - local_irq_save(flags); |
| + cpu_lock_irqsave(cpu, flags); |
| pset = per_cpu_ptr(zone->pageset, cpu); |
| |
| pcp = &pset->pcp; |
| @@ -1148,7 +1161,7 @@ static void drain_pages(unsigned int cpu) |
| free_pcppages_bulk(zone, pcp->count, pcp); |
| pcp->count = 0; |
| } |
| - local_irq_restore(flags); |
| + cpu_unlock_irqrestore(cpu, flags); |
| } |
| } |
| |
| @@ -1201,7 +1214,12 @@ void drain_all_pages(void) |
| else |
| cpumask_clear_cpu(cpu, &cpus_with_pcps); |
| } |
| +#ifndef CONFIG_PREEMPT_RT_BASE |
| on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1); |
| +#else |
| + for_each_cpu(cpu, &cpus_with_pcps) |
| + drain_pages(cpu); |
| +#endif |
| } |
| |
| #ifdef CONFIG_HIBERNATION |
| @@ -1257,7 +1275,7 @@ void free_hot_cold_page(struct page *page, int cold) |
| |
| migratetype = get_pageblock_migratetype(page); |
| set_page_private(page, migratetype); |
| - local_irq_save(flags); |
| + local_lock_irqsave(pa_lock, flags); |
| if (unlikely(wasMlocked)) |
| free_page_mlock(page); |
| __count_vm_event(PGFREE); |
| @@ -1289,7 +1307,7 @@ void free_hot_cold_page(struct page *page, int cold) |
| } |
| |
| out: |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pa_lock, flags); |
| } |
| |
| /* |
| @@ -1397,7 +1415,7 @@ again: |
| struct per_cpu_pages *pcp; |
| struct list_head *list; |
| |
| - local_irq_save(flags); |
| + local_lock_irqsave(pa_lock, flags); |
| pcp = &this_cpu_ptr(zone->pageset)->pcp; |
| list = &pcp->lists[migratetype]; |
| if (list_empty(list)) { |
| @@ -1429,17 +1447,19 @@ again: |
| */ |
| WARN_ON_ONCE(order > 1); |
| } |
| - spin_lock_irqsave(&zone->lock, flags); |
| + local_spin_lock_irqsave(pa_lock, &zone->lock, flags); |
| page = __rmqueue(zone, order, migratetype); |
| - spin_unlock(&zone->lock); |
| - if (!page) |
| + if (!page) { |
| + spin_unlock(&zone->lock); |
| goto failed; |
| + } |
| __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); |
| + spin_unlock(&zone->lock); |
| } |
| |
| __count_zone_vm_events(PGALLOC, zone, 1 << order); |
| zone_statistics(preferred_zone, zone, gfp_flags); |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pa_lock, flags); |
| |
| VM_BUG_ON(bad_range(zone, page)); |
| if (prep_new_page(page, order, gfp_flags)) |
| @@ -1447,7 +1467,7 @@ again: |
| return page; |
| |
| failed: |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pa_lock, flags); |
| return NULL; |
| } |
| |
| @@ -3858,10 +3878,10 @@ static int __zone_pcp_update(void *data) |
| pset = per_cpu_ptr(zone->pageset, cpu); |
| pcp = &pset->pcp; |
| |
| - local_irq_save(flags); |
| + cpu_lock_irqsave(cpu, flags); |
| free_pcppages_bulk(zone, pcp->count, pcp); |
| setup_pageset(pset, batch); |
| - local_irq_restore(flags); |
| + cpu_unlock_irqrestore(cpu, flags); |
| } |
| return 0; |
| } |
| @@ -4892,6 +4912,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self, |
| void __init page_alloc_init(void) |
| { |
| hotcpu_notifier(page_alloc_cpu_notify, 0); |
| + local_irq_lock_init(pa_lock); |
| } |
| |
| /* |
| -- |
| 1.7.10.4 |
| |