| From d088d25ae924b8ae99f5483b61d79337065907de Mon Sep 17 00:00:00 2001 |
| From: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Date: Fri, 3 Jul 2009 08:44:43 -0500 |
| Subject: [PATCH] mm: More lock breaks in slab.c |
| |
| commit 4672c44017f4f0545122dfefd09cdd68c0c848ff in tip. |
| |
| Handle __free_pages outside of the locked regions. This reduces the |
| lock contention on the percpu slab locks in -rt significantly. |
| |
| Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| |
| diff --git a/mm/slab.c b/mm/slab.c |
| index b99df7f..4641c47 100644 |
| --- a/mm/slab.c |
| +++ b/mm/slab.c |
| @@ -180,28 +180,46 @@ static void slab_irq_disable_GFP_WAIT(gfp_t flags, int *cpu) |
| * by a lock. This keeps the code preemptable - albeit at the cost of remote |
| * memory access when the task does get migrated away. |
| */ |
| -DEFINE_PER_CPU_LOCKED(int, slab_irq_locks) = { 0, }; |
| +DEFINE_PER_CPU_LOCKED(struct list_head, slab) = { 0, }; |
| |
| static void _slab_irq_disable(int *cpu) |
| { |
| - get_cpu_var_locked(slab_irq_locks, cpu); |
| + (void)get_cpu_var_locked(slab, cpu); |
| } |
| |
| #define slab_irq_disable(cpu) _slab_irq_disable(&(cpu)) |
| |
| static inline void slab_irq_enable(int cpu) |
| { |
| - put_cpu_var_locked(slab_irq_locks, cpu); |
| + LIST_HEAD(list); |
| + |
| + list_splice_init(&__get_cpu_var_locked(slab, cpu), &list); |
| + put_cpu_var_locked(slab, cpu); |
| + |
| + while (!list_empty(&list)) { |
| + struct page *page = list_first_entry(&list, struct page, lru); |
| + list_del(&page->lru); |
| + __free_pages(page, page->index); |
| + } |
| } |
| |
| static inline void slab_irq_disable_this_rt(int cpu) |
| { |
| - spin_lock(&__get_cpu_lock(slab_irq_locks, cpu)); |
| + spin_lock(&__get_cpu_lock(slab, cpu)); |
| } |
| |
| static inline void slab_irq_enable_rt(int cpu) |
| { |
| - spin_unlock(&__get_cpu_lock(slab_irq_locks, cpu)); |
| + LIST_HEAD(list); |
| + |
| + list_splice_init(&__get_cpu_var_locked(slab, cpu), &list); |
| + spin_unlock(&__get_cpu_lock(slab, cpu)); |
| + |
| + while (!list_empty(&list)) { |
| + struct page *page = list_first_entry(&list, struct page, lru); |
| + list_del(&page->lru); |
| + __free_pages(page, page->index); |
| + } |
| } |
| |
| # define slab_irq_save(flags, cpu) \ |
| @@ -1520,6 +1538,12 @@ void __init kmem_cache_init(void) |
| int order; |
| int node; |
| |
| +#ifdef CONFIG_PREEMPT_RT |
| + for_each_possible_cpu(i) { |
| + INIT_LIST_HEAD(&__get_cpu_var_locked(slab, i)); |
| + } |
| +#endif |
| + |
| if (num_possible_nodes() == 1) |
| use_alien_caches = 0; |
| |
| @@ -1794,12 +1818,14 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) |
| /* |
| * Interface to system's page release. |
| */ |
| -static void kmem_freepages(struct kmem_cache *cachep, void *addr) |
| +static void kmem_freepages(struct kmem_cache *cachep, void *addr, int cpu) |
| { |
| unsigned long i = (1 << cachep->gfporder); |
| - struct page *page = virt_to_page(addr); |
| + struct page *page, *basepage = virt_to_page(addr); |
| const unsigned long nr_freed = i; |
| |
| + page = basepage; |
| + |
| kmemcheck_free_shadow(page, cachep->gfporder); |
| |
| if (cachep->flags & SLAB_RECLAIM_ACCOUNT) |
| @@ -1808,6 +1834,7 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr) |
| else |
| sub_zone_page_state(page_zone(page), |
| NR_SLAB_UNRECLAIMABLE, nr_freed); |
| + |
| while (i--) { |
| BUG_ON(!PageSlab(page)); |
| __ClearPageSlab(page); |
| @@ -1815,6 +1842,13 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr) |
| } |
| if (current->reclaim_state) |
| current->reclaim_state->reclaimed_slab += nr_freed; |
| + |
| +#ifdef CONFIG_PREEMPT_RT |
| + if (cpu >= 0) { |
| + basepage->index = cachep->gfporder; |
| + list_add(&basepage->lru, &__get_cpu_var_locked(slab, cpu)); |
| + } else |
| +#endif |
| free_pages((unsigned long)addr, cachep->gfporder); |
| } |
| |
| @@ -1823,7 +1857,7 @@ static void kmem_rcu_free(struct rcu_head *head) |
| struct slab_rcu *slab_rcu = (struct slab_rcu *)head; |
| struct kmem_cache *cachep = slab_rcu->cachep; |
| |
| - kmem_freepages(cachep, slab_rcu->addr); |
| + kmem_freepages(cachep, slab_rcu->addr, -1); |
| if (OFF_SLAB(cachep)) |
| kmem_cache_free(cachep->slabp_cache, slab_rcu); |
| } |
| @@ -2060,7 +2094,7 @@ slab_destroy(struct kmem_cache *cachep, struct slab *slabp, int *this_cpu) |
| slab_rcu->addr = addr; |
| call_rcu(&slab_rcu->head, kmem_rcu_free); |
| } else { |
| - kmem_freepages(cachep, addr); |
| + kmem_freepages(cachep, addr, *this_cpu); |
| if (OFF_SLAB(cachep)) { |
| if (this_cpu) |
| __cache_free(cachep->slabp_cache, slabp, this_cpu); |
| @@ -2598,9 +2632,9 @@ slab_on_each_cpu(void (*func)(void *arg, int this_cpu), void *arg) |
| |
| check_irq_on(); |
| for_each_online_cpu(i) { |
| - spin_lock(&__get_cpu_lock(slab_irq_locks, i)); |
| + spin_lock(&__get_cpu_lock(slab, i)); |
| func(arg, i); |
| - spin_unlock(&__get_cpu_lock(slab_irq_locks, i)); |
| + spin_unlock(&__get_cpu_lock(slab, i)); |
| } |
| } |
| #else |
| @@ -2991,7 +3025,7 @@ static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid, |
| spin_unlock(&l3->list_lock); |
| return 1; |
| opps1: |
| - kmem_freepages(cachep, objp); |
| + kmem_freepages(cachep, objp, -1); |
| failed: |
| slab_irq_disable_GFP_WAIT(local_flags, this_cpu); |
| return 0; |
| -- |
| 1.7.1.1 |
| |