| From ad867acb5f23f05b8eea1bf89ce7e42f6e9bf8d1 Mon Sep 17 00:00:00 2001 |
| From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| Date: Thu, 29 Jan 2015 17:19:44 +0100 |
| Subject: [PATCH] mm/workingset: Do not protect workingset_shadow_nodes with |
| irq off |
| |
| workingset_shadow_nodes is protected by local_irq_disable(). Some users |
| use spin_lock_irq(). |
| Replace the irq/on with a local_lock(). Rename workingset_shadow_nodes |
| so I catch users of it which will be introduced later. |
| |
| Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| |
| diff --git a/include/linux/swap.h b/include/linux/swap.h |
| index ba5882419a7d..bcdefa817ed5 100644 |
| --- a/include/linux/swap.h |
| +++ b/include/linux/swap.h |
| @@ -11,6 +11,7 @@ |
| #include <linux/fs.h> |
| #include <linux/atomic.h> |
| #include <linux/page-flags.h> |
| +#include <linux/locallock.h> |
| #include <asm/page.h> |
| |
| struct notifier_block; |
| @@ -254,7 +255,8 @@ struct swap_info_struct { |
| void *workingset_eviction(struct address_space *mapping, struct page *page); |
| bool workingset_refault(void *shadow); |
| void workingset_activation(struct page *page); |
| -void workingset_update_node(struct radix_tree_node *node, void *private); |
| +void __workingset_update_node(struct radix_tree_node *node, void *private); |
| +DECLARE_LOCAL_IRQ_LOCK(shadow_nodes_lock); |
| |
| /* linux/mm/page_alloc.c */ |
| extern unsigned long totalram_pages; |
| diff --git a/mm/filemap.c b/mm/filemap.c |
| index 681da61080bc..3d9cd0e58578 100644 |
| --- a/mm/filemap.c |
| +++ b/mm/filemap.c |
| @@ -110,6 +110,7 @@ |
| * ->i_mmap_rwsem |
| * ->tasklist_lock (memory_failure, collect_procs_ao) |
| */ |
| +DECLARE_LOCAL_IRQ_LOCK(shadow_nodes_lock); |
| |
| static int page_cache_tree_insert(struct address_space *mapping, |
| struct page *page, void **shadowp) |
| @@ -142,8 +143,10 @@ static int page_cache_tree_insert(struct address_space *mapping, |
| true); |
| } |
| } |
| + local_lock(shadow_nodes_lock); |
| __radix_tree_replace(&mapping->page_tree, node, slot, page, |
| - workingset_update_node, mapping); |
| + __workingset_update_node, mapping); |
| + local_unlock(shadow_nodes_lock); |
| mapping->nrpages++; |
| return 0; |
| } |
| @@ -160,6 +163,7 @@ static void page_cache_tree_delete(struct address_space *mapping, |
| VM_BUG_ON_PAGE(PageTail(page), page); |
| VM_BUG_ON_PAGE(nr != 1 && shadow, page); |
| |
| + local_lock(shadow_nodes_lock); |
| for (i = 0; i < nr; i++) { |
| struct radix_tree_node *node; |
| void **slot; |
| @@ -171,8 +175,9 @@ static void page_cache_tree_delete(struct address_space *mapping, |
| |
| radix_tree_clear_tags(&mapping->page_tree, node, slot); |
| __radix_tree_replace(&mapping->page_tree, node, slot, shadow, |
| - workingset_update_node, mapping); |
| + __workingset_update_node, mapping); |
| } |
| + local_unlock(shadow_nodes_lock); |
| |
| if (shadow) { |
| mapping->nrexceptional += nr; |
| diff --git a/mm/truncate.c b/mm/truncate.c |
| index 83a059e8cd1d..47ef312bbbee 100644 |
| --- a/mm/truncate.c |
| +++ b/mm/truncate.c |
| @@ -41,8 +41,10 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, |
| goto unlock; |
| if (*slot != entry) |
| goto unlock; |
| + local_lock(shadow_nodes_lock); |
| __radix_tree_replace(&mapping->page_tree, node, slot, NULL, |
| - workingset_update_node, mapping); |
| + __workingset_update_node, mapping); |
| + local_unlock(shadow_nodes_lock); |
| mapping->nrexceptional--; |
| unlock: |
| spin_unlock_irq(&mapping->tree_lock); |
| diff --git a/mm/workingset.c b/mm/workingset.c |
| index b8c9ab678479..5fa253835c59 100644 |
| --- a/mm/workingset.c |
| +++ b/mm/workingset.c |
| @@ -339,9 +339,10 @@ void workingset_activation(struct page *page) |
| * point where they would still be useful. |
| */ |
| |
| -static struct list_lru shadow_nodes; |
| +static struct list_lru __shadow_nodes; |
| +DEFINE_LOCAL_IRQ_LOCK(shadow_nodes_lock); |
| |
| -void workingset_update_node(struct radix_tree_node *node, void *private) |
| +void __workingset_update_node(struct radix_tree_node *node, void *private) |
| { |
| struct address_space *mapping = private; |
| |
| @@ -359,10 +360,10 @@ void workingset_update_node(struct radix_tree_node *node, void *private) |
| */ |
| if (node->count && node->count == node->exceptional) { |
| if (list_empty(&node->private_list)) |
| - list_lru_add(&shadow_nodes, &node->private_list); |
| + list_lru_add(&__shadow_nodes, &node->private_list); |
| } else { |
| if (!list_empty(&node->private_list)) |
| - list_lru_del(&shadow_nodes, &node->private_list); |
| + list_lru_del(&__shadow_nodes, &node->private_list); |
| } |
| } |
| |
| @@ -374,9 +375,9 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, |
| unsigned long cache; |
| |
| /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ |
| - local_irq_disable(); |
| - nodes = list_lru_shrink_count(&shadow_nodes, sc); |
| - local_irq_enable(); |
| + local_lock_irq(shadow_nodes_lock); |
| + nodes = list_lru_shrink_count(&__shadow_nodes, sc); |
| + local_unlock_irq(shadow_nodes_lock); |
| |
| /* |
| * Approximate a reasonable limit for the radix tree nodes |
| @@ -477,15 +478,15 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, |
| inc_node_state(page_pgdat(virt_to_page(node)), WORKINGSET_NODERECLAIM); |
| inc_memcg_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); |
| __radix_tree_delete_node(&mapping->page_tree, node, |
| - workingset_update_node, mapping); |
| + __workingset_update_node, mapping); |
| |
| out_invalid: |
| spin_unlock(&mapping->tree_lock); |
| ret = LRU_REMOVED_RETRY; |
| out: |
| - local_irq_enable(); |
| + local_unlock_irq(shadow_nodes_lock); |
| cond_resched(); |
| - local_irq_disable(); |
| + local_lock_irq(shadow_nodes_lock); |
| spin_lock(lru_lock); |
| return ret; |
| } |
| @@ -496,9 +497,9 @@ static unsigned long scan_shadow_nodes(struct shrinker *shrinker, |
| unsigned long ret; |
| |
| /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ |
| - local_irq_disable(); |
| - ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); |
| - local_irq_enable(); |
| + local_lock_irq(shadow_nodes_lock); |
| + ret = list_lru_shrink_walk(&__shadow_nodes, sc, shadow_lru_isolate, NULL); |
| + local_unlock_irq(shadow_nodes_lock); |
| return ret; |
| } |
| |
| @@ -536,7 +537,7 @@ static int __init workingset_init(void) |
| pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n", |
| timestamp_bits, max_order, bucket_order); |
| |
| - ret = __list_lru_init(&shadow_nodes, true, &shadow_nodes_key); |
| + ret = __list_lru_init(&__shadow_nodes, true, &shadow_nodes_key); |
| if (ret) |
| goto err; |
| ret = register_shrinker(&workingset_shadow_shrinker); |
| @@ -544,7 +545,7 @@ static int __init workingset_init(void) |
| goto err_list_lru; |
| return 0; |
| err_list_lru: |
| - list_lru_destroy(&shadow_nodes); |
| + list_lru_destroy(&__shadow_nodes); |
| err: |
| return ret; |
| } |
| -- |
| 2.1.4 |
| |