| From 8d0f7655d84eaf4a00bb4344e2d2e50d911f946e Mon Sep 17 00:00:00 2001 |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Mon, 6 Jun 2011 12:20:33 +0200 |
| Subject: [PATCH] sched: Move mmdrop to RCU on RT |
| |
| Takes sleeping locks and calls into the memory allocator, so nothing |
| we want to do in task switch and oder atomic contexts. |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| |
| diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h |
| index ab8f7e11c160..75f8b460bf37 100644 |
| --- a/include/linux/mm_types.h |
| +++ b/include/linux/mm_types.h |
| @@ -11,6 +11,7 @@ |
| #include <linux/completion.h> |
| #include <linux/cpumask.h> |
| #include <linux/uprobes.h> |
| +#include <linux/rcupdate.h> |
| #include <linux/page-flags-layout.h> |
| #include <linux/workqueue.h> |
| |
| @@ -500,6 +501,9 @@ struct mm_struct { |
| bool tlb_flush_batched; |
| #endif |
| struct uprobes_state uprobes_state; |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| + struct rcu_head delayed_drop; |
| +#endif |
| #ifdef CONFIG_HUGETLB_PAGE |
| atomic_long_t hugetlb_usage; |
| #endif |
| diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h |
| index 2b24a6974847..0ec83019dfdf 100644 |
| --- a/include/linux/sched/mm.h |
| +++ b/include/linux/sched/mm.h |
| @@ -42,6 +42,17 @@ static inline void mmdrop(struct mm_struct *mm) |
| __mmdrop(mm); |
| } |
| |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| +extern void __mmdrop_delayed(struct rcu_head *rhp); |
| +static inline void mmdrop_delayed(struct mm_struct *mm) |
| +{ |
| + if (atomic_dec_and_test(&mm->mm_count)) |
| + call_rcu(&mm->delayed_drop, __mmdrop_delayed); |
| +} |
| +#else |
| +# define mmdrop_delayed(mm) mmdrop(mm) |
| +#endif |
| + |
| static inline void mmdrop_async_fn(struct work_struct *work) |
| { |
| struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work); |
| diff --git a/kernel/fork.c b/kernel/fork.c |
| index f14cc3f46bcf..e019d3b4d6b4 100644 |
| --- a/kernel/fork.c |
| +++ b/kernel/fork.c |
| @@ -903,6 +903,19 @@ void __mmdrop(struct mm_struct *mm) |
| } |
| EXPORT_SYMBOL_GPL(__mmdrop); |
| |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| +/* |
| + * RCU callback for delayed mm drop. Not strictly rcu, but we don't |
| + * want another facility to make this work. |
| + */ |
| +void __mmdrop_delayed(struct rcu_head *rhp) |
| +{ |
| + struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop); |
| + |
| + __mmdrop(mm); |
| +} |
| +#endif |
| + |
| static inline void __mmput(struct mm_struct *mm) |
| { |
| VM_BUG_ON(atomic_read(&mm->mm_users)); |
| diff --git a/kernel/sched/core.c b/kernel/sched/core.c |
| index 4f25dfd4b15b..7b5ea59aed4b 100644 |
| --- a/kernel/sched/core.c |
| +++ b/kernel/sched/core.c |
| @@ -2785,8 +2785,12 @@ static struct rq *finish_task_switch(struct task_struct *prev) |
| finish_arch_post_lock_switch(); |
| |
| fire_sched_in_preempt_notifiers(current); |
| + /* |
| + * We use mmdrop_delayed() here so we don't have to do the |
| + * full __mmdrop() when we are the last user. |
| + */ |
| if (mm) |
| - mmdrop(mm); |
| + mmdrop_delayed(mm); |
| if (unlikely(prev_state == TASK_DEAD)) { |
| if (prev->sched_class->task_dead) |
| prev->sched_class->task_dead(prev); |
| @@ -5641,6 +5645,8 @@ void sched_setnuma(struct task_struct *p, int nid) |
| #endif /* CONFIG_NUMA_BALANCING */ |
| |
| #ifdef CONFIG_HOTPLUG_CPU |
| +static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm); |
| + |
| /* |
| * Ensure that the idle task is using init_mm right before its CPU goes |
| * offline. |
| @@ -5655,7 +5661,12 @@ void idle_task_exit(void) |
| switch_mm(mm, &init_mm, current); |
| finish_arch_post_lock_switch(); |
| } |
| - mmdrop(mm); |
| + /* |
| + * Defer the cleanup to an alive cpu. On RT we can neither |
| + * call mmdrop() nor mmdrop_delayed() from here. |
| + */ |
| + per_cpu(idle_last_mm, smp_processor_id()) = mm; |
| + |
| } |
| |
| /* |
| @@ -5980,6 +5991,10 @@ int sched_cpu_dying(unsigned int cpu) |
| update_max_interval(); |
| nohz_balance_exit_idle(cpu); |
| hrtick_clear(rq); |
| + if (per_cpu(idle_last_mm, cpu)) { |
| + mmdrop_delayed(per_cpu(idle_last_mm, cpu)); |
| + per_cpu(idle_last_mm, cpu) = NULL; |
| + } |
| return 0; |
| } |
| #endif |
| -- |
| 2.1.4 |
| |