| From 54b7429efffc99e845ba9381bee3244f012a06c2 Mon Sep 17 00:00:00 2001 |
| From: Frederic Weisbecker <frederic@kernel.org> |
| Date: Mon, 1 Feb 2021 00:05:44 +0100 |
| Subject: rcu: Pull deferred rcuog wake up to rcu_eqs_enter() callers |
| |
| From: Frederic Weisbecker <frederic@kernel.org> |
| |
| commit 54b7429efffc99e845ba9381bee3244f012a06c2 upstream. |
| |
| Deferred wakeup of rcuog kthreads upon RCU idle mode entry is going to |
| be handled differently whether initiated by idle, user or guest. Prepare |
| with pulling that control up to rcu_eqs_enter() callers. |
| |
| Signed-off-by: Frederic Weisbecker <frederic@kernel.org> |
| Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> |
| Signed-off-by: Ingo Molnar <mingo@kernel.org> |
| Cc: stable@vger.kernel.org |
| Link: https://lkml.kernel.org/r/20210131230548.32970-2-frederic@kernel.org |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| --- |
| kernel/rcu/tree.c | 11 ++++++++++- |
| 1 file changed, 10 insertions(+), 1 deletion(-) |
| |
| --- a/kernel/rcu/tree.c |
| +++ b/kernel/rcu/tree.c |
| @@ -636,7 +636,6 @@ static noinstr void rcu_eqs_enter(bool u |
| trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks)); |
| WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); |
| rdp = this_cpu_ptr(&rcu_data); |
| - do_nocb_deferred_wakeup(rdp); |
| rcu_prepare_for_idle(); |
| rcu_preempt_deferred_qs(current); |
| |
| @@ -664,7 +663,10 @@ static noinstr void rcu_eqs_enter(bool u |
| */ |
| void rcu_idle_enter(void) |
| { |
| + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
| + |
| lockdep_assert_irqs_disabled(); |
| + do_nocb_deferred_wakeup(rdp); |
| rcu_eqs_enter(false); |
| } |
| EXPORT_SYMBOL_GPL(rcu_idle_enter); |
| @@ -683,7 +685,14 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); |
| */ |
| noinstr void rcu_user_enter(void) |
| { |
| + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
| + |
| lockdep_assert_irqs_disabled(); |
| + |
| + instrumentation_begin(); |
| + do_nocb_deferred_wakeup(rdp); |
| + instrumentation_end(); |
| + |
| rcu_eqs_enter(true); |
| } |
| #endif /* CONFIG_NO_HZ_FULL */ |