| Subject: cpu/hotplug: Implement CPU pinning |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Wed, 19 Jul 2017 17:31:20 +0200 |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| include/linux/sched.h | 1 + |
| kernel/cpu.c | 40 ++++++++++++++++++++++++++++++++++++++++ |
| 2 files changed, 41 insertions(+) |
| |
| --- a/include/linux/sched.h |
| +++ b/include/linux/sched.h |
| @@ -546,6 +546,7 @@ struct task_struct { |
| #if defined(CONFIG_PREEMPT_COUNT) && defined(CONFIG_SMP) |
| int migrate_disable; |
| int migrate_disable_update; |
| + int pinned_on_cpu; |
| # ifdef CONFIG_SCHED_DEBUG |
| int migrate_disable_atomic; |
| # endif |
| --- a/kernel/cpu.c |
| +++ b/kernel/cpu.c |
| @@ -66,6 +66,11 @@ struct cpuhp_cpu_state { |
| |
| static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state); |
| |
| +#ifdef CONFIG_HOTPLUG_CPU |
| +static DEFINE_PER_CPU(struct rt_rw_lock, cpuhp_pin_lock) = \ |
| + __RWLOCK_RT_INITIALIZER(cpuhp_pin_lock); |
| +#endif |
| + |
| #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) |
| static struct lock_class_key cpuhp_state_key; |
| static struct lockdep_map cpuhp_state_lock_map = |
| @@ -216,7 +221,30 @@ static int cpu_hotplug_disabled; |
| */ |
| void pin_current_cpu(void) |
| { |
| + struct rt_rw_lock *cpuhp_pin; |
| + unsigned int cpu; |
| + int ret; |
| |
| +again: |
| + cpuhp_pin = this_cpu_ptr(&cpuhp_pin_lock); |
| + ret = __read_rt_trylock(cpuhp_pin); |
| + if (ret) { |
| + current->pinned_on_cpu = smp_processor_id(); |
| + return; |
| + } |
| + cpu = smp_processor_id(); |
| + preempt_lazy_enable(); |
| + preempt_enable(); |
| + |
| + __read_rt_lock(cpuhp_pin); |
| + |
| + preempt_disable(); |
| + preempt_lazy_disable(); |
| + if (cpu != smp_processor_id()) { |
| + __read_rt_unlock(cpuhp_pin); |
| + goto again; |
| + } |
| + current->pinned_on_cpu = cpu; |
| } |
| |
| /** |
| @@ -224,6 +252,13 @@ void pin_current_cpu(void) |
| */ |
| void unpin_current_cpu(void) |
| { |
| + struct rt_rw_lock *cpuhp_pin = this_cpu_ptr(&cpuhp_pin_lock); |
| + |
| + if (WARN_ON(current->pinned_on_cpu != smp_processor_id())) |
| + cpuhp_pin = per_cpu_ptr(&cpuhp_pin_lock, current->pinned_on_cpu); |
| + |
| + current->pinned_on_cpu = -1; |
| + __read_rt_unlock(cpuhp_pin); |
| } |
| |
| DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); |
| @@ -644,6 +679,7 @@ static int take_cpu_down(void *_param) |
| |
| static int takedown_cpu(unsigned int cpu) |
| { |
| + struct rt_rw_lock *cpuhp_pin = per_cpu_ptr(&cpuhp_pin_lock, cpu); |
| struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); |
| int err; |
| |
| @@ -657,11 +693,14 @@ static int takedown_cpu(unsigned int cpu |
| */ |
| irq_lock_sparse(); |
| |
| + __write_rt_lock(cpuhp_pin); |
| + |
| /* |
| * So now all preempt/rcu users must observe !cpu_active(). |
| */ |
| err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu)); |
| if (err) { |
| + __write_rt_unlock(cpuhp_pin); |
| /* CPU refused to die */ |
| irq_unlock_sparse(); |
| /* Unpark the hotplug thread so we can rollback there */ |
| @@ -680,6 +719,7 @@ static int takedown_cpu(unsigned int cpu |
| wait_for_completion(&st->done); |
| BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); |
| |
| + __write_rt_unlock(cpuhp_pin); |
| /* Interrupts are moved away from the dying cpu, reenable alloc/free */ |
| irq_unlock_sparse(); |
| |