| Subject: sched: Generic migrate_disable |
| From: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Date: Thu Aug 11 15:14:58 CEST 2011 |
| |
| Make migrate_disable() be a preempt_disable() for !rt kernels. This |
| allows generic code to use it but still enforces that these code |
| sections stay relatively small. |
| |
| A preemptible migrate_disable() accessible for general use would allow |
| people growing arbitrary per-cpu crap instead of clean these things |
| up. |
| |
| Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Link: http://lkml.kernel.org/n/tip-275i87sl8e1jcamtchmehonm@git.kernel.org |
| --- |
| include/linux/preempt.h | 21 +++++++++------------ |
| include/linux/sched.h | 13 +++++++++++++ |
| include/linux/smp.h | 9 ++------- |
| kernel/sched/core.c | 6 ++++-- |
| kernel/trace/trace.c | 2 +- |
| lib/smp_processor_id.c | 2 +- |
| 6 files changed, 30 insertions(+), 23 deletions(-) |
| |
| --- a/include/linux/preempt.h |
| +++ b/include/linux/preempt.h |
| @@ -108,28 +108,25 @@ do { \ |
| |
| #endif /* CONFIG_PREEMPT_COUNT */ |
| |
| -#ifdef CONFIG_SMP |
| -extern void migrate_disable(void); |
| -extern void migrate_enable(void); |
| -#else |
| -# define migrate_disable() do { } while (0) |
| -# define migrate_enable() do { } while (0) |
| -#endif |
| - |
| #ifdef CONFIG_PREEMPT_RT_FULL |
| # define preempt_disable_rt() preempt_disable() |
| # define preempt_enable_rt() preempt_enable() |
| # define preempt_disable_nort() do { } while (0) |
| # define preempt_enable_nort() do { } while (0) |
| -# define migrate_disable_rt() migrate_disable() |
| -# define migrate_enable_rt() migrate_enable() |
| +# ifdef CONFIG_SMP |
| + extern void migrate_disable(void); |
| + extern void migrate_enable(void); |
| +# else /* CONFIG_SMP */ |
| +# define migrate_disable() do { } while (0) |
| +# define migrate_enable() do { } while (0) |
| +# endif /* CONFIG_SMP */ |
| #else |
| # define preempt_disable_rt() do { } while (0) |
| # define preempt_enable_rt() do { } while (0) |
| # define preempt_disable_nort() preempt_disable() |
| # define preempt_enable_nort() preempt_enable() |
| -# define migrate_disable_rt() do { } while (0) |
| -# define migrate_enable_rt() do { } while (0) |
| +# define migrate_disable() preempt_disable() |
| +# define migrate_enable() preempt_enable() |
| #endif |
| |
| #ifdef CONFIG_PREEMPT_NOTIFIERS |
| --- a/include/linux/sched.h |
| +++ b/include/linux/sched.h |
| @@ -1279,7 +1279,9 @@ struct task_struct { |
| #endif |
| |
| unsigned int policy; |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| int migrate_disable; |
| +#endif |
| int nr_cpus_allowed; |
| cpumask_t cpus_allowed; |
| |
| @@ -2810,11 +2812,22 @@ static inline void set_task_cpu(struct t |
| |
| #endif /* CONFIG_SMP */ |
| |
| +static inline int __migrate_disabled(struct task_struct *p) |
| +{ |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + return p->migrate_disable; |
| +#else |
| + return 0; |
| +#endif |
| +} |
| + |
| /* Future-safe accessor for struct task_struct's cpus_allowed. */ |
| static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p) |
| { |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| if (p->migrate_disable) |
| return cpumask_of(task_cpu(p)); |
| +#endif |
| |
| return &p->cpus_allowed; |
| } |
| --- a/include/linux/smp.h |
| +++ b/include/linux/smp.h |
| @@ -218,13 +218,8 @@ static inline void kick_all_cpus_sync(vo |
| #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) |
| #define put_cpu() preempt_enable() |
| |
| -#ifndef CONFIG_PREEMPT_RT_FULL |
| -# define get_cpu_light() get_cpu() |
| -# define put_cpu_light() put_cpu() |
| -#else |
| -# define get_cpu_light() ({ migrate_disable(); smp_processor_id(); }) |
| -# define put_cpu_light() migrate_enable() |
| -#endif |
| +#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); }) |
| +#define put_cpu_light() migrate_enable() |
| |
| /* |
| * Callback to arch code if there's nosmp or maxcpus=0 on the |
| --- a/kernel/sched/core.c |
| +++ b/kernel/sched/core.c |
| @@ -4734,7 +4734,7 @@ void __cpuinit init_idle(struct task_str |
| #ifdef CONFIG_SMP |
| void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) |
| { |
| - if (!p->migrate_disable) { |
| + if (!__migrate_disabled(p)) { |
| if (p->sched_class && p->sched_class->set_cpus_allowed) |
| p->sched_class->set_cpus_allowed(p, new_mask); |
| p->nr_cpus_allowed = cpumask_weight(new_mask); |
| @@ -4790,7 +4790,7 @@ int set_cpus_allowed_ptr(struct task_str |
| do_set_cpus_allowed(p, new_mask); |
| |
| /* Can the task run on the task's current CPU? If so, we're done */ |
| - if (cpumask_test_cpu(task_cpu(p), new_mask) || p->migrate_disable) |
| + if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p)) |
| goto out; |
| |
| dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); |
| @@ -4809,6 +4809,7 @@ out: |
| } |
| EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); |
| |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| void migrate_disable(void) |
| { |
| struct task_struct *p = current; |
| @@ -4901,6 +4902,7 @@ void migrate_enable(void) |
| preempt_enable(); |
| } |
| EXPORT_SYMBOL(migrate_enable); |
| +#endif /* CONFIG_PREEMPT_RT_FULL */ |
| |
| /* |
| * Move (not current) task off this cpu, onto dest cpu. We're doing |
| --- a/kernel/trace/trace.c |
| +++ b/kernel/trace/trace.c |
| @@ -1178,7 +1178,7 @@ tracing_generic_entry_update(struct trac |
| ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | |
| (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); |
| |
| - entry->migrate_disable = (tsk) ? tsk->migrate_disable & 0xFF : 0; |
| + entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0; |
| } |
| EXPORT_SYMBOL_GPL(tracing_generic_entry_update); |
| |
| --- a/lib/smp_processor_id.c |
| +++ b/lib/smp_processor_id.c |
| @@ -41,7 +41,7 @@ notrace unsigned int debug_smp_processor |
| |
| printk(KERN_ERR "BUG: using smp_processor_id() in preemptible [%08x %08x] " |
| "code: %s/%d\n", preempt_count() - 1, |
| - current->migrate_disable, current->comm, current->pid); |
| + __migrate_disabled(current), current->comm, current->pid); |
| print_symbol("caller is %s\n", (long)__builtin_return_address(0)); |
| dump_stack(); |
| |