| From acc91991542212f243d7e8293530aa224a98f77f Mon Sep 17 00:00:00 2001 |
| From: Ingo Molnar <mingo@elte.hu> |
| Date: Fri, 3 Jul 2009 08:44:01 -0500 |
| Subject: [PATCH] softirq: per cpu assumptions fixes |
| |
| commit 33fe41da821c6c542bcaa612a46695425525fdc6 in tip. |
| |
| Signed-off-by: Ingo Molnar <mingo@elte.hu> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com> |
| |
| diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c |
| index c586357..82efa41 100644 |
| --- a/kernel/hrtimer.c |
| +++ b/kernel/hrtimer.c |
| @@ -499,9 +499,9 @@ static inline int hrtimer_is_hres_enabled(void) |
| /* |
| * Is the high resolution mode active ? |
| */ |
| -static inline int hrtimer_hres_active(void) |
| +static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) |
| { |
| - return __get_cpu_var(hrtimer_bases).hres_active; |
| + return cpu_base->hres_active; |
| } |
| |
| /* |
| @@ -611,11 +611,11 @@ static int hrtimer_reprogram(struct hrtimer *timer, |
| */ |
| static void retrigger_next_event(void *arg) |
| { |
| - struct hrtimer_cpu_base *base; |
| + struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases); |
| struct timespec realtime_offset; |
| unsigned long seq; |
| |
| - if (!hrtimer_hres_active()) |
| + if (!hrtimer_hres_active(base)) |
| return; |
| |
| do { |
| @@ -625,8 +625,6 @@ static void retrigger_next_event(void *arg) |
| -wall_to_monotonic.tv_nsec); |
| } while (read_raw_seqretry(&xtime_lock, seq)); |
| |
| - base = &__get_cpu_var(hrtimer_bases); |
| - |
| /* Adjust CLOCK_REALTIME offset */ |
| raw_spin_lock(&base->lock); |
| base->clock_base[CLOCK_REALTIME].offset = |
| @@ -709,10 +707,8 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, |
| /* |
| * Switch to high resolution mode |
| */ |
| -static int hrtimer_switch_to_hres(void) |
| +static int hrtimer_switch_to_hres(struct hrtimer_cpu_base *base) |
| { |
| - int cpu = smp_processor_id(); |
| - struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); |
| unsigned long flags; |
| |
| if (base->hres_active) |
| @@ -723,7 +719,7 @@ static int hrtimer_switch_to_hres(void) |
| if (tick_init_highres()) { |
| local_irq_restore(flags); |
| printk(KERN_WARNING "Could not switch to high resolution " |
| - "mode on CPU %d\n", cpu); |
| + "mode on CPU %d\n", raw_smp_processor_id()); |
| return 0; |
| } |
| base->hres_active = 1; |
| @@ -740,9 +736,15 @@ static int hrtimer_switch_to_hres(void) |
| |
| #else |
| |
| -static inline int hrtimer_hres_active(void) { return 0; } |
| +static inline int hrtimer_hres_active(struct hrtimer_cpu_base *base) |
| +{ |
| + return 0; |
| +} |
| static inline int hrtimer_is_hres_enabled(void) { return 0; } |
| -static inline int hrtimer_switch_to_hres(void) { return 0; } |
| +static inline int hrtimer_switch_to_hres(struct hrtimer_cpu_base *base) |
| +{ |
| + return 0; |
| +} |
| static inline void |
| hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } |
| static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, |
| @@ -940,7 +942,7 @@ static void __remove_hrtimer(struct hrtimer *timer, |
| base->first = rb_next(&timer->node); |
| #ifdef CONFIG_HIGH_RES_TIMERS |
| /* Reprogram the clock event device. if enabled */ |
| - if (reprogram && hrtimer_hres_active()) { |
| + if (reprogram && hrtimer_hres_active(base->cpu_base)) { |
| ktime_t expires; |
| |
| expires = ktime_sub(hrtimer_get_expires(timer), |
| @@ -1153,7 +1155,7 @@ ktime_t hrtimer_get_next_event(void) |
| |
| raw_spin_lock_irqsave(&cpu_base->lock, flags); |
| |
| - if (!hrtimer_hres_active()) { |
| + if (!hrtimer_hres_active(cpu_base)) { |
| for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { |
| struct hrtimer *timer; |
| |
| @@ -1398,9 +1400,11 @@ retry: |
| */ |
| static void __hrtimer_peek_ahead_timers(void) |
| { |
| + struct hrtimer_cpu_base *cpu_base; |
| struct tick_device *td; |
| |
| - if (!hrtimer_hres_active()) |
| + cpu_base = &__get_cpu_var(hrtimer_bases); |
| + if (!hrtimer_hres_active(cpu_base)) |
| return; |
| |
| td = &__get_cpu_var(tick_cpu_device); |
| @@ -1446,7 +1450,9 @@ static inline void __hrtimer_peek_ahead_timers(void) { } |
| */ |
| void hrtimer_run_pending(void) |
| { |
| - if (hrtimer_hres_active()) |
| + struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
| + |
| + if (hrtimer_hres_active(cpu_base)) |
| return; |
| |
| /* |
| @@ -1458,7 +1464,7 @@ void hrtimer_run_pending(void) |
| * deadlock vs. xtime_lock. |
| */ |
| if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) |
| - hrtimer_switch_to_hres(); |
| + hrtimer_switch_to_hres(cpu_base); |
| } |
| |
| /* |
| @@ -1467,11 +1473,12 @@ void hrtimer_run_pending(void) |
| void hrtimer_run_queues(void) |
| { |
| struct rb_node *node; |
| - struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
| + struct hrtimer_cpu_base *cpu_base; |
| struct hrtimer_clock_base *base; |
| int index, gettime = 1; |
| |
| - if (hrtimer_hres_active()) |
| + cpu_base = &per_cpu(hrtimer_bases, raw_smp_processor_id()); |
| + if (hrtimer_hres_active(cpu_base)) |
| return; |
| |
| for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { |
| diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c |
| index d5a267d..4881995 100644 |
| --- a/kernel/sched_fair.c |
| +++ b/kernel/sched_fair.c |
| @@ -3438,7 +3438,7 @@ out: |
| */ |
| static void run_rebalance_domains(struct softirq_action *h) |
| { |
| - int this_cpu = smp_processor_id(); |
| + int this_cpu = raw_smp_processor_id(); |
| struct rq *this_rq = cpu_rq(this_cpu); |
| enum cpu_idle_type idle = this_rq->idle_at_tick ? |
| CPU_IDLE : CPU_NOT_IDLE; |
| diff --git a/kernel/timer.c b/kernel/timer.c |
| index f0d478a..8f29bd0 100644 |
| --- a/kernel/timer.c |
| +++ b/kernel/timer.c |
| @@ -1274,7 +1274,7 @@ void update_process_times(int user_tick) |
| */ |
| static void run_timer_softirq(struct softirq_action *h) |
| { |
| - struct tvec_base *base = __get_cpu_var(tvec_bases); |
| + struct tvec_base *base = per_cpu(tvec_bases, raw_smp_processor_id()); |
| |
| printk_tick(); |
| hrtimer_run_pending(); |
| -- |
| 1.7.1.1 |
| |