| From 950680b1d7b6d31c07a15fefb32df89477b9122c Mon Sep 17 00:00:00 2001 |
| From: Ingo Molnar <mingo@elte.hu> |
| Date: Fri, 3 Jul 2009 08:29:34 -0500 |
| Subject: [PATCH] hrtimers: Prepare full preemption |
| |
| Make cancellation of a running callback in softirq context safe |
| against preemption. |
| |
| Signed-off-by: Ingo Molnar <mingo@elte.hu> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| |
| diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h |
| index 3214d9ac2002..2f977fc8b679 100644 |
| --- a/include/linux/hrtimer.h |
| +++ b/include/linux/hrtimer.h |
| @@ -22,6 +22,7 @@ |
| #include <linux/percpu.h> |
| #include <linux/timer.h> |
| #include <linux/timerqueue.h> |
| +#include <linux/wait.h> |
| |
| /* |
| * Clock ids for hrtimers which expire in softirq context. These clock ids |
| @@ -212,6 +213,9 @@ struct hrtimer_cpu_base { |
| ktime_t expires_next; |
| struct hrtimer *next_timer; |
| ktime_t softirq_expires_next; |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| + wait_queue_head_t wait; |
| +#endif |
| struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; |
| } ____cacheline_aligned; |
| |
| @@ -426,6 +430,13 @@ static inline void hrtimer_restart(struct hrtimer *timer) |
| hrtimer_start_expires(timer, HRTIMER_MODE_ABS); |
| } |
| |
| +/* Softirq preemption could deadlock timer removal */ |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| + extern void hrtimer_wait_for_timer(const struct hrtimer *timer); |
| +#else |
| +# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0) |
| +#endif |
| + |
| /* Query timers: */ |
| extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust); |
| |
| @@ -450,7 +461,7 @@ static inline int hrtimer_is_queued(struct hrtimer *timer) |
| * Helper function to check, whether the timer is running the callback |
| * function |
| */ |
| -static inline int hrtimer_callback_running(struct hrtimer *timer) |
| +static inline int hrtimer_callback_running(const struct hrtimer *timer) |
| { |
| return timer->base->running == timer; |
| } |
| diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c |
| index f2815da015d8..e54dfd1aa231 100644 |
| --- a/kernel/time/hrtimer.c |
| +++ b/kernel/time/hrtimer.c |
| @@ -862,6 +862,33 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) |
| } |
| EXPORT_SYMBOL_GPL(hrtimer_forward); |
| |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| +# define wake_up_timer_waiters(b) wake_up(&(b)->wait) |
| + |
| +/** |
| + * hrtimer_wait_for_timer - Wait for a running timer |
| + * |
| + * @timer: timer to wait for |
| + * |
| + * The function waits in case the timers callback function is |
| + * currently executed on the waitqueue of the timer base. The |
| + * waitqueue is woken up after the timer callback function has |
| + * finished execution. |
| + */ |
| +void hrtimer_wait_for_timer(const struct hrtimer *timer) |
| +{ |
| + struct hrtimer_clock_base *base = timer->base; |
| + |
| + if (base && base->cpu_base && |
| + base->index >= HRTIMER_BASE_MONOTONIC_SOFT) |
| + wait_event(base->cpu_base->wait, |
| + !(hrtimer_callback_running(timer))); |
| +} |
| + |
| +#else |
| +# define wake_up_timer_waiters(b) do { } while (0) |
| +#endif |
| + |
| /* |
| * enqueue_hrtimer - internal function to (re)start a timer |
| * |
| @@ -1109,7 +1136,7 @@ int hrtimer_cancel(struct hrtimer *timer) |
| |
| if (ret >= 0) |
| return ret; |
| - cpu_relax(); |
| + hrtimer_wait_for_timer(timer); |
| } |
| } |
| EXPORT_SYMBOL_GPL(hrtimer_cancel); |
| @@ -1385,6 +1412,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h) |
| hrtimer_update_softirq_timer(cpu_base, true); |
| |
| raw_spin_unlock_irq(&cpu_base->lock); |
| + wake_up_timer_waiters(cpu_base); |
| } |
| |
| #ifdef CONFIG_HIGH_RES_TIMERS |
| @@ -1729,6 +1757,9 @@ int hrtimers_prepare_cpu(unsigned int cpu) |
| cpu_base->hres_active = 0; |
| cpu_base->expires_next = KTIME_MAX; |
| cpu_base->softirq_expires_next = KTIME_MAX; |
| +#ifdef CONFIG_PREEMPT_RT_BASE |
| + init_waitqueue_head(&cpu_base->wait); |
| +#endif |
| return 0; |
| } |
| |
| diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c |
| index 087d6a1279b8..48b57e458294 100644 |
| --- a/kernel/time/itimer.c |
| +++ b/kernel/time/itimer.c |
| @@ -195,6 +195,7 @@ int do_setitimer(int which, struct itimerval *value, struct itimerval *ovalue) |
| /* We are sharing ->siglock with it_real_fn() */ |
| if (hrtimer_try_to_cancel(timer) < 0) { |
| spin_unlock_irq(&tsk->sighand->siglock); |
| + hrtimer_wait_for_timer(&tsk->signal->real_timer); |
| goto again; |
| } |
| expires = timeval_to_ktime(value->it_value); |
| diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c |
| index 2e608587bd1e..651bd0b53352 100644 |
| --- a/kernel/time/posix-timers.c |
| +++ b/kernel/time/posix-timers.c |
| @@ -831,6 +831,20 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id) |
| return overrun; |
| } |
| |
| +/* |
| + * Protected by RCU! |
| + */ |
| +static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr) |
| +{ |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + if (kc->timer_set == common_timer_set) |
| + hrtimer_wait_for_timer(&timr->it.real.timer); |
| + else |
| + /* FIXME: Whacky hack for posix-cpu-timers */ |
| + schedule_timeout(1); |
| +#endif |
| +} |
| + |
| /* Set a POSIX.1b interval timer. */ |
| /* timr->it_lock is taken. */ |
| static int |
| @@ -910,6 +924,7 @@ SYSCALL_DEFINE4(timer_settime, timer_t, timer_id, int, flags, |
| if (!timr) |
| return -EINVAL; |
| |
| + rcu_read_lock(); |
| kc = clockid_to_kclock(timr->it_clock); |
| if (WARN_ON_ONCE(!kc || !kc->timer_set)) |
| error = -EINVAL; |
| @@ -918,9 +933,12 @@ SYSCALL_DEFINE4(timer_settime, timer_t, timer_id, int, flags, |
| |
| unlock_timer(timr, flag); |
| if (error == TIMER_RETRY) { |
| + timer_wait_for_callback(kc, timr); |
| rtn = NULL; // We already got the old time... |
| + rcu_read_unlock(); |
| goto retry; |
| } |
| + rcu_read_unlock(); |
| |
| old_spec = itimerspec64_to_itimerspec(&old_spec64); |
| if (old_setting && !error && |
| @@ -959,10 +977,15 @@ SYSCALL_DEFINE1(timer_delete, timer_t, timer_id) |
| if (!timer) |
| return -EINVAL; |
| |
| + rcu_read_lock(); |
| if (timer_delete_hook(timer) == TIMER_RETRY) { |
| unlock_timer(timer, flags); |
| + timer_wait_for_callback(clockid_to_kclock(timer->it_clock), |
| + timer); |
| + rcu_read_unlock(); |
| goto retry_delete; |
| } |
| + rcu_read_unlock(); |
| |
| spin_lock(¤t->sighand->siglock); |
| list_del(&timer->list); |
| @@ -988,8 +1011,18 @@ static void itimer_delete(struct k_itimer *timer) |
| retry_delete: |
| spin_lock_irqsave(&timer->it_lock, flags); |
| |
| + /* On RT we can race with a deletion */ |
| + if (!timer->it_signal) { |
| + unlock_timer(timer, flags); |
| + return; |
| + } |
| + |
| if (timer_delete_hook(timer) == TIMER_RETRY) { |
| + rcu_read_lock(); |
| unlock_timer(timer, flags); |
| + timer_wait_for_callback(clockid_to_kclock(timer->it_clock), |
| + timer); |
| + rcu_read_unlock(); |
| goto retry_delete; |
| } |
| list_del(&timer->list); |
| -- |
| 2.1.4 |
| |