| Subject: Use local irq lock instead of irq disable regions |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Sun, 17 Jul 2011 21:42:26 +0200 |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| kernel/workqueue.c | 33 ++++++++++++++++++--------------- |
| 1 file changed, 18 insertions(+), 15 deletions(-) |
| |
| --- a/kernel/workqueue.c |
| +++ b/kernel/workqueue.c |
| @@ -47,6 +47,7 @@ |
| #include <linux/nodemask.h> |
| #include <linux/moduleparam.h> |
| #include <linux/uaccess.h> |
| +#include <linux/locallock.h> |
| |
| #include "workqueue_internal.h" |
| |
| @@ -306,6 +307,8 @@ EXPORT_SYMBOL_GPL(system_unbound_wq); |
| struct workqueue_struct *system_freezable_wq __read_mostly; |
| EXPORT_SYMBOL_GPL(system_freezable_wq); |
| |
| +static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock); |
| + |
| static int worker_thread(void *__worker); |
| static void copy_workqueue_attrs(struct workqueue_attrs *to, |
| const struct workqueue_attrs *from); |
| @@ -1063,9 +1066,9 @@ static void put_pwq_unlocked(struct pool |
| * As both pwqs and pools are RCU protected, the |
| * following lock operations are safe. |
| */ |
| - spin_lock_irq(&pwq->pool->lock); |
| + local_spin_lock_irq(pendingb_lock, &pwq->pool->lock); |
| put_pwq(pwq); |
| - spin_unlock_irq(&pwq->pool->lock); |
| + local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock); |
| } |
| } |
| |
| @@ -1165,7 +1168,7 @@ static int try_to_grab_pending(struct wo |
| struct worker_pool *pool; |
| struct pool_workqueue *pwq; |
| |
| - local_irq_save(*flags); |
| + local_lock_irqsave(pendingb_lock, *flags); |
| |
| /* try to steal the timer if it exists */ |
| if (is_dwork) { |
| @@ -1229,7 +1232,7 @@ static int try_to_grab_pending(struct wo |
| spin_unlock(&pool->lock); |
| fail: |
| rcu_read_unlock(); |
| - local_irq_restore(*flags); |
| + local_unlock_irqrestore(pendingb_lock, *flags); |
| if (work_is_canceling(work)) |
| return -ENOENT; |
| cpu_relax(); |
| @@ -1301,7 +1304,7 @@ static void __queue_work(int cpu, struct |
| * queued or lose PENDING. Grabbing PENDING and queueing should |
| * happen with IRQ disabled. |
| */ |
| - WARN_ON_ONCE(!irqs_disabled()); |
| + WARN_ON_ONCE_NONRT(!irqs_disabled()); |
| |
| debug_work_activate(work); |
| |
| @@ -1406,14 +1409,14 @@ bool queue_work_on(int cpu, struct workq |
| bool ret = false; |
| unsigned long flags; |
| |
| - local_irq_save(flags); |
| + local_lock_irqsave(pendingb_lock,flags); |
| |
| if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { |
| __queue_work(cpu, wq, work); |
| ret = true; |
| } |
| |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pendingb_lock, flags); |
| return ret; |
| } |
| EXPORT_SYMBOL(queue_work_on); |
| @@ -1480,14 +1483,14 @@ bool queue_delayed_work_on(int cpu, stru |
| unsigned long flags; |
| |
| /* read the comment in __queue_work() */ |
| - local_irq_save(flags); |
| + local_lock_irqsave(pendingb_lock, flags); |
| |
| if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { |
| __queue_delayed_work(cpu, wq, dwork, delay); |
| ret = true; |
| } |
| |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pendingb_lock, flags); |
| return ret; |
| } |
| EXPORT_SYMBOL(queue_delayed_work_on); |
| @@ -1522,7 +1525,7 @@ bool mod_delayed_work_on(int cpu, struct |
| |
| if (likely(ret >= 0)) { |
| __queue_delayed_work(cpu, wq, dwork, delay); |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pendingb_lock, flags); |
| } |
| |
| /* -ENOENT from try_to_grab_pending() becomes %true */ |
| @@ -2855,7 +2858,7 @@ static bool __cancel_work_timer(struct w |
| |
| /* tell other tasks trying to grab @work to back off */ |
| mark_work_canceling(work); |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pendingb_lock, flags); |
| |
| flush_work(work); |
| clear_work_data(work); |
| @@ -2900,10 +2903,10 @@ EXPORT_SYMBOL_GPL(cancel_work_sync); |
| */ |
| bool flush_delayed_work(struct delayed_work *dwork) |
| { |
| - local_irq_disable(); |
| + local_lock_irq(pendingb_lock); |
| if (del_timer_sync(&dwork->timer)) |
| __queue_work(dwork->cpu, dwork->wq, &dwork->work); |
| - local_irq_enable(); |
| + local_unlock_irq(pendingb_lock); |
| return flush_work(&dwork->work); |
| } |
| EXPORT_SYMBOL(flush_delayed_work); |
| @@ -2934,7 +2937,7 @@ bool cancel_delayed_work(struct delayed_ |
| |
| set_work_pool_and_clear_pending(&dwork->work, |
| get_work_pool_id(&dwork->work)); |
| - local_irq_restore(flags); |
| + local_unlock_irqrestore(pendingb_lock, flags); |
| return ret; |
| } |
| EXPORT_SYMBOL(cancel_delayed_work); |
| @@ -4385,7 +4388,7 @@ unsigned int work_busy(struct work_struc |
| if (work_pending(work)) |
| ret |= WORK_BUSY_PENDING; |
| |
| - rcu_read_lock() |
| + rcu_read_lock(); |
| pool = get_work_pool(work); |
| if (pool) { |
| spin_lock_irqsave(&pool->lock, flags); |