| Subject: completion: Use simple wait queues |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Fri, 11 Jan 2013 11:23:51 +0100 |
| |
| Completions have no long lasting callbacks and therefor do not need |
| the complex waitqueue variant. Use simple waitqueues which reduces the |
| contention on the waitqueue lock. |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| [cminyard@mvista.com: Move __prepare_to_swait() into the do loop because |
| swake_up_locked() removes the waiter on wake from the queue while in the |
| original code it is not the case] |
| Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| --- |
| arch/powerpc/platforms/ps3/device-init.c | 4 +- |
| drivers/net/wireless/intersil/orinoco/orinoco_usb.c | 4 +- |
| drivers/usb/gadget/function/f_fs.c | 2 - |
| drivers/usb/gadget/legacy/inode.c | 4 +- |
| include/linux/completion.h | 8 ++-- |
| include/linux/swait.h | 2 + |
| kernel/sched/completion.c | 34 ++++++++++---------- |
| kernel/sched/swait.c | 16 ++++++++- |
| 8 files changed, 45 insertions(+), 29 deletions(-) |
| |
| --- a/arch/powerpc/platforms/ps3/device-init.c |
| +++ b/arch/powerpc/platforms/ps3/device-init.c |
| @@ -738,8 +738,8 @@ static int ps3_notification_read_write(s |
| } |
| pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op); |
| |
| - res = wait_event_interruptible(dev->done.wait, |
| - dev->done.done || kthread_should_stop()); |
| + res = swait_event_interruptible_exclusive(dev->done.wait, |
| + dev->done.done || kthread_should_stop()); |
| if (kthread_should_stop()) |
| res = -EINTR; |
| if (res) { |
| --- a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c |
| +++ b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c |
| @@ -693,8 +693,8 @@ static void ezusb_req_ctx_wait(struct ez |
| while (!ctx->done.done && msecs--) |
| udelay(1000); |
| } else { |
| - wait_event_interruptible(ctx->done.wait, |
| - ctx->done.done); |
| + swait_event_interruptible_exclusive(ctx->done.wait, |
| + ctx->done.done); |
| } |
| break; |
| default: |
| --- a/drivers/usb/gadget/function/f_fs.c |
| +++ b/drivers/usb/gadget/function/f_fs.c |
| @@ -1704,7 +1704,7 @@ static void ffs_data_put(struct ffs_data |
| pr_info("%s(): freeing\n", __func__); |
| ffs_data_clear(ffs); |
| BUG_ON(waitqueue_active(&ffs->ev.waitq) || |
| - waitqueue_active(&ffs->ep0req_completion.wait) || |
| + swait_active(&ffs->ep0req_completion.wait) || |
| waitqueue_active(&ffs->wait)); |
| destroy_workqueue(ffs->io_completion_wq); |
| kfree(ffs->dev_name); |
| --- a/drivers/usb/gadget/legacy/inode.c |
| +++ b/drivers/usb/gadget/legacy/inode.c |
| @@ -344,7 +344,7 @@ ep_io (struct ep_data *epdata, void *buf |
| spin_unlock_irq (&epdata->dev->lock); |
| |
| if (likely (value == 0)) { |
| - value = wait_event_interruptible (done.wait, done.done); |
| + value = swait_event_interruptible_exclusive(done.wait, done.done); |
| if (value != 0) { |
| spin_lock_irq (&epdata->dev->lock); |
| if (likely (epdata->ep != NULL)) { |
| @@ -353,7 +353,7 @@ ep_io (struct ep_data *epdata, void *buf |
| usb_ep_dequeue (epdata->ep, epdata->req); |
| spin_unlock_irq (&epdata->dev->lock); |
| |
| - wait_event (done.wait, done.done); |
| + swait_event_exclusive(done.wait, done.done); |
| if (epdata->status == -ECONNRESET) |
| epdata->status = -EINTR; |
| } else { |
| --- a/include/linux/completion.h |
| +++ b/include/linux/completion.h |
| @@ -9,7 +9,7 @@ |
| * See kernel/sched/completion.c for details. |
| */ |
| |
| -#include <linux/wait.h> |
| +#include <linux/swait.h> |
| |
| /* |
| * struct completion - structure used to maintain state for a "completion" |
| @@ -25,7 +25,7 @@ |
| */ |
| struct completion { |
| unsigned int done; |
| - wait_queue_head_t wait; |
| + struct swait_queue_head wait; |
| }; |
| |
| #define init_completion_map(x, m) __init_completion(x) |
| @@ -34,7 +34,7 @@ static inline void complete_acquire(stru |
| static inline void complete_release(struct completion *x) {} |
| |
| #define COMPLETION_INITIALIZER(work) \ |
| - { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) } |
| + { 0, __SWAIT_QUEUE_HEAD_INITIALIZER((work).wait) } |
| |
| #define COMPLETION_INITIALIZER_ONSTACK_MAP(work, map) \ |
| (*({ init_completion_map(&(work), &(map)); &(work); })) |
| @@ -85,7 +85,7 @@ static inline void complete_release(stru |
| static inline void __init_completion(struct completion *x) |
| { |
| x->done = 0; |
| - init_waitqueue_head(&x->wait); |
| + init_swait_queue_head(&x->wait); |
| } |
| |
| /** |
| --- a/include/linux/swait.h |
| +++ b/include/linux/swait.h |
| @@ -160,7 +160,9 @@ static inline bool swq_has_sleeper(struc |
| extern void swake_up_one(struct swait_queue_head *q); |
| extern void swake_up_all(struct swait_queue_head *q); |
| extern void swake_up_locked(struct swait_queue_head *q); |
| +extern void swake_up_all_locked(struct swait_queue_head *q); |
| |
| +extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait); |
| extern void prepare_to_swait_exclusive(struct swait_queue_head *q, struct swait_queue *wait, int state); |
| extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state); |
| |
| --- a/kernel/sched/completion.c |
| +++ b/kernel/sched/completion.c |
| @@ -29,12 +29,12 @@ void complete(struct completion *x) |
| { |
| unsigned long flags; |
| |
| - spin_lock_irqsave(&x->wait.lock, flags); |
| + raw_spin_lock_irqsave(&x->wait.lock, flags); |
| |
| if (x->done != UINT_MAX) |
| x->done++; |
| - __wake_up_locked(&x->wait, TASK_NORMAL, 1); |
| - spin_unlock_irqrestore(&x->wait.lock, flags); |
| + swake_up_locked(&x->wait); |
| + raw_spin_unlock_irqrestore(&x->wait.lock, flags); |
| } |
| EXPORT_SYMBOL(complete); |
| |
| @@ -58,10 +58,10 @@ void complete_all(struct completion *x) |
| { |
| unsigned long flags; |
| |
| - spin_lock_irqsave(&x->wait.lock, flags); |
| + raw_spin_lock_irqsave(&x->wait.lock, flags); |
| x->done = UINT_MAX; |
| - __wake_up_locked(&x->wait, TASK_NORMAL, 0); |
| - spin_unlock_irqrestore(&x->wait.lock, flags); |
| + swake_up_all_locked(&x->wait); |
| + raw_spin_unlock_irqrestore(&x->wait.lock, flags); |
| } |
| EXPORT_SYMBOL(complete_all); |
| |
| @@ -70,20 +70,20 @@ do_wait_for_common(struct completion *x, |
| long (*action)(long), long timeout, int state) |
| { |
| if (!x->done) { |
| - DECLARE_WAITQUEUE(wait, current); |
| + DECLARE_SWAITQUEUE(wait); |
| |
| - __add_wait_queue_entry_tail_exclusive(&x->wait, &wait); |
| do { |
| if (signal_pending_state(state, current)) { |
| timeout = -ERESTARTSYS; |
| break; |
| } |
| + __prepare_to_swait(&x->wait, &wait); |
| __set_current_state(state); |
| - spin_unlock_irq(&x->wait.lock); |
| + raw_spin_unlock_irq(&x->wait.lock); |
| timeout = action(timeout); |
| - spin_lock_irq(&x->wait.lock); |
| + raw_spin_lock_irq(&x->wait.lock); |
| } while (!x->done && timeout); |
| - __remove_wait_queue(&x->wait, &wait); |
| + __finish_swait(&x->wait, &wait); |
| if (!x->done) |
| return timeout; |
| } |
| @@ -100,9 +100,9 @@ static inline long __sched |
| |
| complete_acquire(x); |
| |
| - spin_lock_irq(&x->wait.lock); |
| + raw_spin_lock_irq(&x->wait.lock); |
| timeout = do_wait_for_common(x, action, timeout, state); |
| - spin_unlock_irq(&x->wait.lock); |
| + raw_spin_unlock_irq(&x->wait.lock); |
| |
| complete_release(x); |
| |
| @@ -291,12 +291,12 @@ bool try_wait_for_completion(struct comp |
| if (!READ_ONCE(x->done)) |
| return false; |
| |
| - spin_lock_irqsave(&x->wait.lock, flags); |
| + raw_spin_lock_irqsave(&x->wait.lock, flags); |
| if (!x->done) |
| ret = false; |
| else if (x->done != UINT_MAX) |
| x->done--; |
| - spin_unlock_irqrestore(&x->wait.lock, flags); |
| + raw_spin_unlock_irqrestore(&x->wait.lock, flags); |
| return ret; |
| } |
| EXPORT_SYMBOL(try_wait_for_completion); |
| @@ -322,8 +322,8 @@ bool completion_done(struct completion * |
| * otherwise we can end up freeing the completion before complete() |
| * is done referencing it. |
| */ |
| - spin_lock_irqsave(&x->wait.lock, flags); |
| - spin_unlock_irqrestore(&x->wait.lock, flags); |
| + raw_spin_lock_irqsave(&x->wait.lock, flags); |
| + raw_spin_unlock_irqrestore(&x->wait.lock, flags); |
| return true; |
| } |
| EXPORT_SYMBOL(completion_done); |
| --- a/kernel/sched/swait.c |
| +++ b/kernel/sched/swait.c |
| @@ -32,6 +32,20 @@ void swake_up_locked(struct swait_queue_ |
| } |
| EXPORT_SYMBOL(swake_up_locked); |
| |
| +void swake_up_all_locked(struct swait_queue_head *q) |
| +{ |
| + struct swait_queue *curr; |
| + |
| + while (!list_empty(&q->task_list)) { |
| + |
| + curr = list_first_entry(&q->task_list, typeof(*curr), |
| + task_list); |
| + wake_up_process(curr->task); |
| + list_del_init(&curr->task_list); |
| + } |
| +} |
| +EXPORT_SYMBOL(swake_up_all_locked); |
| + |
| void swake_up_one(struct swait_queue_head *q) |
| { |
| unsigned long flags; |
| @@ -70,7 +84,7 @@ void swake_up_all(struct swait_queue_hea |
| } |
| EXPORT_SYMBOL(swake_up_all); |
| |
| -static void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait) |
| +void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait) |
| { |
| wait->task = current; |
| if (list_empty(&wait->task_list)) |