| Subject: wait-simple: Rework for use with completions |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Thu, 10 Jan 2013 11:47:35 +0100 |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| include/linux/wait-simple.h | 56 +++++++---------------------------- |
| kernel/wait-simple.c | 69 ++++++++++++++++++++++++++++++++++++++------ |
| 2 files changed, 72 insertions(+), 53 deletions(-) |
| |
| --- a/include/linux/wait-simple.h |
| +++ b/include/linux/wait-simple.h |
| @@ -22,12 +22,14 @@ struct swait_head { |
| struct list_head list; |
| }; |
| |
| -#define DEFINE_SWAIT_HEAD(name) \ |
| - struct swait_head name = { \ |
| +#define SWAIT_HEAD_INITIALIZER(name) { \ |
| .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ |
| .list = LIST_HEAD_INIT((name).list), \ |
| } |
| |
| +#define DEFINE_SWAIT_HEAD(name) \ |
| + struct swait_head name = SWAIT_HEAD_INITIALIZER(name) |
| + |
| extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); |
| |
| #define init_swait_head(swh) \ |
| @@ -40,59 +42,25 @@ extern void __init_swait_head(struct swa |
| /* |
| * Waiter functions |
| */ |
| -static inline bool swaiter_enqueued(struct swaiter *w) |
| -{ |
| - return w->task != NULL; |
| -} |
| - |
| +extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w); |
| extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state); |
| +extern void swait_finish_locked(struct swait_head *head, struct swaiter *w); |
| extern void swait_finish(struct swait_head *head, struct swaiter *w); |
| |
| /* |
| - * Adds w to head->list. Must be called with head->lock locked. |
| - */ |
| -static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) |
| -{ |
| - list_add(&w->node, &head->list); |
| -} |
| - |
| -/* |
| - * Removes w from head->list. Must be called with head->lock locked. |
| - */ |
| -static inline void __swait_dequeue(struct swaiter *w) |
| -{ |
| - list_del_init(&w->node); |
| -} |
| - |
| -/* |
| - * Check whether a head has waiters enqueued |
| - */ |
| -static inline bool swait_head_has_waiters(struct swait_head *h) |
| -{ |
| - return !list_empty(&h->list); |
| -} |
| - |
| -/* |
| * Wakeup functions |
| */ |
| -extern int __swait_wake(struct swait_head *head, unsigned int state); |
| +extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num); |
| +extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num); |
| |
| -static inline int swait_wake(struct swait_head *head) |
| -{ |
| - return swait_head_has_waiters(head) ? |
| - __swait_wake(head, TASK_NORMAL) : 0; |
| -} |
| - |
| -static inline int swait_wake_interruptible(struct swait_head *head) |
| -{ |
| - return swait_head_has_waiters(head) ? |
| - __swait_wake(head, TASK_INTERRUPTIBLE) : 0; |
| -} |
| +#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1) |
| +#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1) |
| +#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0) |
| +#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0) |
| |
| /* |
| * Event API |
| */ |
| - |
| #define __swait_event(wq, condition) \ |
| do { \ |
| DEFINE_SWAITER(__wait); \ |
| --- a/kernel/wait-simple.c |
| +++ b/kernel/wait-simple.c |
| @@ -12,6 +12,24 @@ |
| #include <linux/sched.h> |
| #include <linux/wait-simple.h> |
| |
| +/* Adds w to head->list. Must be called with head->lock locked. */ |
| +static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) |
| +{ |
| + list_add(&w->node, &head->list); |
| +} |
| + |
| +/* Removes w from head->list. Must be called with head->lock locked. */ |
| +static inline void __swait_dequeue(struct swaiter *w) |
| +{ |
| + list_del_init(&w->node); |
| +} |
| + |
| +/* Check whether a head has waiters enqueued */ |
| +static inline bool swait_head_has_waiters(struct swait_head *h) |
| +{ |
| + return !list_empty(&h->list); |
| +} |
| + |
| void __init_swait_head(struct swait_head *head, struct lock_class_key *key) |
| { |
| raw_spin_lock_init(&head->lock); |
| @@ -20,19 +38,31 @@ void __init_swait_head(struct swait_head |
| } |
| EXPORT_SYMBOL(__init_swait_head); |
| |
| +void swait_prepare_locked(struct swait_head *head, struct swaiter *w) |
| +{ |
| + w->task = current; |
| + if (list_empty(&w->node)) |
| + __swait_enqueue(head, w); |
| +} |
| + |
| void swait_prepare(struct swait_head *head, struct swaiter *w, int state) |
| { |
| unsigned long flags; |
| |
| raw_spin_lock_irqsave(&head->lock, flags); |
| - w->task = current; |
| - if (list_empty(&w->node)) |
| - __swait_enqueue(head, w); |
| - set_current_state(state); |
| + swait_prepare_locked(head, w); |
| + __set_current_state(state); |
| raw_spin_unlock_irqrestore(&head->lock, flags); |
| } |
| EXPORT_SYMBOL(swait_prepare); |
| |
| +void swait_finish_locked(struct swait_head *head, struct swaiter *w) |
| +{ |
| + __set_current_state(TASK_RUNNING); |
| + if (w->task) |
| + __swait_dequeue(w); |
| +} |
| + |
| void swait_finish(struct swait_head *head, struct swaiter *w) |
| { |
| unsigned long flags; |
| @@ -46,22 +76,43 @@ void swait_finish(struct swait_head *hea |
| } |
| EXPORT_SYMBOL(swait_finish); |
| |
| -int __swait_wake(struct swait_head *head, unsigned int state) |
| +unsigned int |
| +__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num) |
| { |
| struct swaiter *curr, *next; |
| - unsigned long flags; |
| int woken = 0; |
| |
| - raw_spin_lock_irqsave(&head->lock, flags); |
| - |
| list_for_each_entry_safe(curr, next, &head->list, node) { |
| if (wake_up_state(curr->task, state)) { |
| __swait_dequeue(curr); |
| + /* |
| + * The waiting task can free the waiter as |
| + * soon as curr->task = NULL is written, |
| + * without taking any locks. A memory barrier |
| + * is required here to prevent the following |
| + * store to curr->task from getting ahead of |
| + * the dequeue operation. |
| + */ |
| + smp_wmb(); |
| curr->task = NULL; |
| - woken++; |
| + if (++woken == num) |
| + break; |
| } |
| } |
| + return woken; |
| +} |
| + |
| +unsigned int |
| +__swait_wake(struct swait_head *head, unsigned int state, unsigned int num) |
| +{ |
| + unsigned long flags; |
| + int woken; |
| |
| + if (!swait_head_has_waiters(head)) |
| + return 0; |
| + |
| + raw_spin_lock_irqsave(&head->lock, flags); |
| + woken = __swait_wake_locked(head, state, num); |
| raw_spin_unlock_irqrestore(&head->lock, flags); |
| return woken; |
| } |