block: optimize rq allocation path for less queue locking
Signed-off-by: Jens Axboe <jaxboe@fusionio.com>
diff --git a/block/blk-core.c b/block/blk-core.c
index 91bc13d..aa94ba3 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -714,8 +714,6 @@
/*
* Get a free request, queue_lock must be held.
- * Returns NULL on failure, with queue_lock held.
- * Returns !NULL on success, with queue_lock *not held*.
*/
static struct request *get_request(struct request_queue *q, int rw_flags,
struct bio *bio, gfp_t gfp_mask)
@@ -723,6 +721,7 @@
struct request *rq = NULL;
struct request_list *rl = &q->rq;
const bool is_sync = rw_is_sync(rw_flags) != 0;
+ const bool drop_lock = (gfp_mask & __GFP_WAIT) != 0;
int may_queue, priv;
may_queue = elv_may_queue(q, rw_flags);
@@ -737,7 +736,9 @@
if (blk_queue_io_stat(q))
rw_flags |= REQ_IO_STAT;
- spin_unlock_irq(q->queue_lock);
+
+ if (drop_lock)
+ spin_unlock_irq(q->queue_lock);
rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
if (unlikely(!rq)) {
@@ -748,12 +749,17 @@
* Allocating task should really be put onto the front of the
* wait queue, but this is pretty rare.
*/
- spin_lock_irq(q->queue_lock);
+ if (drop_lock)
+ spin_lock_irq(q->queue_lock);
+
freed_request(q, is_sync, priv);
goto out;
}
trace_block_getrq(q, bio, rw_flags & 1);
+
+ if (drop_lock)
+ spin_lock_irq(q->queue_lock);
out:
return rq;
}
@@ -762,7 +768,7 @@
* No available requests for this queue, unplug the device and wait for some
* requests to become available.
*
- * Called with q->queue_lock held, and returns with it unlocked.
+ * Called with q->queue_lock held.
*/
static struct request *get_request_wait(struct request_queue *q, int rw_flags,
struct bio *bio)
@@ -770,7 +776,7 @@
const bool is_sync = rw_is_sync(rw_flags) != 0;
struct request *rq;
- rq = get_request(q, rw_flags, bio, GFP_NOIO);
+ rq = get_request(q, rw_flags, bio, GFP_ATOMIC);
while (!rq) {
DEFINE_WAIT(wait);
struct request_list *rl = &q->rq;
@@ -800,15 +806,13 @@
BUG_ON(rw != READ && rw != WRITE);
spin_lock_irq(q->queue_lock);
- if (gfp_mask & __GFP_WAIT) {
- rq = get_request_wait(q, rw, NULL);
- } else {
- rq = get_request(q, rw, NULL, gfp_mask);
- if (!rq)
- spin_unlock_irq(q->queue_lock);
- }
- /* q->queue_lock is unlocked at this point */
+ if (gfp_mask & __GFP_WAIT)
+ rq = get_request_wait(q, rw, NULL);
+ else
+ rq = get_request(q, rw, NULL, gfp_mask);
+
+ spin_unlock_irq(q->queue_lock);
return rq;
}
EXPORT_SYMBOL(blk_get_request);
@@ -1200,8 +1204,7 @@
rw_flags |= REQ_SYNC;
/*
- * Grab a free request. This is might sleep but can not fail.
- * Returns with the queue unlocked.
+ * Grab a free request.
*/
req = get_request_wait(q, rw_flags, bio);
@@ -1213,7 +1216,6 @@
*/
init_request_from_bio(req, bio);
- spin_lock_irq(q->queue_lock);
if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) ||
bio_flagged(bio, BIO_CPU_AFFINE))
req->cpu = blk_cpu_to_group(smp_processor_id());
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index f65c6f0..3d8635d 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -2543,12 +2543,12 @@
cfq_put_cfqg(orig_cfqg);
}
+typedef void (cic_call_fn)(struct io_context *, struct cfq_io_context *);
+
/*
* Must always be called with the rcu_read_lock() held
*/
-static void
-__call_for_each_cic(struct io_context *ioc,
- void (*func)(struct io_context *, struct cfq_io_context *))
+static void __call_for_each_cic(struct io_context *ioc, cic_call_fn *func)
{
struct cfq_io_context *cic;
struct hlist_node *n;
@@ -2560,9 +2560,7 @@
/*
* Call func for each cic attached to this ioc.
*/
-static void
-call_for_each_cic(struct io_context *ioc,
- void (*func)(struct io_context *, struct cfq_io_context *))
+static void call_for_each_cic(struct io_context *ioc, cic_call_fn *func)
{
rcu_read_lock();
__call_for_each_cic(ioc, func);
@@ -2787,13 +2785,10 @@
{
struct cfq_data *cfqd = cic_to_cfqd(cic);
struct cfq_queue *cfqq;
- unsigned long flags;
if (unlikely(!cfqd))
return;
- spin_lock_irqsave(cfqd->queue->queue_lock, flags);
-
cfqq = cic->cfqq[BLK_RW_ASYNC];
if (cfqq) {
struct cfq_queue *new_cfqq;
@@ -2808,8 +2803,6 @@
cfqq = cic->cfqq[BLK_RW_SYNC];
if (cfqq)
cfq_mark_cfqq_prio_changed(cfqq);
-
- spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
}
static void cfq_ioc_set_ioprio(struct io_context *ioc)
@@ -3057,11 +3050,8 @@
radix_tree_preload_end();
- if (!ret) {
- spin_lock_irqsave(cfqd->queue->queue_lock, flags);
+ if (!ret)
list_add(&cic->queue_list, &cfqd->cic_list);
- spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
- }
}
if (ret)
@@ -3081,8 +3071,6 @@
struct io_context *ioc = NULL;
struct cfq_io_context *cic;
- might_sleep_if(gfp_mask & __GFP_WAIT);
-
ioc = get_io_context(gfp_mask, cfqd->queue->node);
if (!ioc)
return NULL;
@@ -3633,14 +3621,10 @@
const int rw = rq_data_dir(rq);
const bool is_sync = rq_is_sync(rq);
struct cfq_queue *cfqq;
- unsigned long flags;
might_sleep_if(gfp_mask & __GFP_WAIT);
cic = cfq_get_io_context(cfqd, gfp_mask);
-
- spin_lock_irqsave(q->queue_lock, flags);
-
if (!cic)
goto queue_fail;
@@ -3673,8 +3657,6 @@
cfqq->allocated[rw]++;
atomic_inc(&cfqq->ref);
- spin_unlock_irqrestore(q->queue_lock, flags);
-
rq->elevator_private = cic;
rq->elevator_private2 = cfqq;
rq->elevator_private3 = cfq_ref_get_cfqg(cfqq->cfqg);
@@ -3685,7 +3667,6 @@
put_io_context(cic->ioc);
cfq_schedule_dispatch(cfqd);
- spin_unlock_irqrestore(q->queue_lock, flags);
cfq_log(cfqd, "set_request fail");
return 1;
}
