| From 04c363099ee47fe5baf8a0951a398ea8651366a2 Mon Sep 17 00:00:00 2001 |
| From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| Date: Thu, 29 Jan 2015 15:10:08 +0100 |
| Subject: [PATCH] block/mq: don't complete requests via IPI |
| |
| The IPI runs in hardirq context and there are sleeping locks. This patch |
| moves the completion into a workqueue. |
| |
| Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| |
| diff --git a/block/blk-core.c b/block/blk-core.c |
| index 895a92f9ea62..04a7d3edfdfb 100644 |
| --- a/block/blk-core.c |
| +++ b/block/blk-core.c |
| @@ -125,6 +125,9 @@ void blk_rq_init(struct request_queue *q, struct request *rq) |
| |
| INIT_LIST_HEAD(&rq->queuelist); |
| INIT_LIST_HEAD(&rq->timeout_list); |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work); |
| +#endif |
| rq->cpu = -1; |
| rq->q = q; |
| rq->__sector = (sector_t) -1; |
| diff --git a/block/blk-mq.c b/block/blk-mq.c |
| index bbe37088b9f4..4e76bab012e3 100644 |
| --- a/block/blk-mq.c |
| +++ b/block/blk-mq.c |
| @@ -197,6 +197,9 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, |
| rq->resid_len = 0; |
| rq->sense = NULL; |
| |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + INIT_WORK(&rq->work, __blk_mq_complete_request_remote_work); |
| +#endif |
| INIT_LIST_HEAD(&rq->timeout_list); |
| rq->timeout = 0; |
| |
| @@ -324,6 +327,17 @@ void blk_mq_end_request(struct request *rq, int error) |
| } |
| EXPORT_SYMBOL(blk_mq_end_request); |
| |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + |
| +void __blk_mq_complete_request_remote_work(struct work_struct *work) |
| +{ |
| + struct request *rq = container_of(work, struct request, work); |
| + |
| + rq->q->softirq_done_fn(rq); |
| +} |
| + |
| +#else |
| + |
| static void __blk_mq_complete_request_remote(void *data) |
| { |
| struct request *rq = data; |
| @@ -331,6 +345,8 @@ static void __blk_mq_complete_request_remote(void *data) |
| rq->q->softirq_done_fn(rq); |
| } |
| |
| +#endif |
| + |
| static void blk_mq_ipi_complete_request(struct request *rq) |
| { |
| struct blk_mq_ctx *ctx = rq->mq_ctx; |
| @@ -347,10 +363,14 @@ static void blk_mq_ipi_complete_request(struct request *rq) |
| shared = cpus_share_cache(cpu, ctx->cpu); |
| |
| if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| + schedule_work_on(ctx->cpu, &rq->work); |
| +#else |
| rq->csd.func = __blk_mq_complete_request_remote; |
| rq->csd.info = rq; |
| rq->csd.flags = 0; |
| smp_call_function_single_async(ctx->cpu, &rq->csd); |
| +#endif |
| } else { |
| rq->q->softirq_done_fn(rq); |
| } |
| diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h |
| index 2498fdf3a503..2473b2cd2c75 100644 |
| --- a/include/linux/blk-mq.h |
| +++ b/include/linux/blk-mq.h |
| @@ -218,6 +218,7 @@ static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) |
| |
| struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); |
| struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); |
| +void __blk_mq_complete_request_remote_work(struct work_struct *work); |
| |
| int blk_mq_request_started(struct request *rq); |
| void blk_mq_start_request(struct request *rq); |
| diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h |
| index 48f05d768a53..6d51128b68bd 100644 |
| --- a/include/linux/blkdev.h |
| +++ b/include/linux/blkdev.h |
| @@ -90,6 +90,7 @@ struct request { |
| struct list_head queuelist; |
| union { |
| struct call_single_data csd; |
| + struct work_struct work; |
| u64 fifo_time; |
| }; |
| |
| -- |
| 2.5.0 |
| |