| From 9b7adba9eaec28e0e4343c96d0dbeb9578802f5f Mon Sep 17 00:00:00 2001 |
| From: Jens Axboe <axboe@kernel.dk> |
| Date: Mon, 10 Aug 2020 10:54:02 -0600 |
| Subject: io_uring: add missing REQ_F_COMP_LOCKED for nested requests |
| |
| From: Jens Axboe <axboe@kernel.dk> |
| |
| commit 9b7adba9eaec28e0e4343c96d0dbeb9578802f5f upstream. |
| |
| When we traverse into failing links or timeouts, we need to ensure we |
| propagate the REQ_F_COMP_LOCKED flag to ensure that we correctly signal |
| to the completion side that we already hold the completion lock. |
| |
| Signed-off-by: Jens Axboe <axboe@kernel.dk> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| |
| --- |
| fs/io_uring.c | 24 +++++++++++++++++++----- |
| 1 file changed, 19 insertions(+), 5 deletions(-) |
| |
| --- a/fs/io_uring.c |
| +++ b/fs/io_uring.c |
| @@ -1484,12 +1484,9 @@ static void io_req_link_next(struct io_k |
| /* |
| * Called if REQ_F_LINK_HEAD is set, and we fail the head request |
| */ |
| -static void io_fail_links(struct io_kiocb *req) |
| +static void __io_fail_links(struct io_kiocb *req) |
| { |
| struct io_ring_ctx *ctx = req->ctx; |
| - unsigned long flags; |
| - |
| - spin_lock_irqsave(&ctx->completion_lock, flags); |
| |
| while (!list_empty(&req->link_list)) { |
| struct io_kiocb *link = list_first_entry(&req->link_list, |
| @@ -1503,13 +1500,29 @@ static void io_fail_links(struct io_kioc |
| io_link_cancel_timeout(link); |
| } else { |
| io_cqring_fill_event(link, -ECANCELED); |
| + link->flags |= REQ_F_COMP_LOCKED; |
| __io_double_put_req(link); |
| } |
| req->flags &= ~REQ_F_LINK_TIMEOUT; |
| } |
| |
| io_commit_cqring(ctx); |
| - spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| +} |
| + |
| +static void io_fail_links(struct io_kiocb *req) |
| +{ |
| + struct io_ring_ctx *ctx = req->ctx; |
| + |
| + if (!(req->flags & REQ_F_COMP_LOCKED)) { |
| + unsigned long flags; |
| + |
| + spin_lock_irqsave(&ctx->completion_lock, flags); |
| + __io_fail_links(req); |
| + spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| + } else { |
| + __io_fail_links(req); |
| + } |
| + |
| io_cqring_ev_posted(ctx); |
| } |
| |
| @@ -4828,6 +4841,7 @@ static int io_timeout_cancel(struct io_r |
| return -EALREADY; |
| |
| req_set_fail_links(req); |
| + req->flags |= REQ_F_COMP_LOCKED; |
| io_cqring_fill_event(req, -ECANCELED); |
| io_put_req(req); |
| return 0; |