| From d4e7cd36a90e38e0276d6ce0c20f5ccef17ec38c Mon Sep 17 00:00:00 2001 |
| From: Jens Axboe <axboe@kernel.dk> |
| Date: Sat, 15 Aug 2020 11:44:50 -0700 |
| Subject: io_uring: sanitize double poll handling |
| |
| From: Jens Axboe <axboe@kernel.dk> |
| |
| commit d4e7cd36a90e38e0276d6ce0c20f5ccef17ec38c upstream. |
| |
| There's a bit of confusion on the matching pairs of poll vs double poll, |
| depending on if the request is a pure poll (IORING_OP_POLL_ADD) or |
| poll driven retry. |
| |
| Add io_poll_get_double() that returns the double poll waitqueue, if any, |
| and io_poll_get_single() that returns the original poll waitqueue. With |
| that, remove the argument to io_poll_remove_double(). |
| |
| Finally ensure that wait->private is cleared once the double poll handler |
| has run, so that remove knows it's already been seen. |
| |
| Cc: stable@vger.kernel.org # v5.8 |
| Reported-by: syzbot+7f617d4a9369028b8a2c@syzkaller.appspotmail.com |
| Fixes: 18bceab101ad ("io_uring: allow POLL_ADD with double poll_wait() users") |
| Signed-off-by: Jens Axboe <axboe@kernel.dk> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| |
| --- |
| fs/io_uring.c | 32 ++++++++++++++++++++++++-------- |
| 1 file changed, 24 insertions(+), 8 deletions(-) |
| |
| --- a/fs/io_uring.c |
| +++ b/fs/io_uring.c |
| @@ -4233,9 +4233,24 @@ static bool io_poll_rewait(struct io_kio |
| return false; |
| } |
| |
| -static void io_poll_remove_double(struct io_kiocb *req, void *data) |
| +static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req) |
| { |
| - struct io_poll_iocb *poll = data; |
| + /* pure poll stashes this in ->io, poll driven retry elsewhere */ |
| + if (req->opcode == IORING_OP_POLL_ADD) |
| + return (struct io_poll_iocb *) req->io; |
| + return req->apoll->double_poll; |
| +} |
| + |
| +static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req) |
| +{ |
| + if (req->opcode == IORING_OP_POLL_ADD) |
| + return &req->poll; |
| + return &req->apoll->poll; |
| +} |
| + |
| +static void io_poll_remove_double(struct io_kiocb *req) |
| +{ |
| + struct io_poll_iocb *poll = io_poll_get_double(req); |
| |
| lockdep_assert_held(&req->ctx->completion_lock); |
| |
| @@ -4255,7 +4270,7 @@ static void io_poll_complete(struct io_k |
| { |
| struct io_ring_ctx *ctx = req->ctx; |
| |
| - io_poll_remove_double(req, req->io); |
| + io_poll_remove_double(req); |
| req->poll.done = true; |
| io_cqring_fill_event(req, error ? error : mangle_poll(mask)); |
| io_commit_cqring(ctx); |
| @@ -4297,7 +4312,7 @@ static int io_poll_double_wake(struct wa |
| int sync, void *key) |
| { |
| struct io_kiocb *req = wait->private; |
| - struct io_poll_iocb *poll = req->apoll->double_poll; |
| + struct io_poll_iocb *poll = io_poll_get_single(req); |
| __poll_t mask = key_to_poll(key); |
| |
| /* for instances that support it check for an event match first: */ |
| @@ -4311,6 +4326,8 @@ static int io_poll_double_wake(struct wa |
| done = list_empty(&poll->wait.entry); |
| if (!done) |
| list_del_init(&poll->wait.entry); |
| + /* make sure double remove sees this as being gone */ |
| + wait->private = NULL; |
| spin_unlock(&poll->head->lock); |
| if (!done) |
| __io_async_wake(req, poll, mask, io_poll_task_func); |
| @@ -4545,7 +4562,7 @@ static bool io_arm_poll_handler(struct i |
| ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, |
| io_async_wake); |
| if (ret || ipt.error) { |
| - io_poll_remove_double(req, apoll->double_poll); |
| + io_poll_remove_double(req); |
| spin_unlock_irq(&ctx->completion_lock); |
| memcpy(&req->work, &apoll->work, sizeof(req->work)); |
| kfree(apoll->double_poll); |
| @@ -4578,14 +4595,13 @@ static bool io_poll_remove_one(struct io |
| { |
| bool do_complete; |
| |
| + io_poll_remove_double(req); |
| + |
| if (req->opcode == IORING_OP_POLL_ADD) { |
| - io_poll_remove_double(req, req->io); |
| do_complete = __io_poll_remove_one(req, &req->poll); |
| } else { |
| struct async_poll *apoll = req->apoll; |
| |
| - io_poll_remove_double(req, apoll->double_poll); |
| - |
| /* non-poll requests have submit ref still */ |
| do_complete = __io_poll_remove_one(req, &apoll->poll); |
| if (do_complete) { |