| From 880668254d7a1af40a1d61935c320ef6aa09c19f Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Sat, 1 Feb 2020 21:30:11 -0700 |
| Subject: io_uring: prevent potential eventfd recursion on poll |
| |
| From: Jens Axboe <axboe@kernel.dk> |
| |
| [ Upstream commit f0b493e6b9a8959356983f57112229e69c2f7b8c ] |
| |
| If we have nested or circular eventfd wakeups, then we can deadlock if |
| we run them inline from our poll waitqueue wakeup handler. It's also |
| possible to have very long chains of notifications, to the extent where |
| we could risk blowing the stack. |
| |
| Check the eventfd recursion count before calling eventfd_signal(). If |
| it's non-zero, then punt the signaling to async context. This is always |
| safe, as it takes us out-of-line in terms of stack and locking context. |
| |
| Cc: stable@vger.kernel.org # 5.1+ |
| Signed-off-by: Jens Axboe <axboe@kernel.dk> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| fs/io_uring.c | 35 +++++++++++++++++++++++++++++------ |
| 1 file changed, 29 insertions(+), 6 deletions(-) |
| |
| diff --git a/fs/io_uring.c b/fs/io_uring.c |
| index 131087782bec9..f470fb21467e4 100644 |
| --- a/fs/io_uring.c |
| +++ b/fs/io_uring.c |
| @@ -738,21 +738,28 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) |
| |
| static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) |
| { |
| + if (!ctx->cq_ev_fd) |
| + return false; |
| if (!ctx->eventfd_async) |
| return true; |
| return io_wq_current_is_worker() || in_interrupt(); |
| } |
| |
| -static void io_cqring_ev_posted(struct io_ring_ctx *ctx) |
| +static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev) |
| { |
| if (waitqueue_active(&ctx->wait)) |
| wake_up(&ctx->wait); |
| if (waitqueue_active(&ctx->sqo_wait)) |
| wake_up(&ctx->sqo_wait); |
| - if (ctx->cq_ev_fd && io_should_trigger_evfd(ctx)) |
| + if (trigger_ev) |
| eventfd_signal(ctx->cq_ev_fd, 1); |
| } |
| |
| +static void io_cqring_ev_posted(struct io_ring_ctx *ctx) |
| +{ |
| + __io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx)); |
| +} |
| + |
| /* Returns true if there are no backlogged entries after the flush */ |
| static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) |
| { |
| @@ -2645,6 +2652,14 @@ static void io_poll_complete_work(struct io_wq_work **workptr) |
| io_wq_assign_next(workptr, nxt); |
| } |
| |
| +static void io_poll_trigger_evfd(struct io_wq_work **workptr) |
| +{ |
| + struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); |
| + |
| + eventfd_signal(req->ctx->cq_ev_fd, 1); |
| + io_put_req(req); |
| +} |
| + |
| static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, |
| void *key) |
| { |
| @@ -2667,13 +2682,21 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, |
| * for finalizing the request, mark us as having grabbed that already. |
| */ |
| if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) { |
| + bool trigger_ev; |
| + |
| hash_del(&req->hash_node); |
| io_poll_complete(req, mask, 0); |
| - req->flags |= REQ_F_COMP_LOCKED; |
| - io_put_req(req); |
| + trigger_ev = io_should_trigger_evfd(ctx); |
| + if (trigger_ev && eventfd_signal_count()) { |
| + trigger_ev = false; |
| + req->work.func = io_poll_trigger_evfd; |
| + } else { |
| + req->flags |= REQ_F_COMP_LOCKED; |
| + io_put_req(req); |
| + req = NULL; |
| + } |
| spin_unlock_irqrestore(&ctx->completion_lock, flags); |
| - |
| - io_cqring_ev_posted(ctx); |
| + __io_cqring_ev_posted(ctx, trigger_ev); |
| } else { |
| io_queue_async_work(req); |
| } |
| -- |
| 2.20.1 |
| |