Merge branch 'for-6.4/io_uring' into for-next
* for-6.4/io_uring:
io_uring/rsrc: extract SCM file put helper
io_uring/rsrc: refactor io_rsrc_node_switch
io_uring/rsrc: zero node's rsrc data on alloc
io_uring/rsrc: consolidate node caching
io_uring/rsrc: add lockdep checks
io_uring: add irq lockdep checks
io_uring/kbuf: remove extra ->buf_ring null check
io_uring: shut io_prep_async_work warning
io_uring/uring_cmd: take advantage of completion batching
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index fa621a5..40cab42 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -326,7 +326,6 @@ struct io_ring_ctx {
struct io_restriction restrictions;
/* slow path rsrc auxilary data, used by update/register */
- struct io_rsrc_node *rsrc_backup_node;
struct io_mapped_ubuf *dummy_ubuf;
struct io_rsrc_data *file_data;
struct io_rsrc_data *buf_data;
diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h
index 851a527..241245c 100644
--- a/io_uring/alloc_cache.h
+++ b/io_uring/alloc_cache.h
@@ -23,6 +23,11 @@ static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,
return false;
}
+static inline bool io_alloc_cache_empty(struct io_alloc_cache *cache)
+{
+ return !cache->list.next;
+}
+
static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cache)
{
if (cache->list.next) {
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 9bbf582..9083a84 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -425,7 +425,7 @@ static void io_prep_async_work(struct io_kiocb *req)
if (req->file && !io_req_ffs_set(req))
req->flags |= io_file_get_flags(req->file) << REQ_F_SUPPORT_NOWAIT_BIT;
- if (req->flags & REQ_F_ISREG) {
+ if (req->file && (req->flags & REQ_F_ISREG)) {
bool should_hash = def->hash_reg_file;
/* don't serialize this request if the fs doesn't need it */
@@ -2852,8 +2852,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
/* there are no registered resources left, nobody uses it */
if (ctx->rsrc_node)
io_rsrc_node_destroy(ctx, ctx->rsrc_node);
- if (ctx->rsrc_backup_node)
- io_rsrc_node_destroy(ctx, ctx->rsrc_backup_node);
WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
@@ -3883,11 +3881,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
ret = io_sq_offload_create(ctx, p);
if (ret)
goto err;
- /* always set a rsrc node */
- ret = io_rsrc_node_switch_start(ctx);
+
+ ret = io_rsrc_init(ctx);
if (ret)
goto err;
- io_rsrc_node_switch(ctx, NULL);
memset(&p->sq_off, 0, sizeof(p->sq_off));
p->sq_off.head = offsetof(struct io_rings, sq.head);
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index ef449e4..25515d6 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -94,6 +94,8 @@ bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
#define io_lockdep_assert_cq_locked(ctx) \
do { \
+ lockdep_assert(in_task()); \
+ \
if (ctx->flags & IORING_SETUP_IOPOLL) { \
lockdep_assert_held(&ctx->uring_lock); \
} else if (!ctx->task_complete) { \
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 79c2545..0905c17 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -218,14 +218,12 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx,
if (bl->is_mapped) {
i = bl->buf_ring->tail - bl->head;
if (bl->is_mmap) {
- if (bl->buf_ring) {
- struct page *page;
+ struct page *page;
- page = virt_to_head_page(bl->buf_ring);
- if (put_page_testzero(page))
- free_compound_page(page);
- bl->buf_ring = NULL;
- }
+ page = virt_to_head_page(bl->buf_ring);
+ if (put_page_testzero(page))
+ free_compound_page(page);
+ bl->buf_ring = NULL;
bl->is_mmap = 0;
} else if (bl->buf_nr_pages) {
int j;
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 603a783..11058e2 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -204,7 +204,7 @@ void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
}
}
-static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
+struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
{
struct io_rsrc_node *ref_node;
struct io_cache_entry *entry;
@@ -218,6 +218,7 @@ static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
return NULL;
}
+ ref_node->rsrc_data = NULL;
ref_node->refs = 1;
INIT_LIST_HEAD(&ref_node->node);
INIT_LIST_HEAD(&ref_node->item_list);
@@ -230,33 +231,30 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx,
struct io_rsrc_data *data_to_kill)
__must_hold(&ctx->uring_lock)
{
- WARN_ON_ONCE(!ctx->rsrc_backup_node);
- WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
+ struct io_rsrc_node *node = ctx->rsrc_node;
+ struct io_rsrc_node *backup = io_rsrc_node_alloc(ctx);
- if (data_to_kill) {
- struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
+ if (WARN_ON_ONCE(!backup))
+ return;
- rsrc_node->rsrc_data = data_to_kill;
- list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
-
- data_to_kill->refs++;
- /* put master ref */
- io_put_rsrc_node(ctx, rsrc_node);
- ctx->rsrc_node = NULL;
- }
-
- if (!ctx->rsrc_node) {
- ctx->rsrc_node = ctx->rsrc_backup_node;
- ctx->rsrc_backup_node = NULL;
- }
+ data_to_kill->refs++;
+ node->rsrc_data = data_to_kill;
+ list_add_tail(&node->node, &ctx->rsrc_ref_list);
+ /* put master ref */
+ io_put_rsrc_node(ctx, node);
+ ctx->rsrc_node = backup;
}
int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
{
- if (ctx->rsrc_backup_node)
- return 0;
- ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
- return ctx->rsrc_backup_node ? 0 : -ENOMEM;
+ if (io_alloc_cache_empty(&ctx->rsrc_node_cache)) {
+ struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
+
+ if (!node)
+ return -ENOMEM;
+ io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
+ }
+ return 0;
}
__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
@@ -534,6 +532,8 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
__u32 tmp;
int err;
+ lockdep_assert_held(&ctx->uring_lock);
+
if (check_add_overflow(up->offset, nr_args, &tmp))
return -EOVERFLOW;
err = io_rsrc_node_switch_start(ctx);
@@ -832,20 +832,14 @@ int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file)
return 0;
}
-static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
+static __cold void io_rsrc_file_scm_put(struct io_ring_ctx *ctx, struct file *file)
{
- struct file *file = prsrc->file;
#if defined(CONFIG_UNIX)
struct sock *sock = ctx->ring_sock->sk;
struct sk_buff_head list, *head = &sock->sk_receive_queue;
struct sk_buff *skb;
int i;
- if (!io_file_need_scm(file)) {
- fput(file);
- return;
- }
-
__skb_queue_head_init(&list);
/*
@@ -895,11 +889,19 @@ static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
__skb_queue_tail(head, skb);
spin_unlock_irq(&head->lock);
}
-#else
- fput(file);
#endif
}
+static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
+{
+ struct file *file = prsrc->file;
+
+ if (likely(!io_file_need_scm(file)))
+ fput(file);
+ else
+ io_rsrc_file_scm_put(ctx, file);
+}
+
int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
unsigned nr_args, u64 __user *tags)
{
@@ -916,9 +918,6 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
return -EMFILE;
if (nr_args > rlimit(RLIMIT_NOFILE))
return -EMFILE;
- ret = io_rsrc_node_switch_start(ctx);
- if (ret)
- return ret;
ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
&ctx->file_data);
if (ret)
@@ -973,7 +972,6 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
/* default it to the whole table */
io_file_table_set_alloc_range(ctx, 0, ctx->nr_user_files);
- io_rsrc_node_switch(ctx, NULL);
return 0;
fail:
__io_sqe_files_unregister(ctx);
@@ -1255,9 +1253,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
return -EBUSY;
if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
return -EINVAL;
- ret = io_rsrc_node_switch_start(ctx);
- if (ret)
- return ret;
ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
if (ret)
return ret;
@@ -1295,8 +1290,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
ctx->buf_data = data;
if (ret)
__io_sqe_buffers_unregister(ctx);
- else
- io_rsrc_node_switch(ctx, NULL);
return ret;
}
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 8729f2f..17dfe18 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -74,6 +74,7 @@ void io_rsrc_put_work(struct work_struct *work);
void io_wait_rsrc_data(struct io_rsrc_data *data);
void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
+struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
struct io_rsrc_node *node, void *rsrc);
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
@@ -164,6 +165,12 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
return &data->tags[table_idx][off];
}
+static inline int io_rsrc_init(struct io_ring_ctx *ctx)
+{
+ ctx->rsrc_node = io_rsrc_node_alloc(ctx);
+ return ctx->rsrc_node ? 0 : -ENOMEM;
+}
+
int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index f7a96bc7..5113c9a 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -54,11 +54,15 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2,
io_req_set_res(req, ret, 0);
if (req->ctx->flags & IORING_SETUP_CQE32)
io_req_set_cqe32_extra(req, res2, 0);
- if (req->ctx->flags & IORING_SETUP_IOPOLL)
+ if (req->ctx->flags & IORING_SETUP_IOPOLL) {
/* order with io_iopoll_req_issued() checking ->iopoll_complete */
smp_store_release(&req->iopoll_completed, 1);
- else
- io_req_complete_post(req, issue_flags);
+ } else {
+ struct io_tw_state ts = {
+ .locked = !(issue_flags & IO_URING_F_UNLOCKED),
+ };
+ io_req_task_complete(req, &ts);
+ }
}
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
