--- x/io_uring/poll.c +++ y/io_uring/poll.c @@ -714,6 +714,8 @@ int io_arm_poll_handler(struct io_kiocb io_kbuf_recycle(req, issue_flags); + get_file(req->file); + req->poll_file = req->file; ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags); if (ret) return ret > 0 ? IO_APOLL_READY : IO_APOLL_ABORTED; @@ -943,6 +945,8 @@ int io_poll_add(struct io_kiocb *req, un if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER)) req->flags |= REQ_F_HASH_LOCKED; + get_file(req->file); + req->poll_file = req->file; ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags); if (ret > 0) { io_req_set_res(req, ipt.result_mask, 0); --- x/include/linux/io_uring_types.h +++ y/include/linux/io_uring_types.h @@ -578,6 +578,8 @@ struct io_kiocb { struct io_kiocb *link; /* custom credentials, valid IFF REQ_F_CREDS is set */ const struct cred *creds; + struct file *poll_file; + int in_free_cache; struct io_wq_work work; }; --- x/io_uring/io_uring.h +++ y/io_uring/io_uring.h @@ -352,9 +352,13 @@ static inline bool io_alloc_req_refill(s static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_wq_work_node *node; + struct io_kiocb *req; node = wq_stack_extract(&ctx->submit_state.free_list); - return container_of(node, struct io_kiocb, comp_list); + req = container_of(node, struct io_kiocb, comp_list); + BUG_ON(req->in_free_cache == 0); + req->in_free_cache = 0; + return req; } static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx) --- x/io_uring/io_uring.c +++ y/io_uring/io_uring.c @@ -229,6 +229,8 @@ static inline void req_fail_link_node(st static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) { + BUG_ON(req->in_free_cache != 0); + req->in_free_cache = 1; wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); } @@ -975,7 +977,8 @@ static void __io_req_complete_post(struc */ io_put_kbuf_comp(req); io_dismantle_req(req); - io_put_task(req->task, 1); + if (req->poll_file) + fput(req->poll_file); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; } @@ -1024,6 +1027,7 @@ static void io_preinit_req(struct io_kio req->async_data = NULL; /* not necessary, but safer to zero */ req->cqe.res = 0; + req->in_free_cache = 0; } static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, @@ -1099,6 +1103,8 @@ __cold void io_free_req(struct io_kiocb io_req_put_rsrc(req); io_dismantle_req(req); io_put_task(req->task, 1); + if (req->poll_file) + fput(req->poll_file); spin_lock(&ctx->completion_lock); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); @@ -1296,12 +1302,15 @@ void __io_req_task_work_add(struct io_ki if (!llist_add(&req->io_task_work.node, &tctx->task_list)) return; + if (!(req->flags & REQ_F_REFCOUNT)) + goto out; + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method))) return; - +out: io_fallback_tw(tctx); } @@ -1421,16 +1430,17 @@ void io_free_batch_list(struct io_ring_c struct task_struct *task = NULL; int task_refs = 0; - do { - struct io_kiocb *req = container_of(node, struct io_kiocb, - comp_list); + while (node) { + struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); + int put = 0; + + node = req->comp_list.next; if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) { - if (req->flags & REQ_F_REFCOUNT) { - node = req->comp_list.next; - if (!req_ref_put_and_test(req)) - continue; - } + put = 1; + if (!req_ref_put_and_test(req)) + continue; + if ((req->flags & REQ_F_POLLED) && req->apoll) { struct async_poll *apoll = req->apoll; @@ -1457,9 +1467,9 @@ void io_free_batch_list(struct io_ring_c task_refs = 0; } task_refs++; - node = req->comp_list.next; - io_req_add_to_cache(req, ctx); - } while (node); + if (put == 0 && req_ref_put_and_test(req)) + io_req_add_to_cache(req, ctx); + } if (task) io_put_task(task, task_refs); @@ -2140,6 +2150,8 @@ static int io_init_req(struct io_ring_ct req->flags = sqe_flags = READ_ONCE(sqe->flags); req->cqe.user_data = READ_ONCE(sqe->user_data); req->file = NULL; + req->poll_file = NULL; + // req->in_free_cache = 0; req->rsrc_node = NULL; req->task = current; @@ -2728,6 +2740,7 @@ static void io_req_caches_free(struct io { int nr = 0; + flush_delayed_work(&ctx->fallback_work); mutex_lock(&ctx->uring_lock); io_flush_cached_locked_reqs(ctx, &ctx->submit_state);