From 9cc63f8bcd560c760d0b12e15bba8f81c86237cf Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 14 Aug 2026 15:43:51 -0700 Subject: Merge branches 'expcb.2026.07.24a', 'misc.2026.07.30a', 'rcu-tasks.2026.07.30a', 'srcu.2026.08.11a' and 'torture.2026.08.14a' into HEAD Changes: Make expedited grace periods expedite normal RCU callbacks Miscellaneous fixes: * Improve diagnostic output with character task states. * Mark accesses to inform KCSAN of concurrency design. * Move from kmalloc() to kmalloc_obj(). * Documentation updates. * Improve handling of RCU deferred quiescent states. * Clean up unused function arguments and structure fields. * Reduce show_rcu_gp_kthreads() stack space. Tasks RCU updates: * Clean up after SRCU re-implementation of Tasks Trace RCU. * Mark accesses to inform KCSAN of concurrency design. * Add ->lazy_timer status to diagnostic output. * Remove an unnecessary memory barrier. * Fix a data race, courtesy of KCSAN. * Documentation updates. * Convert cond_resched_tasks_rcu_qs() from macro to static inline function. SRCU updates: * Add Rust helpers for SRCU. * Avoid losing queued work at cleanup_srcu_struct() time. Torture-test updates: * Preparation work for immediate RCU priority deboosting. * Test RCU readers from real interrupt handlers (as opposed to softirq). * Simplify code through use of cpumask_next_wrap(). * Improve diagnostic output with character task states. * Add rcutorture.nwriters parameter to allow lightweight stall testing, and rcutorture.stall_only to make doing so easier. * Test an RCU Tasks Trace grace period implying an RCU grace period. * Make RCU Tasks Trace torturing track reader batches. * Fix a data race, courtesy of KCSAN. * Plug a shuffle_tmp_mask memory leak on kthread spawn failure. --- io_uring/notif.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 io_uring/notif.c (limited to 'io_uring/notif.c') diff --git a/io_uring/notif.c b/io_uring/notif.c new file mode 100644 index 000000000..efce8ae12 --- /dev/null +++ b/io_uring/notif.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include "io_uring.h" +#include "notif.h" +#include "rsrc.h" + +static const struct ubuf_info_ops io_ubuf_ops; + +static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw) +{ + struct io_kiocb *notif = tw_req.req; + struct io_notif_data *nd = io_notif_to_data(notif); + struct io_ring_ctx *ctx = notif->ctx; + + lockdep_assert_held(&ctx->uring_lock); + + do { + notif = cmd_to_io_kiocb(nd); + + if (WARN_ON_ONCE(ctx != notif->ctx)) + return; + lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0); + + if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used)) + notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; + + if (nd->account_pages && notif->ctx->user) { + __io_unaccount_mem(notif->ctx->user, nd->account_pages); + nd->account_pages = 0; + } + + nd = nd->next; + io_req_task_complete((struct io_tw_req){notif}, tw); + } while (nd); +} + +void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) +{ + struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); + struct io_kiocb *notif = cmd_to_io_kiocb(nd); + unsigned tw_flags; + + if (nd->zc_report) { + if (success && !nd->zc_used && skb) + WRITE_ONCE(nd->zc_used, true); + else if (!success && !nd->zc_copied) + WRITE_ONCE(nd->zc_copied, true); + } + + if (!refcount_dec_and_test(&uarg->refcnt)) + return; + + if (nd->head != nd) { + io_tx_ubuf_complete(skb, &nd->head->uarg, success); + return; + } + + tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE; + notif->io_task_work.func = io_notif_tw_complete; + __io_req_task_work_add(notif, tw_flags); +} + +static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg) +{ + struct io_notif_data *nd, *prev_nd; + struct io_kiocb *prev_notif, *notif; + struct ubuf_info *prev_uarg = skb_zcopy(skb); + + nd = container_of(uarg, struct io_notif_data, uarg); + notif = cmd_to_io_kiocb(nd); + + if (!prev_uarg) { + net_zcopy_get(&nd->uarg); + skb_zcopy_init(skb, &nd->uarg); + return 0; + } + /* handle it separately as we can't link a notif to itself */ + if (unlikely(prev_uarg == &nd->uarg)) + return 0; + /* we can't join two links together, just request a fresh skb */ + if (unlikely(nd->head != nd || nd->next)) + return -EEXIST; + /* don't mix zc providers */ + if (unlikely(prev_uarg->ops != &io_ubuf_ops)) + return -EEXIST; + + prev_nd = container_of(prev_uarg, struct io_notif_data, uarg); + prev_notif = cmd_to_io_kiocb(prev_nd); + + /* make sure all notifications can be finished in the same task_work */ + if (unlikely(notif->ctx != prev_notif->ctx || + notif->tctx != prev_notif->tctx)) + return -EEXIST; + + nd->head = prev_nd->head; + nd->next = prev_nd->next; + prev_nd->next = nd; + net_zcopy_get(&nd->head->uarg); + return 0; +} + +static const struct ubuf_info_ops io_ubuf_ops = { + .complete = io_tx_ubuf_complete, + .link_skb = io_link_skb, +}; + +struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + struct io_kiocb *notif; + struct io_notif_data *nd; + + if (unlikely(!io_alloc_req(ctx, ¬if))) + return NULL; + notif->ctx = ctx; + notif->opcode = IORING_OP_NOP; + notif->flags = 0; + notif->file = NULL; + notif->tctx = current->io_uring; + io_get_task_refs(1); + notif->file_node = NULL; + notif->buf_node = NULL; + + nd = io_notif_to_data(notif); + nd->zc_report = false; + nd->account_pages = 0; + nd->next = NULL; + nd->head = nd; + + nd->uarg.flags = IO_NOTIF_UBUF_FLAGS; + nd->uarg.ops = &io_ubuf_ops; + refcount_set(&nd->uarg.refcnt, 1); + return notif; +} -- cgit v1.3.1