diff options
| author | Paul E. McKenney <paulmck@kernel.org> | 2026-08-14 15:43:51 -0700 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@kernel.org> | 2026-08-14 15:43:51 -0700 |
| commit | 9cc63f8bcd560c760d0b12e15bba8f81c86237cf (patch) | |
| tree | 3b5269b1bcf6714171754946367c51e0590f64da /lib/once.c | |
| download | linux-stable-9cc63f8bcd560c760d0b12e15bba8f81c86237cf.tar.gz linux-stable-9cc63f8bcd560c760d0b12e15bba8f81c86237cf.zip | |
Merge branches 'expcb.2026.07.24a', 'misc.2026.07.30a', 'rcu-tasks.2026.07.30a', 'srcu.2026.08.11a' and 'torture.2026.08.14a' into HEADgrafted
Changes:
Make expedited grace periods expedite normal RCU callbacks
Miscellaneous fixes:
* Improve diagnostic output with character task states.
* Mark accesses to inform KCSAN of concurrency design.
* Move from kmalloc() to kmalloc_obj().
* Documentation updates.
* Improve handling of RCU deferred quiescent states.
* Clean up unused function arguments and structure fields.
* Reduce show_rcu_gp_kthreads() stack space.
Tasks RCU updates:
* Clean up after SRCU re-implementation of Tasks Trace RCU.
* Mark accesses to inform KCSAN of concurrency design.
* Add ->lazy_timer status to diagnostic output.
* Remove an unnecessary memory barrier.
* Fix a data race, courtesy of KCSAN.
* Documentation updates.
* Convert cond_resched_tasks_rcu_qs() from macro to static inline
function.
SRCU updates:
* Add Rust helpers for SRCU.
* Avoid losing queued work at cleanup_srcu_struct() time.
Torture-test updates:
* Preparation work for immediate RCU priority deboosting.
* Test RCU readers from real interrupt handlers (as opposed to softirq).
* Simplify code through use of cpumask_next_wrap().
* Improve diagnostic output with character task states.
* Add rcutorture.nwriters parameter to allow lightweight stall testing,
and rcutorture.stall_only to make doing so easier.
* Test an RCU Tasks Trace grace period implying an RCU grace period.
* Make RCU Tasks Trace torturing track reader batches.
* Fix a data race, courtesy of KCSAN.
* Plug a shuffle_tmp_mask memory leak on kthread spawn failure.
Diffstat (limited to 'lib/once.c')
| -rw-r--r-- | lib/once.c | 98 |
1 files changed, 98 insertions, 0 deletions
diff --git a/lib/once.c b/lib/once.c new file mode 100644 index 000000000..d801bfa94 --- /dev/null +++ b/lib/once.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/once.h> +#include <linux/random.h> +#include <linux/module.h> + +struct once_work { + struct work_struct work; + struct static_key_true *key; + struct module *module; +}; + +static void once_deferred(struct work_struct *w) +{ + struct once_work *work; + + work = container_of(w, struct once_work, work); + BUG_ON(!static_key_enabled(work->key)); + static_branch_disable(work->key); + module_put(work->module); + kfree(work); +} + +static void once_disable_jump(struct static_key_true *key, struct module *mod) +{ + struct once_work *w; + + w = kmalloc_obj(*w, GFP_ATOMIC); + if (!w) + return; + + INIT_WORK(&w->work, once_deferred); + w->key = key; + w->module = mod; + __module_get(mod); + schedule_work(&w->work); +} + +static DEFINE_SPINLOCK(once_lock); + +bool __do_once_start(bool *done, unsigned long *flags) + __acquires(once_lock) +{ + spin_lock_irqsave(&once_lock, *flags); + if (*done) { + spin_unlock_irqrestore(&once_lock, *flags); + /* Keep sparse happy by restoring an even lock count on + * this lock. In case we return here, we don't call into + * __do_once_done but return early in the DO_ONCE() macro. + */ + __acquire(once_lock); + return false; + } + + return true; +} +EXPORT_SYMBOL(__do_once_start); + +void __do_once_done(bool *done, struct static_key_true *once_key, + unsigned long *flags, struct module *mod) + __releases(once_lock) +{ + *done = true; + spin_unlock_irqrestore(&once_lock, *flags); + once_disable_jump(once_key, mod); +} +EXPORT_SYMBOL(__do_once_done); + +static DEFINE_MUTEX(once_mutex); + +bool __do_once_sleepable_start(bool *done) + __acquires(once_mutex) +{ + mutex_lock(&once_mutex); + if (*done) { + mutex_unlock(&once_mutex); + /* Keep sparse happy by restoring an even lock count on + * this mutex. In case we return here, we don't call into + * __do_once_done but return early in the DO_ONCE_SLEEPABLE() macro. + */ + __acquire(once_mutex); + return false; + } + + return true; +} +EXPORT_SYMBOL(__do_once_sleepable_start); + +void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, + struct module *mod) + __releases(once_mutex) +{ + *done = true; + mutex_unlock(&once_mutex); + static_branch_disable(once_key); +} +EXPORT_SYMBOL(__do_once_sleepable_done); |
