From 9cc63f8bcd560c760d0b12e15bba8f81c86237cf Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 14 Aug 2026 15:43:51 -0700 Subject: Merge branches 'expcb.2026.07.24a', 'misc.2026.07.30a', 'rcu-tasks.2026.07.30a', 'srcu.2026.08.11a' and 'torture.2026.08.14a' into HEAD Changes: Make expedited grace periods expedite normal RCU callbacks Miscellaneous fixes: * Improve diagnostic output with character task states. * Mark accesses to inform KCSAN of concurrency design. * Move from kmalloc() to kmalloc_obj(). * Documentation updates. * Improve handling of RCU deferred quiescent states. * Clean up unused function arguments and structure fields. * Reduce show_rcu_gp_kthreads() stack space. Tasks RCU updates: * Clean up after SRCU re-implementation of Tasks Trace RCU. * Mark accesses to inform KCSAN of concurrency design. * Add ->lazy_timer status to diagnostic output. * Remove an unnecessary memory barrier. * Fix a data race, courtesy of KCSAN. * Documentation updates. * Convert cond_resched_tasks_rcu_qs() from macro to static inline function. SRCU updates: * Add Rust helpers for SRCU. * Avoid losing queued work at cleanup_srcu_struct() time. Torture-test updates: * Preparation work for immediate RCU priority deboosting. * Test RCU readers from real interrupt handlers (as opposed to softirq). * Simplify code through use of cpumask_next_wrap(). * Improve diagnostic output with character task states. * Add rcutorture.nwriters parameter to allow lightweight stall testing, and rcutorture.stall_only to make doing so easier. * Test an RCU Tasks Trace grace period implying an RCU grace period. * Make RCU Tasks Trace torturing track reader batches. * Fix a data race, courtesy of KCSAN. * Plug a shuffle_tmp_mask memory leak on kthread spawn failure. --- include/asm-generic/qrwlock.h | 147 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 include/asm-generic/qrwlock.h (limited to 'include/asm-generic/qrwlock.h') diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h new file mode 100644 index 000000000..75b8f4601 --- /dev/null +++ b/include/asm-generic/qrwlock.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Queue read/write lock + * + * These use generic atomic and locking routines, but depend on a fair spinlock + * implementation in order to be fair themselves. The implementation in + * asm-generic/spinlock.h meets these requirements. + * + * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. + * + * Authors: Waiman Long + */ +#ifndef __ASM_GENERIC_QRWLOCK_H +#define __ASM_GENERIC_QRWLOCK_H + +#include +#include +#include + +#include + +/* Must be included from asm/spinlock.h after defining arch_spin_is_locked. */ + +/* + * Writer states & reader shift and bias. + */ +#define _QW_WAITING 0x100 /* A writer is waiting */ +#define _QW_LOCKED 0x0ff /* A writer holds the lock */ +#define _QW_WMASK 0x1ff /* Writer mask */ +#define _QR_SHIFT 9 /* Reader count shift */ +#define _QR_BIAS (1U << _QR_SHIFT) + +/* + * External function declarations + */ +extern void queued_read_lock_slowpath(struct qrwlock *lock); +extern void queued_write_lock_slowpath(struct qrwlock *lock); + +/** + * queued_read_trylock - try to acquire read lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock acquired, 0 if failed + */ +static inline int queued_read_trylock(struct qrwlock *lock) +{ + int cnts; + + cnts = atomic_read(&lock->cnts); + if (likely(!(cnts & _QW_WMASK))) { + cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts); + if (likely(!(cnts & _QW_WMASK))) + return 1; + atomic_sub(_QR_BIAS, &lock->cnts); + } + return 0; +} + +/** + * queued_write_trylock - try to acquire write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock acquired, 0 if failed + */ +static inline int queued_write_trylock(struct qrwlock *lock) +{ + int cnts; + + cnts = atomic_read(&lock->cnts); + if (unlikely(cnts)) + return 0; + + return likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, + _QW_LOCKED)); +} +/** + * queued_read_lock - acquire read lock of a queued rwlock + * @lock: Pointer to queued rwlock structure + */ +static inline void queued_read_lock(struct qrwlock *lock) +{ + int cnts; + + cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts); + if (likely(!(cnts & _QW_WMASK))) + return; + + /* The slowpath will decrement the reader count, if necessary. */ + queued_read_lock_slowpath(lock); +} + +/** + * queued_write_lock - acquire write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + */ +static inline void queued_write_lock(struct qrwlock *lock) +{ + int cnts = 0; + /* Optimize for the unfair lock case where the fair flag is 0. */ + if (likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))) + return; + + queued_write_lock_slowpath(lock); +} + +/** + * queued_read_unlock - release read lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + */ +static inline void queued_read_unlock(struct qrwlock *lock) +{ + /* + * Atomically decrement the reader count + */ + (void)atomic_sub_return_release(_QR_BIAS, &lock->cnts); +} + +/** + * queued_write_unlock - release write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + */ +static inline void queued_write_unlock(struct qrwlock *lock) +{ + smp_store_release(&lock->wlocked, 0); +} + +/** + * queued_rwlock_is_contended - check if the lock is contended + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock contended, 0 otherwise + */ +static inline int queued_rwlock_is_contended(struct qrwlock *lock) +{ + return arch_spin_is_locked(&lock->wait_lock); +} + +/* + * Remapping rwlock architecture specific functions to the corresponding + * queued rwlock functions. + */ +#define arch_read_lock(l) queued_read_lock(l) +#define arch_write_lock(l) queued_write_lock(l) +#define arch_read_trylock(l) queued_read_trylock(l) +#define arch_write_trylock(l) queued_write_trylock(l) +#define arch_read_unlock(l) queued_read_unlock(l) +#define arch_write_unlock(l) queued_write_unlock(l) +#define arch_rwlock_is_contended(l) queued_rwlock_is_contended(l) + +#endif /* __ASM_GENERIC_QRWLOCK_H */ -- cgit v1.3.1