From 6668ed271eaefaa63e686bdfbedaeb7b8e492722 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 4 Sep 2026 10:43:19 +0200 Subject: selftests/bpf: Reject graph kptr use after RCU unlock Add a sleepable verifier test that loads a graph-node local kptr in an explicit RCU read-side critical section, then passes its node to bpf_rbtree_remove() after the section ends. Before the verifier fix, the stale NON_OWN_REF flag makes the node look like a live borrowed reference and the program is accepted. After the fix, the pointer is demoted without NON_OWN_REF and the graph kfunc argument is rejected. Also exercise a graph kptr loaded while a spin lock provides implicit RCU protection. The pointer must be invalidated when the lock is released, which guards the required ordering between non-owning-reference invalidation and RCU demotion. Update the existing fault-protected load test state description. The post-unlock pointer no longer carries NON_OWN_REF, but remains readable because the load is rewritten to use BPF_PROBE_MEM. Signed-off-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20260904084325.52250-7-memxor@gmail.com Signed-off-by: Alexei Starovoitov --- include/asm-generic/qrwlock.h | 147 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 include/asm-generic/qrwlock.h (limited to 'include/asm-generic/qrwlock.h') diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h new file mode 100644 index 000000000..75b8f4601 --- /dev/null +++ b/include/asm-generic/qrwlock.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Queue read/write lock + * + * These use generic atomic and locking routines, but depend on a fair spinlock + * implementation in order to be fair themselves. The implementation in + * asm-generic/spinlock.h meets these requirements. + * + * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. + * + * Authors: Waiman Long + */ +#ifndef __ASM_GENERIC_QRWLOCK_H +#define __ASM_GENERIC_QRWLOCK_H + +#include +#include +#include + +#include + +/* Must be included from asm/spinlock.h after defining arch_spin_is_locked. */ + +/* + * Writer states & reader shift and bias. + */ +#define _QW_WAITING 0x100 /* A writer is waiting */ +#define _QW_LOCKED 0x0ff /* A writer holds the lock */ +#define _QW_WMASK 0x1ff /* Writer mask */ +#define _QR_SHIFT 9 /* Reader count shift */ +#define _QR_BIAS (1U << _QR_SHIFT) + +/* + * External function declarations + */ +extern void queued_read_lock_slowpath(struct qrwlock *lock); +extern void queued_write_lock_slowpath(struct qrwlock *lock); + +/** + * queued_read_trylock - try to acquire read lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock acquired, 0 if failed + */ +static inline int queued_read_trylock(struct qrwlock *lock) +{ + int cnts; + + cnts = atomic_read(&lock->cnts); + if (likely(!(cnts & _QW_WMASK))) { + cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts); + if (likely(!(cnts & _QW_WMASK))) + return 1; + atomic_sub(_QR_BIAS, &lock->cnts); + } + return 0; +} + +/** + * queued_write_trylock - try to acquire write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock acquired, 0 if failed + */ +static inline int queued_write_trylock(struct qrwlock *lock) +{ + int cnts; + + cnts = atomic_read(&lock->cnts); + if (unlikely(cnts)) + return 0; + + return likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, + _QW_LOCKED)); +} +/** + * queued_read_lock - acquire read lock of a queued rwlock + * @lock: Pointer to queued rwlock structure + */ +static inline void queued_read_lock(struct qrwlock *lock) +{ + int cnts; + + cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts); + if (likely(!(cnts & _QW_WMASK))) + return; + + /* The slowpath will decrement the reader count, if necessary. */ + queued_read_lock_slowpath(lock); +} + +/** + * queued_write_lock - acquire write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + */ +static inline void queued_write_lock(struct qrwlock *lock) +{ + int cnts = 0; + /* Optimize for the unfair lock case where the fair flag is 0. */ + if (likely(atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))) + return; + + queued_write_lock_slowpath(lock); +} + +/** + * queued_read_unlock - release read lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + */ +static inline void queued_read_unlock(struct qrwlock *lock) +{ + /* + * Atomically decrement the reader count + */ + (void)atomic_sub_return_release(_QR_BIAS, &lock->cnts); +} + +/** + * queued_write_unlock - release write lock of a queued rwlock + * @lock : Pointer to queued rwlock structure + */ +static inline void queued_write_unlock(struct qrwlock *lock) +{ + smp_store_release(&lock->wlocked, 0); +} + +/** + * queued_rwlock_is_contended - check if the lock is contended + * @lock : Pointer to queued rwlock structure + * Return: 1 if lock contended, 0 otherwise + */ +static inline int queued_rwlock_is_contended(struct qrwlock *lock) +{ + return arch_spin_is_locked(&lock->wait_lock); +} + +/* + * Remapping rwlock architecture specific functions to the corresponding + * queued rwlock functions. + */ +#define arch_read_lock(l) queued_read_lock(l) +#define arch_write_lock(l) queued_write_lock(l) +#define arch_read_trylock(l) queued_read_trylock(l) +#define arch_write_trylock(l) queued_write_trylock(l) +#define arch_read_unlock(l) queued_read_unlock(l) +#define arch_write_unlock(l) queued_write_unlock(l) +#define arch_rwlock_is_contended(l) queued_rwlock_is_contended(l) + +#endif /* __ASM_GENERIC_QRWLOCK_H */ -- cgit v1.3.1