From 6668ed271eaefaa63e686bdfbedaeb7b8e492722 Mon Sep 17 00:00:00 2001
From: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Date: Fri, 4 Sep 2026 10:43:19 +0200
Subject: selftests/bpf: Reject graph kptr use after RCU unlock

Add a sleepable verifier test that loads a graph-node local kptr in an
explicit RCU read-side critical section, then passes its node to
bpf_rbtree_remove() after the section ends.

Before the verifier fix, the stale NON_OWN_REF flag makes the node look like
a live borrowed reference and the program is accepted. After the fix, the
pointer is demoted without NON_OWN_REF and the graph kfunc argument is
rejected.

Also exercise a graph kptr loaded while a spin lock provides implicit RCU
protection. The pointer must be invalidated when the lock is released, which
guards the required ordering between non-owning-reference invalidation and
RCU demotion.

Update the existing fault-protected load test state description. The
post-unlock pointer no longer carries NON_OWN_REF, but remains readable
because the load is rewritten to use BPF_PROBE_MEM.

Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20260904084325.52250-7-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
---
 scripts/atomic/fallbacks/acquire             |  5 +++++
 scripts/atomic/fallbacks/add_negative        |  3 +++
 scripts/atomic/fallbacks/add_unless          |  3 +++
 scripts/atomic/fallbacks/andnot              |  3 +++
 scripts/atomic/fallbacks/cmpxchg             |  3 +++
 scripts/atomic/fallbacks/dec                 |  3 +++
 scripts/atomic/fallbacks/dec_and_test        |  3 +++
 scripts/atomic/fallbacks/dec_if_positive     | 11 +++++++++++
 scripts/atomic/fallbacks/dec_unless_positive | 10 ++++++++++
 scripts/atomic/fallbacks/fence               |  7 +++++++
 scripts/atomic/fallbacks/fetch_add_unless    | 10 ++++++++++
 scripts/atomic/fallbacks/inc                 |  3 +++
 scripts/atomic/fallbacks/inc_and_test        |  3 +++
 scripts/atomic/fallbacks/inc_not_zero        |  3 +++
 scripts/atomic/fallbacks/inc_unless_negative | 10 ++++++++++
 scripts/atomic/fallbacks/read_acquire        | 12 ++++++++++++
 scripts/atomic/fallbacks/release             |  4 ++++
 scripts/atomic/fallbacks/set_release         |  8 ++++++++
 scripts/atomic/fallbacks/sub_and_test        |  3 +++
 scripts/atomic/fallbacks/try_cmpxchg         |  7 +++++++
 scripts/atomic/fallbacks/xchg                |  3 +++
 21 files changed, 117 insertions(+)
 create mode 100755 scripts/atomic/fallbacks/acquire
 create mode 100755 scripts/atomic/fallbacks/add_negative
 create mode 100755 scripts/atomic/fallbacks/add_unless
 create mode 100755 scripts/atomic/fallbacks/andnot
 create mode 100644 scripts/atomic/fallbacks/cmpxchg
 create mode 100755 scripts/atomic/fallbacks/dec
 create mode 100755 scripts/atomic/fallbacks/dec_and_test
 create mode 100755 scripts/atomic/fallbacks/dec_if_positive
 create mode 100755 scripts/atomic/fallbacks/dec_unless_positive
 create mode 100755 scripts/atomic/fallbacks/fence
 create mode 100755 scripts/atomic/fallbacks/fetch_add_unless
 create mode 100755 scripts/atomic/fallbacks/inc
 create mode 100755 scripts/atomic/fallbacks/inc_and_test
 create mode 100755 scripts/atomic/fallbacks/inc_not_zero
 create mode 100755 scripts/atomic/fallbacks/inc_unless_negative
 create mode 100755 scripts/atomic/fallbacks/read_acquire
 create mode 100755 scripts/atomic/fallbacks/release
 create mode 100755 scripts/atomic/fallbacks/set_release
 create mode 100755 scripts/atomic/fallbacks/sub_and_test
 create mode 100755 scripts/atomic/fallbacks/try_cmpxchg
 create mode 100644 scripts/atomic/fallbacks/xchg

(limited to 'scripts/atomic/fallbacks')

diff --git a/scripts/atomic/fallbacks/acquire b/scripts/atomic/fallbacks/acquire
new file mode 100755
index 000000000..4da0cab36
--- /dev/null
+++ b/scripts/atomic/fallbacks/acquire
@@ -0,0 +1,5 @@
+cat <<EOF
+	${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
+	__atomic_acquire_fence();
+	return ret;
+EOF
diff --git a/scripts/atomic/fallbacks/add_negative b/scripts/atomic/fallbacks/add_negative
new file mode 100755
index 000000000..1d3d4ab3a
--- /dev/null
+++ b/scripts/atomic/fallbacks/add_negative
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_${atomic}_add_return${order}(i, v) < 0;
+EOF
diff --git a/scripts/atomic/fallbacks/add_unless b/scripts/atomic/fallbacks/add_unless
new file mode 100755
index 000000000..95ecb2b74
--- /dev/null
+++ b/scripts/atomic/fallbacks/add_unless
@@ -0,0 +1,3 @@
+cat << EOF
+	return raw_${atomic}_fetch_add_unless(v, a, u) != u;
+EOF
diff --git a/scripts/atomic/fallbacks/andnot b/scripts/atomic/fallbacks/andnot
new file mode 100755
index 000000000..66760457e
--- /dev/null
+++ b/scripts/atomic/fallbacks/andnot
@@ -0,0 +1,3 @@
+cat <<EOF
+	${retstmt}raw_${atomic}_${pfx}and${sfx}${order}(~i, v);
+EOF
diff --git a/scripts/atomic/fallbacks/cmpxchg b/scripts/atomic/fallbacks/cmpxchg
new file mode 100644
index 000000000..1c8507f62
--- /dev/null
+++ b/scripts/atomic/fallbacks/cmpxchg
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_cmpxchg${order}(&v->counter, old, new);
+EOF
diff --git a/scripts/atomic/fallbacks/dec b/scripts/atomic/fallbacks/dec
new file mode 100755
index 000000000..60d286d40
--- /dev/null
+++ b/scripts/atomic/fallbacks/dec
@@ -0,0 +1,3 @@
+cat <<EOF
+	${retstmt}raw_${atomic}_${pfx}sub${sfx}${order}(1, v);
+EOF
diff --git a/scripts/atomic/fallbacks/dec_and_test b/scripts/atomic/fallbacks/dec_and_test
new file mode 100755
index 000000000..3a0278e0d
--- /dev/null
+++ b/scripts/atomic/fallbacks/dec_and_test
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_${atomic}_dec_return(v) == 0;
+EOF
diff --git a/scripts/atomic/fallbacks/dec_if_positive b/scripts/atomic/fallbacks/dec_if_positive
new file mode 100755
index 000000000..f65c11b4b
--- /dev/null
+++ b/scripts/atomic/fallbacks/dec_if_positive
@@ -0,0 +1,11 @@
+cat <<EOF
+	${int} dec, c = raw_${atomic}_read(v);
+
+	do {
+		dec = c - 1;
+		if (unlikely(dec < 0))
+			break;
+	} while (!raw_${atomic}_try_cmpxchg(v, &c, dec));
+
+	return dec;
+EOF
diff --git a/scripts/atomic/fallbacks/dec_unless_positive b/scripts/atomic/fallbacks/dec_unless_positive
new file mode 100755
index 000000000..d025361d7
--- /dev/null
+++ b/scripts/atomic/fallbacks/dec_unless_positive
@@ -0,0 +1,10 @@
+cat <<EOF
+	${int} c = raw_${atomic}_read(v);
+
+	do {
+		if (unlikely(c > 0))
+			return false;
+	} while (!raw_${atomic}_try_cmpxchg(v, &c, c - 1));
+
+	return true;
+EOF
diff --git a/scripts/atomic/fallbacks/fence b/scripts/atomic/fallbacks/fence
new file mode 100755
index 000000000..40d5b3976
--- /dev/null
+++ b/scripts/atomic/fallbacks/fence
@@ -0,0 +1,7 @@
+cat <<EOF
+	${ret} ret;
+	__atomic_pre_full_fence();
+	ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
+	__atomic_post_full_fence();
+	return ret;
+EOF
diff --git a/scripts/atomic/fallbacks/fetch_add_unless b/scripts/atomic/fallbacks/fetch_add_unless
new file mode 100755
index 000000000..8db7e9e17
--- /dev/null
+++ b/scripts/atomic/fallbacks/fetch_add_unless
@@ -0,0 +1,10 @@
+cat << EOF
+	${int} c = raw_${atomic}_read(v);
+
+	do {
+		if (unlikely(c == u))
+			break;
+	} while (!raw_${atomic}_try_cmpxchg(v, &c, c + a));
+
+	return c;
+EOF
diff --git a/scripts/atomic/fallbacks/inc b/scripts/atomic/fallbacks/inc
new file mode 100755
index 000000000..56c770f59
--- /dev/null
+++ b/scripts/atomic/fallbacks/inc
@@ -0,0 +1,3 @@
+cat <<EOF
+	${retstmt}raw_${atomic}_${pfx}add${sfx}${order}(1, v);
+EOF
diff --git a/scripts/atomic/fallbacks/inc_and_test b/scripts/atomic/fallbacks/inc_and_test
new file mode 100755
index 000000000..7d16a10f2
--- /dev/null
+++ b/scripts/atomic/fallbacks/inc_and_test
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_${atomic}_inc_return(v) == 0;
+EOF
diff --git a/scripts/atomic/fallbacks/inc_not_zero b/scripts/atomic/fallbacks/inc_not_zero
new file mode 100755
index 000000000..1fcef1e55
--- /dev/null
+++ b/scripts/atomic/fallbacks/inc_not_zero
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_${atomic}_add_unless(v, 1, 0);
+EOF
diff --git a/scripts/atomic/fallbacks/inc_unless_negative b/scripts/atomic/fallbacks/inc_unless_negative
new file mode 100755
index 000000000..7b4b09868
--- /dev/null
+++ b/scripts/atomic/fallbacks/inc_unless_negative
@@ -0,0 +1,10 @@
+cat <<EOF
+	${int} c = raw_${atomic}_read(v);
+
+	do {
+		if (unlikely(c < 0))
+			return false;
+	} while (!raw_${atomic}_try_cmpxchg(v, &c, c + 1));
+
+	return true;
+EOF
diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire
new file mode 100755
index 000000000..e319862d2
--- /dev/null
+++ b/scripts/atomic/fallbacks/read_acquire
@@ -0,0 +1,12 @@
+cat <<EOF
+	${int} ret;
+
+	if (__native_word(${atomic}_t)) {
+		ret = smp_load_acquire(&(v)->counter);
+	} else {
+		ret = raw_${atomic}_read(v);
+		__atomic_acquire_fence();
+	}
+
+	return ret;
+EOF
diff --git a/scripts/atomic/fallbacks/release b/scripts/atomic/fallbacks/release
new file mode 100755
index 000000000..1e6daf57b
--- /dev/null
+++ b/scripts/atomic/fallbacks/release
@@ -0,0 +1,4 @@
+cat <<EOF
+	__atomic_release_fence();
+	${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
+EOF
diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release
new file mode 100755
index 000000000..16a374ae6
--- /dev/null
+++ b/scripts/atomic/fallbacks/set_release
@@ -0,0 +1,8 @@
+cat <<EOF
+	if (__native_word(${atomic}_t)) {
+		smp_store_release(&(v)->counter, i);
+	} else {
+		__atomic_release_fence();
+		raw_${atomic}_set(v, i);
+	}
+EOF
diff --git a/scripts/atomic/fallbacks/sub_and_test b/scripts/atomic/fallbacks/sub_and_test
new file mode 100755
index 000000000..d1f746fe0
--- /dev/null
+++ b/scripts/atomic/fallbacks/sub_and_test
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_${atomic}_sub_return(i, v) == 0;
+EOF
diff --git a/scripts/atomic/fallbacks/try_cmpxchg b/scripts/atomic/fallbacks/try_cmpxchg
new file mode 100755
index 000000000..d4da82092
--- /dev/null
+++ b/scripts/atomic/fallbacks/try_cmpxchg
@@ -0,0 +1,7 @@
+cat <<EOF
+	${int} r, o = *old;
+	r = raw_${atomic}_cmpxchg${order}(v, o, new);
+	if (unlikely(r != o))
+		*old = r;
+	return likely(r == o);
+EOF
diff --git a/scripts/atomic/fallbacks/xchg b/scripts/atomic/fallbacks/xchg
new file mode 100644
index 000000000..e4def1e0d
--- /dev/null
+++ b/scripts/atomic/fallbacks/xchg
@@ -0,0 +1,3 @@
+cat <<EOF
+	return raw_xchg${order}(&v->counter, new);
+EOF
-- 
cgit v1.3.1