sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 208 insertions, 0 deletions

diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig
new file mode 100644
index 000000000..e12111462
--- /dev/null
+++ b/lib/crypto/tests/Kconfig
@@ -0,0 +1,208 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST
+	tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message
+	  authentication codes.
+
+config CRYPTO_LIB_AES_CCM_KUNIT_TEST
+	tristate "KUnit tests for AES-CCM" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_CCM
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	# This test uses BLAKE2s, but that's always built-in.
+	help
+	  KUnit tests for the AES-CCM authenticated encryption algorithm.
+
+config CRYPTO_LIB_AES_GCM_KUNIT_TEST
+	tristate "KUnit tests for AES-GCM" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_GCM
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	# This test uses BLAKE2s, but that's always built-in.
+	help
+	  KUnit tests for the AES-GCM authenticated encryption algorithm.
+
+config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
+	tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_BLAKE2B
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the BLAKE2b cryptographic hash function.
+
+config CRYPTO_LIB_BLAKE2S_KUNIT_TEST
+	tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	# No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't
+	# exist; the BLAKE2s code is always built-in for the /dev/random driver.
+	help
+	  KUnit tests for the BLAKE2s cryptographic hash function.
+
+config CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST
+	tristate "KUnit tests for ChaCha20Poly1305" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_CHACHA20POLY1305
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the ChaCha20Poly1305 authenticated encryption
+	  algorithm.
+
+config CRYPTO_LIB_CURVE25519_KUNIT_TEST
+	tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_CURVE25519
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the Curve25519 Diffie-Hellman function.
+
+config CRYPTO_LIB_GHASH_KUNIT_TEST
+	tristate "KUnit tests for GHASH" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_GF128HASH
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the GHASH library functions.
+
+config CRYPTO_LIB_MD5_KUNIT_TEST
+	tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_MD5
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the MD5 cryptographic hash function and its
+	  corresponding HMAC.
+
+config CRYPTO_LIB_MLDSA_KUNIT_TEST
+	tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_MLDSA
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the ML-DSA digital signature algorithm.
+
+config CRYPTO_LIB_NH_KUNIT_TEST
+	tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_NH
+	default KUNIT_ALL_TESTS
+	help
+	  KUnit tests for the NH almost-universal hash function.
+
+config CRYPTO_LIB_POLY1305_KUNIT_TEST
+	tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_POLY1305
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the Poly1305 library functions.
+
+config CRYPTO_LIB_POLYVAL_KUNIT_TEST
+	tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_GF128HASH
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the POLYVAL library functions.
+
+config CRYPTO_LIB_SHA1_KUNIT_TEST
+	tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SHA1
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SHA-1 cryptographic hash function and its
+	  corresponding HMAC.
+
+# Option is named *_SHA256_KUNIT_TEST, though both SHA-224 and SHA-256 tests are
+# included, for consistency with the naming used elsewhere (e.g. CRYPTO_SHA256).
+config CRYPTO_LIB_SHA256_KUNIT_TEST
+	tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SHA256
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions
+	  and their corresponding HMACs.
+
+# Option is named *_SHA512_KUNIT_TEST, though both SHA-384 and SHA-512 tests are
+# included, for consistency with the naming used elsewhere (e.g. CRYPTO_SHA512).
+config CRYPTO_LIB_SHA512_KUNIT_TEST
+	tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SHA512
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions
+	  and their corresponding HMACs.
+
+config CRYPTO_LIB_SHA3_KUNIT_TEST
+	tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SHA3
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SHA3 cryptographic hash and XOF functions,
+	  including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and
+	  SHAKE256.
+
+config CRYPTO_LIB_SM3_KUNIT_TEST
+	tristate "KUnit tests for SM3" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_SM3
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  KUnit tests for the SM3 cryptographic hash function.
+
+config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT
+	tristate "Enable all crypto library code for KUnit tests"
+	depends on KUNIT
+	select CRYPTO_LIB_AES_CBC
+	select CRYPTO_LIB_AES_CBC_MACS
+	select CRYPTO_LIB_AES_CCM
+	select CRYPTO_LIB_AES_CTR
+	select CRYPTO_LIB_AES_ECB
+	select CRYPTO_LIB_AES_GCM
+	select CRYPTO_LIB_AES_XTS
+	select CRYPTO_LIB_BLAKE2B
+	select CRYPTO_LIB_CHACHA20POLY1305
+	select CRYPTO_LIB_CURVE25519
+	select CRYPTO_LIB_GF128HASH
+	select CRYPTO_LIB_MD5
+	select CRYPTO_LIB_MLDSA
+	select CRYPTO_LIB_NH
+	select CRYPTO_LIB_POLY1305
+	select CRYPTO_LIB_SHA1
+	select CRYPTO_LIB_SHA256
+	select CRYPTO_LIB_SHA512
+	select CRYPTO_LIB_SHA3
+	select CRYPTO_LIB_SM3
+	help
+	  Enable all the crypto library code that has KUnit tests.
+
+	  Enable this only if you'd like to test all the crypto library code,
+	  even code that wouldn't otherwise need to be built.
+
+	  You'll still need to enable the tests themselves, either individually
+	  or using KUNIT_ALL_TESTS.
+
+config CRYPTO_LIB_BENCHMARK_VISIBLE
+	bool
+
+config CRYPTO_LIB_BENCHMARK
+	bool "Include benchmarks in KUnit tests for cryptographic functions"
+	depends on CRYPTO_LIB_BENCHMARK_VISIBLE
+	help
+	  Include benchmarks in the KUnit tests for cryptographic functions.
+	  The benchmark results are printed to the kernel log when the
+	  corresponding KUnit test suite runs.
+
+	  This is useful for evaluating the performance of the cryptographic
+	  functions.  However, it will increase the runtime of the KUnit tests.
+
+	  If you're only interested in correctness testing, leave this disabled.