sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 196 insertions, 0 deletions

diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c
new file mode 100644
index 000000000..368018cfa
--- /dev/null
+++ b/crypto/twofish_generic.c
@@ -0,0 +1,196 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Twofish for CryptoAPI
+ *
+ * Originally Twofish for GPG
+ * By Matthew Skala <mskala@ansuz.sooke.bc.ca>, July 26, 1998
+ * 256-bit key length added March 20, 1999
+ * Some modifications to reduce the text size by Werner Koch, April, 1998
+ * Ported to the kerneli patch by Marc Mutz <Marc@Mutz.com>
+ * Ported to CryptoAPI by Colin Slater <hoho@tacomeat.net>
+ *
+ * The original author has disclaimed all copyright interest in this
+ * code and thus put it in the public domain. The subsequent authors 
+ * have put this under the GNU General Public License.
+ *
+ * This code is a "clean room" implementation, written from the paper
+ * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey,
+ * Doug Whiting, David Wagner, Chris Hall, and Niels Ferguson, available
+ * through http://www.counterpane.com/twofish.html
+ *
+ * For background information on multiplication in finite fields, used for
+ * the matrix operations in the key schedule, see the book _Contemporary
+ * Abstract Algebra_ by Joseph A. Gallian, especially chapter 22 in the
+ * Third Edition.
+ */
+
+#include <linux/unaligned.h>
+#include <crypto/algapi.h>
+#include <crypto/twofish.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/bitops.h>
+
+/* Macros to compute the g() function in the encryption and decryption
+ * rounds.  G1 is the straight g() function; G2 includes the 8-bit
+ * rotation for the high 32-bit word. */
+
+#define G1(a) \
+     (ctx->s[0][(a) & 0xFF]) ^ (ctx->s[1][((a) >> 8) & 0xFF]) \
+   ^ (ctx->s[2][((a) >> 16) & 0xFF]) ^ (ctx->s[3][(a) >> 24])
+
+#define G2(b) \
+     (ctx->s[1][(b) & 0xFF]) ^ (ctx->s[2][((b) >> 8) & 0xFF]) \
+   ^ (ctx->s[3][((b) >> 16) & 0xFF]) ^ (ctx->s[0][(b) >> 24])
+
+/* Encryption and decryption Feistel rounds.  Each one calls the two g()
+ * macros, does the PHT, and performs the XOR and the appropriate bit
+ * rotations.  The parameters are the round number (used to select subkeys),
+ * and the four 32-bit chunks of the text. */
+
+#define ENCROUND(n, a, b, c, d) \
+   x = G1 (a); y = G2 (b); \
+   x += y; y += x + ctx->k[2 * (n) + 1]; \
+   (c) ^= x + ctx->k[2 * (n)]; \
+   (c) = ror32((c), 1); \
+   (d) = rol32((d), 1) ^ y
+
+#define DECROUND(n, a, b, c, d) \
+   x = G1 (a); y = G2 (b); \
+   x += y; y += x; \
+   (d) ^= y + ctx->k[2 * (n) + 1]; \
+   (d) = ror32((d), 1); \
+   (c) = rol32((c), 1); \
+   (c) ^= (x + ctx->k[2 * (n)])
+
+/* Encryption and decryption cycles; each one is simply two Feistel rounds
+ * with the 32-bit chunks re-ordered to simulate the "swap" */
+
+#define ENCCYCLE(n) \
+   ENCROUND (2 * (n), a, b, c, d); \
+   ENCROUND (2 * (n) + 1, c, d, a, b)
+
+#define DECCYCLE(n) \
+   DECROUND (2 * (n) + 1, c, d, a, b); \
+   DECROUND (2 * (n), a, b, c, d)
+
+/* Macros to convert the input and output bytes into 32-bit words,
+ * and simultaneously perform the whitening step.  INPACK packs word
+ * number n into the variable named by x, using whitening subkey number m.
+ * OUTUNPACK unpacks word number n from the variable named by x, using
+ * whitening subkey number m. */
+
+#define INPACK(n, x, m) \
+   x = get_unaligned_le32(in + (n) * 4) ^ ctx->w[m]
+
+#define OUTUNPACK(n, x, m) \
+   x ^= ctx->w[m]; \
+   put_unaligned_le32(x, out + (n) * 4)
+
+
+
+/* Encrypt one block.  in and out may be the same. */
+static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct twofish_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	/* The four 32-bit chunks of the text. */
+	u32 a, b, c, d;
+	
+	/* Temporaries used by the round function. */
+	u32 x, y;
+
+	/* Input whitening and packing. */
+	INPACK (0, a, 0);
+	INPACK (1, b, 1);
+	INPACK (2, c, 2);
+	INPACK (3, d, 3);
+	
+	/* Encryption Feistel cycles. */
+	ENCCYCLE (0);
+	ENCCYCLE (1);
+	ENCCYCLE (2);
+	ENCCYCLE (3);
+	ENCCYCLE (4);
+	ENCCYCLE (5);
+	ENCCYCLE (6);
+	ENCCYCLE (7);
+	
+	/* Output whitening and unpacking. */
+	OUTUNPACK (0, c, 4);
+	OUTUNPACK (1, d, 5);
+	OUTUNPACK (2, a, 6);
+	OUTUNPACK (3, b, 7);
+	
+}
+
+/* Decrypt one block.  in and out may be the same. */
+static void twofish_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	struct twofish_ctx *ctx = crypto_tfm_ctx(tfm);
+  
+	/* The four 32-bit chunks of the text. */
+	u32 a, b, c, d;
+	
+	/* Temporaries used by the round function. */
+	u32 x, y;
+	
+	/* Input whitening and packing. */
+	INPACK (0, c, 4);
+	INPACK (1, d, 5);
+	INPACK (2, a, 6);
+	INPACK (3, b, 7);
+	
+	/* Encryption Feistel cycles. */
+	DECCYCLE (7);
+	DECCYCLE (6);
+	DECCYCLE (5);
+	DECCYCLE (4);
+	DECCYCLE (3);
+	DECCYCLE (2);
+	DECCYCLE (1);
+	DECCYCLE (0);
+
+	/* Output whitening and unpacking. */
+	OUTUNPACK (0, a, 0);
+	OUTUNPACK (1, b, 1);
+	OUTUNPACK (2, c, 2);
+	OUTUNPACK (3, d, 3);
+
+}
+
+static struct crypto_alg alg = {
+	.cra_name           =   "twofish",
+	.cra_driver_name    =   "twofish-generic",
+	.cra_priority       =   100,
+	.cra_flags          =   CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize      =   TF_BLOCK_SIZE,
+	.cra_ctxsize        =   sizeof(struct twofish_ctx),
+	.cra_module         =   THIS_MODULE,
+	.cra_u              =   { .cipher = {
+	.cia_min_keysize    =   TF_MIN_KEY_SIZE,
+	.cia_max_keysize    =   TF_MAX_KEY_SIZE,
+	.cia_setkey         =   twofish_setkey,
+	.cia_encrypt        =   twofish_encrypt,
+	.cia_decrypt        =   twofish_decrypt } }
+};
+
+static int __init twofish_mod_init(void)
+{
+	return crypto_register_alg(&alg);
+}
+
+static void __exit twofish_mod_fini(void)
+{
+	crypto_unregister_alg(&alg);
+}
+
+module_init(twofish_mod_init);
+module_exit(twofish_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION ("Twofish Cipher Algorithm");
+MODULE_ALIAS_CRYPTO("twofish");
+MODULE_ALIAS_CRYPTO("twofish-generic");