From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- crypto/cipher.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 crypto/cipher.c (limited to 'crypto/cipher.c') diff --git a/crypto/cipher.c b/crypto/cipher.c new file mode 100644 index 000000000..c9dab656a --- /dev/null +++ b/crypto/cipher.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API. + * + * Single-block cipher operations. + * + * Copyright (c) 2002 James Morris + * Copyright (c) 2005 Herbert Xu + */ + +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int setkey_unaligned(struct crypto_cipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct cipher_alg *cia = crypto_cipher_alg(tfm); + unsigned long alignmask = crypto_cipher_alignmask(tfm); + int ret; + u8 *buffer, *alignbuffer; + unsigned long absize; + + absize = keylen + alignmask; + buffer = kmalloc(absize, GFP_ATOMIC); + if (!buffer) + return -ENOMEM; + + alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); + memcpy(alignbuffer, key, keylen); + ret = cia->cia_setkey(crypto_cipher_tfm(tfm), alignbuffer, keylen); + kfree_sensitive(buffer); + return ret; + +} + +int crypto_cipher_setkey(struct crypto_cipher *tfm, + const u8 *key, unsigned int keylen) +{ + struct cipher_alg *cia = crypto_cipher_alg(tfm); + unsigned long alignmask = crypto_cipher_alignmask(tfm); + + if (keylen < cia->cia_min_keysize || keylen > cia->cia_max_keysize) + return -EINVAL; + + if ((unsigned long)key & alignmask) + return setkey_unaligned(tfm, key, keylen); + + return cia->cia_setkey(crypto_cipher_tfm(tfm), key, keylen); +} +EXPORT_SYMBOL_NS_GPL(crypto_cipher_setkey, "CRYPTO_INTERNAL"); + +static inline void cipher_crypt_one(struct crypto_cipher *tfm, + u8 *dst, const u8 *src, bool enc) +{ + unsigned long alignmask = crypto_cipher_alignmask(tfm); + struct cipher_alg *cia = crypto_cipher_alg(tfm); + void (*fn)(struct crypto_tfm *, u8 *, const u8 *) = + enc ? cia->cia_encrypt : cia->cia_decrypt; + + if (unlikely(((unsigned long)dst | (unsigned long)src) & alignmask)) { + unsigned int bs = crypto_cipher_blocksize(tfm); + u8 buffer[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; + u8 *tmp = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); + + memcpy(tmp, src, bs); + fn(crypto_cipher_tfm(tfm), tmp, tmp); + memcpy(dst, tmp, bs); + } else { + fn(crypto_cipher_tfm(tfm), dst, src); + } +} + +void crypto_cipher_encrypt_one(struct crypto_cipher *tfm, + u8 *dst, const u8 *src) +{ + cipher_crypt_one(tfm, dst, src, true); +} +EXPORT_SYMBOL_NS_GPL(crypto_cipher_encrypt_one, "CRYPTO_INTERNAL"); + +void crypto_cipher_decrypt_one(struct crypto_cipher *tfm, + u8 *dst, const u8 *src) +{ + cipher_crypt_one(tfm, dst, src, false); +} +EXPORT_SYMBOL_NS_GPL(crypto_cipher_decrypt_one, "CRYPTO_INTERNAL"); -- cgit v1.3.1