From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- crypto/ecb.c | 228 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 crypto/ecb.c (limited to 'crypto/ecb.c') diff --git a/crypto/ecb.c b/crypto/ecb.c new file mode 100644 index 000000000..cd1b20456 --- /dev/null +++ b/crypto/ecb.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * ECB: Electronic CodeBook mode + * + * Copyright (c) 2006 Herbert Xu + */ + +#include +#include +#include +#include +#include +#include +#include + +static int crypto_ecb_crypt(struct crypto_cipher *cipher, const u8 *src, + u8 *dst, unsigned nbytes, bool final, + void (*fn)(struct crypto_tfm *, u8 *, const u8 *)) +{ + const unsigned int bsize = crypto_cipher_blocksize(cipher); + + while (nbytes >= bsize) { + fn(crypto_cipher_tfm(cipher), dst, src); + + src += bsize; + dst += bsize; + + nbytes -= bsize; + } + + return nbytes && final ? -EINVAL : nbytes; +} + +static int crypto_ecb_encrypt2(struct crypto_lskcipher *tfm, const u8 *src, + u8 *dst, unsigned len, u8 *iv, u32 flags) +{ + struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); + struct crypto_cipher *cipher = *ctx; + + return crypto_ecb_crypt(cipher, src, dst, len, + flags & CRYPTO_LSKCIPHER_FLAG_FINAL, + crypto_cipher_alg(cipher)->cia_encrypt); +} + +static int crypto_ecb_decrypt2(struct crypto_lskcipher *tfm, const u8 *src, + u8 *dst, unsigned len, u8 *iv, u32 flags) +{ + struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); + struct crypto_cipher *cipher = *ctx; + + return crypto_ecb_crypt(cipher, src, dst, len, + flags & CRYPTO_LSKCIPHER_FLAG_FINAL, + crypto_cipher_alg(cipher)->cia_decrypt); +} + +static int lskcipher_setkey_simple2(struct crypto_lskcipher *tfm, + const u8 *key, unsigned int keylen) +{ + struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); + struct crypto_cipher *cipher = *ctx; + + crypto_cipher_clear_flags(cipher, CRYPTO_TFM_REQ_MASK); + crypto_cipher_set_flags(cipher, crypto_lskcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + return crypto_cipher_setkey(cipher, key, keylen); +} + +static int lskcipher_init_tfm_simple2(struct crypto_lskcipher *tfm) +{ + struct lskcipher_instance *inst = lskcipher_alg_instance(tfm); + struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); + struct crypto_cipher_spawn *spawn; + struct crypto_cipher *cipher; + + spawn = lskcipher_instance_ctx(inst); + cipher = crypto_spawn_cipher(spawn); + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + *ctx = cipher; + return 0; +} + +static void lskcipher_exit_tfm_simple2(struct crypto_lskcipher *tfm) +{ + struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); + + crypto_free_cipher(*ctx); +} + +static void lskcipher_free_instance_simple2(struct lskcipher_instance *inst) +{ + crypto_drop_cipher(lskcipher_instance_ctx(inst)); + kfree(inst); +} + +static struct lskcipher_instance *lskcipher_alloc_instance_simple2( + struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_cipher_spawn *spawn; + struct lskcipher_instance *inst; + struct crypto_alg *cipher_alg; + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_LSKCIPHER, &mask); + if (err) + return ERR_PTR(err); + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return ERR_PTR(-ENOMEM); + spawn = lskcipher_instance_ctx(inst); + + err = crypto_grab_cipher(spawn, lskcipher_crypto_instance(inst), + crypto_attr_alg_name(tb[1]), 0, mask); + if (err) + goto err_free_inst; + cipher_alg = crypto_spawn_cipher_alg(spawn); + + err = crypto_inst_setname(lskcipher_crypto_instance(inst), tmpl->name, + cipher_alg); + if (err) + goto err_free_inst; + + inst->free = lskcipher_free_instance_simple2; + + /* Default algorithm properties, can be overridden */ + inst->alg.co.base.cra_blocksize = cipher_alg->cra_blocksize; + inst->alg.co.base.cra_alignmask = cipher_alg->cra_alignmask; + inst->alg.co.base.cra_priority = cipher_alg->cra_priority; + inst->alg.co.min_keysize = cipher_alg->cra_cipher.cia_min_keysize; + inst->alg.co.max_keysize = cipher_alg->cra_cipher.cia_max_keysize; + inst->alg.co.ivsize = cipher_alg->cra_blocksize; + + /* Use struct crypto_cipher * by default, can be overridden */ + inst->alg.co.base.cra_ctxsize = sizeof(struct crypto_cipher *); + inst->alg.setkey = lskcipher_setkey_simple2; + inst->alg.init = lskcipher_init_tfm_simple2; + inst->alg.exit = lskcipher_exit_tfm_simple2; + + return inst; + +err_free_inst: + lskcipher_free_instance_simple2(inst); + return ERR_PTR(err); +} + +static int crypto_ecb_create2(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct lskcipher_instance *inst; + int err; + + inst = lskcipher_alloc_instance_simple2(tmpl, tb); + if (IS_ERR(inst)) + return PTR_ERR(inst); + + /* ECB mode doesn't take an IV */ + inst->alg.co.ivsize = 0; + + inst->alg.encrypt = crypto_ecb_encrypt2; + inst->alg.decrypt = crypto_ecb_decrypt2; + + err = lskcipher_register_instance(tmpl, inst); + if (err) + inst->free(inst); + + return err; +} + +static int crypto_ecb_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_lskcipher_spawn *spawn; + struct lskcipher_alg *cipher_alg; + struct lskcipher_instance *inst; + int err; + + inst = lskcipher_alloc_instance_simple(tmpl, tb); + if (IS_ERR(inst)) { + err = crypto_ecb_create2(tmpl, tb); + return err; + } + + spawn = lskcipher_instance_ctx(inst); + cipher_alg = crypto_lskcipher_spawn_alg(spawn); + + /* ECB mode doesn't take an IV */ + inst->alg.co.ivsize = 0; + if (cipher_alg->co.ivsize) + return -EINVAL; + + inst->alg.co.base.cra_ctxsize = cipher_alg->co.base.cra_ctxsize; + inst->alg.setkey = cipher_alg->setkey; + inst->alg.encrypt = cipher_alg->encrypt; + inst->alg.decrypt = cipher_alg->decrypt; + inst->alg.init = cipher_alg->init; + inst->alg.exit = cipher_alg->exit; + + err = lskcipher_register_instance(tmpl, inst); + if (err) + inst->free(inst); + + return err; +} + +static struct crypto_template crypto_ecb_tmpl = { + .name = "ecb", + .create = crypto_ecb_create, + .module = THIS_MODULE, +}; + +static int __init crypto_ecb_module_init(void) +{ + return crypto_register_template(&crypto_ecb_tmpl); +} + +static void __exit crypto_ecb_module_exit(void) +{ + crypto_unregister_template(&crypto_ecb_tmpl); +} + +module_init(crypto_ecb_module_init); +module_exit(crypto_ecb_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ECB block cipher mode of operation"); +MODULE_ALIAS_CRYPTO("ecb"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); -- cgit v1.3.1