From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- crypto/algboss.c | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 crypto/algboss.c (limited to 'crypto/algboss.c') diff --git a/crypto/algboss.c b/crypto/algboss.c new file mode 100644 index 000000000..8b936cae6 --- /dev/null +++ b/crypto/algboss.c @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Create default crypto algorithm instances. + * + * Copyright (c) 2006 Herbert Xu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct cryptomgr_param { + struct rtattr *tb[CRYPTO_MAX_ATTRS + 2]; + + struct { + struct rtattr attr; + struct crypto_attr_type data; + } type; + + struct { + struct rtattr attr; + struct crypto_attr_alg data; + } attrs[CRYPTO_MAX_ATTRS]; + + char template[CRYPTO_MAX_ALG_NAME]; + + struct crypto_larval *larval; + + u32 otype; + u32 omask; +}; + +struct crypto_test_param { + char driver[CRYPTO_MAX_ALG_NAME]; + char alg[CRYPTO_MAX_ALG_NAME]; + u32 type; +}; + +static int cryptomgr_probe(void *data) +{ + struct cryptomgr_param *param = data; + struct crypto_template *tmpl; + int err = -ENOENT; + + tmpl = crypto_lookup_template(param->template); + if (!tmpl) + goto out; + + do { + err = tmpl->create(tmpl, param->tb); + } while (err == -EAGAIN && !signal_pending(current)); + + crypto_tmpl_put(tmpl); + +out: + param->larval->adult = ERR_PTR(err); + param->larval->alg.cra_flags |= CRYPTO_ALG_DEAD; + complete_all(¶m->larval->completion); + crypto_alg_put(¶m->larval->alg); + kfree(param); + module_put_and_kthread_exit(0); +} + +static int cryptomgr_schedule_probe(struct crypto_larval *larval) +{ + struct task_struct *thread; + struct cryptomgr_param *param; + const char *name = larval->alg.cra_name; + const char *p; + unsigned int len; + int i; + + if (!try_module_get(THIS_MODULE)) + goto err; + + param = kzalloc_obj(*param); + if (!param) + goto err_put_module; + + for (p = name; isalnum(*p) || *p == '-' || *p == '_'; p++) + ; + + len = p - name; + if (!len || *p != '(') + goto err_free_param; + + memcpy(param->template, name, len); + + i = 0; + for (;;) { + name = ++p; + + for (; isalnum(*p) || *p == '-' || *p == '_'; p++) + ; + + if (*p == '(') { + int recursion = 0; + + for (;;) { + if (!*++p) + goto err_free_param; + if (*p == '(') + recursion++; + else if (*p == ')' && !recursion--) + break; + } + + p++; + } + + len = p - name; + if (!len) + goto err_free_param; + + param->attrs[i].attr.rta_len = sizeof(param->attrs[i]); + param->attrs[i].attr.rta_type = CRYPTOA_ALG; + memcpy(param->attrs[i].data.name, name, len); + + param->tb[i + 1] = ¶m->attrs[i].attr; + i++; + + if (i >= CRYPTO_MAX_ATTRS) + goto err_free_param; + + if (*p == ')') + break; + + if (*p != ',') + goto err_free_param; + } + + param->tb[i + 1] = NULL; + + param->type.attr.rta_len = sizeof(param->type); + param->type.attr.rta_type = CRYPTOA_TYPE; + param->type.data.type = larval->alg.cra_flags & ~CRYPTO_ALG_TESTED; + param->type.data.mask = larval->mask & ~CRYPTO_ALG_TESTED; + param->tb[0] = ¶m->type.attr; + + param->otype = larval->alg.cra_flags; + param->omask = larval->mask; + + crypto_alg_get(&larval->alg); + param->larval = larval; + + thread = kthread_run(cryptomgr_probe, param, "cryptomgr_probe"); + if (IS_ERR(thread)) + goto err_put_larval; + + return NOTIFY_STOP; + +err_put_larval: + crypto_alg_put(&larval->alg); +err_free_param: + kfree(param); +err_put_module: + module_put(THIS_MODULE); +err: + return NOTIFY_OK; +} + +static int cryptomgr_test(void *data) +{ + struct crypto_test_param *param = data; + u32 type = param->type; + int err; + + err = alg_test(param->driver, param->alg, type, CRYPTO_ALG_TESTED); + + crypto_alg_tested(param->driver, err); + + kfree(param); + module_put_and_kthread_exit(0); +} + +static int cryptomgr_schedule_test(struct crypto_alg *alg) +{ + struct task_struct *thread; + struct crypto_test_param *param; + + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) + return NOTIFY_DONE; + + if (!try_module_get(THIS_MODULE)) + goto err; + + param = kzalloc_obj(*param); + if (!param) + goto err_put_module; + + memcpy(param->driver, alg->cra_driver_name, sizeof(param->driver)); + memcpy(param->alg, alg->cra_name, sizeof(param->alg)); + param->type = alg->cra_flags; + + thread = kthread_run(cryptomgr_test, param, "cryptomgr_test"); + if (IS_ERR(thread)) + goto err_free_param; + + return NOTIFY_STOP; + +err_free_param: + kfree(param); +err_put_module: + module_put(THIS_MODULE); +err: + return NOTIFY_OK; +} + +static int cryptomgr_notify(struct notifier_block *this, unsigned long msg, + void *data) +{ + switch (msg) { + case CRYPTO_MSG_ALG_REQUEST: + return cryptomgr_schedule_probe(data); + case CRYPTO_MSG_ALG_REGISTER: + return cryptomgr_schedule_test(data); + case CRYPTO_MSG_ALG_LOADED: + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block cryptomgr_notifier = { + .notifier_call = cryptomgr_notify, +}; + +static int __init cryptomgr_init(void) +{ + return crypto_register_notifier(&cryptomgr_notifier); +} + +static void __exit cryptomgr_exit(void) +{ + int err = crypto_unregister_notifier(&cryptomgr_notifier); + BUG_ON(err); +} + +module_init(cryptomgr_init); +module_exit(cryptomgr_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Crypto Algorithm Manager"); -- cgit v1.3.1