From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- scripts/include/hashtable.h | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 scripts/include/hashtable.h (limited to 'scripts/include/hashtable.h') diff --git a/scripts/include/hashtable.h b/scripts/include/hashtable.h new file mode 100644 index 000000000..45abcb12b --- /dev/null +++ b/scripts/include/hashtable.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef HASHTABLE_H +#define HASHTABLE_H + +#include "array_size.h" +#include "list.h" + +#define HASH_SIZE(name) (ARRAY_SIZE(name)) + +#define HASHTABLE_DECLARE(name, size) struct hlist_head name[size] + +#define HASHTABLE_DEFINE(name, size) \ + HASHTABLE_DECLARE(name, size) = \ + { [0 ... ((size) - 1)] = HLIST_HEAD_INIT } + +#define hash_head(table, key) (&(table)[(key) % HASH_SIZE(table)]) + +static inline void __hash_init(struct hlist_head *ht, unsigned int sz) +{ + unsigned int i; + + for (i = 0; i < sz; i++) + INIT_HLIST_HEAD(&ht[i]); +} + +/** + * hash_init - initialize a hash table + * @table: hashtable to be initialized + * + * This has to be a macro since HASH_SIZE() will not work on pointers since + * it calculates the size during preprocessing. + */ +#define hash_init(table) __hash_init(table, HASH_SIZE(table)) + +/** + * hash_add - add an object to a hashtable + * @table: hashtable to add to + * @node: the &struct hlist_node of the object to be added + * @key: the key of the object to be added + */ +#define hash_add(table, node, key) \ + hlist_add_head(node, hash_head(table, key)) + +/** + * hash_del - remove an object from a hashtable + * @node: &struct hlist_node of the object to remove + */ +static inline void hash_del(struct hlist_node *node) +{ + hlist_del_init(node); +} + +/** + * hash_for_each - iterate over a hashtable + * @table: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + */ +#define hash_for_each(table, obj, member) \ + for (int _bkt = 0; _bkt < HASH_SIZE(table); _bkt++) \ + hlist_for_each_entry(obj, &table[_bkt], member) + +/** + * hash_for_each_safe - iterate over a hashtable safe against removal of + * hash entry + * @table: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @tmp: a &struct hlist_node used for temporary storage + * @member: the name of the hlist_node within the struct + */ +#define hash_for_each_safe(table, obj, tmp, member) \ + for (int _bkt = 0; _bkt < HASH_SIZE(table); _bkt++) \ + hlist_for_each_entry_safe(obj, tmp, &table[_bkt], member) + +/** + * hash_for_each_possible - iterate over all possible objects hashing to the + * same bucket + * @table: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + * @key: the key of the objects to iterate over + */ +#define hash_for_each_possible(table, obj, member, key) \ + hlist_for_each_entry(obj, hash_head(table, key), member) + +/** + * hash_for_each_possible_safe - iterate over all possible objects hashing to the + * same bucket safe against removals + * @table: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @tmp: a &struct hlist_node used for temporary storage + * @member: the name of the hlist_node within the struct + * @key: the key of the objects to iterate over + */ +#define hash_for_each_possible_safe(table, obj, tmp, member, key) \ + hlist_for_each_entry_safe(obj, tmp, hash_head(table, key), member) + +#endif /* HASHTABLE_H */ -- cgit v1.3.1