From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- lib/tests/test_fprobe.c | 327 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 lib/tests/test_fprobe.c (limited to 'lib/tests/test_fprobe.c') diff --git a/lib/tests/test_fprobe.c b/lib/tests/test_fprobe.c new file mode 100644 index 000000000..108c7aa33 --- /dev/null +++ b/lib/tests/test_fprobe.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * test_fprobe.c - simple sanity test for fprobe + */ + +#include +#include +#include +#include + +#define div_factor 3 + +static struct kunit *current_test; + +static u32 rand1, entry_only_val, entry_val, exit_val; +static u32 entry_only_count, entry_count, exit_count; + +/* Use indirect calls to avoid inlining the target functions */ +static u32 (*target)(u32 value); +static u32 (*target2)(u32 value); +static unsigned long target_ip; +static unsigned long target2_ip; +static int entry_return_value; + +static noinline u32 fprobe_selftest_target(u32 value) +{ + return (value / div_factor); +} + +static noinline u32 fprobe_selftest_target2(u32 value) +{ + return (value / div_factor) + 1; +} + +static notrace int fp_entry_handler(struct fprobe *fp, unsigned long ip, + unsigned long ret_ip, + struct ftrace_regs *fregs, void *data) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + /* This can be called on the fprobe_selftest_target and the fprobe_selftest_target2 */ + if (ip != target_ip) + KUNIT_EXPECT_EQ(current_test, ip, target2_ip); + entry_val = (rand1 / div_factor); + if (fp->entry_data_size) { + KUNIT_EXPECT_NOT_NULL(current_test, data); + if (data) + *(u32 *)data = entry_val; + } else + KUNIT_EXPECT_NULL(current_test, data); + + return entry_return_value; +} + +static notrace void fp_exit_handler(struct fprobe *fp, unsigned long ip, + unsigned long ret_ip, + struct ftrace_regs *fregs, void *data) +{ + unsigned long ret = ftrace_regs_get_return_value(fregs); + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + if (ip != target_ip) { + KUNIT_EXPECT_EQ(current_test, ip, target2_ip); + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor) + 1); + } else + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor)); + KUNIT_EXPECT_EQ(current_test, entry_val, (rand1 / div_factor)); + exit_val = entry_val + div_factor; + if (fp->entry_data_size) { + KUNIT_EXPECT_NOT_NULL(current_test, data); + if (data) + KUNIT_EXPECT_EQ(current_test, *(u32 *)data, entry_val); + } else + KUNIT_EXPECT_NULL(current_test, data); +} + +/* Test entry only (no rethook) */ +static void test_fprobe_entry(struct kunit *test) +{ + struct fprobe fp_entry = { + .entry_handler = fp_entry_handler, + }; + + current_test = test; + + /* Before register, unregister should be failed. */ + KUNIT_EXPECT_NE(test, 0, unregister_fprobe(&fp_entry)); + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp_entry, "fprobe_selftest_target*", NULL)); + + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, 0, exit_val); + + entry_val = 0; + exit_val = 0; + target2(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, 0, exit_val); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp_entry)); +} + +static void test_fprobe(struct kunit *test) +{ + struct fprobe fp = { + .entry_handler = fp_entry_handler, + .exit_handler = fp_exit_handler, + }; + + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp, "fprobe_selftest_target*", NULL)); + + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + entry_val = 0; + exit_val = 0; + target2(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); +} + +static void test_fprobe_syms(struct kunit *test) +{ + static const char *syms[] = {"fprobe_selftest_target", "fprobe_selftest_target2"}; + struct fprobe fp = { + .entry_handler = fp_entry_handler, + .exit_handler = fp_exit_handler, + }; + + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_fprobe_syms(&fp, syms, 2)); + + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + entry_val = 0; + exit_val = 0; + target2(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); +} + +/* Test private entry_data */ +static void test_fprobe_data(struct kunit *test) +{ + struct fprobe fp = { + .entry_handler = fp_entry_handler, + .exit_handler = fp_exit_handler, + .entry_data_size = sizeof(u32), + }; + + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp, "fprobe_selftest_target", NULL)); + + target(rand1); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); +} + +static void test_fprobe_skip(struct kunit *test) +{ + struct fprobe fp = { + .entry_handler = fp_entry_handler, + .exit_handler = fp_exit_handler, + }; + + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp, "fprobe_selftest_target", NULL)); + + entry_return_value = 1; + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, 0, exit_val); + KUNIT_EXPECT_EQ(test, 0, fp.nmissed); + entry_return_value = 0; + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); +} + +/* Handler for fprobe entry only case */ +static notrace int entry_only_handler(struct fprobe *fp, unsigned long ip, + unsigned long ret_ip, + struct ftrace_regs *fregs, void *data) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, ip, target_ip); + + entry_only_count++; + entry_only_val = (rand1 / div_factor); + + return 0; +} + +static notrace int fprobe_entry_multi_handler(struct fprobe *fp, unsigned long ip, + unsigned long ret_ip, + struct ftrace_regs *fregs, + void *data) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, ip, target_ip); + + entry_count++; + entry_val = (rand1 / div_factor); + + return 0; +} + +static notrace void fprobe_exit_multi_handler(struct fprobe *fp, unsigned long ip, + unsigned long ret_ip, + struct ftrace_regs *fregs, + void *data) +{ + unsigned long ret = ftrace_regs_get_return_value(fregs); + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, ip, target_ip); + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor)); + + exit_count++; + exit_val = ret; +} + +static void check_fprobe_multi(struct kunit *test) +{ + entry_only_count = entry_count = exit_count = 0; + entry_only_val = entry_val = exit_val = 0; + + target(rand1); + + /* Verify all handlers were called */ + KUNIT_EXPECT_EQ(test, 1, entry_only_count); + KUNIT_EXPECT_EQ(test, 1, entry_count); + KUNIT_EXPECT_EQ(test, 1, exit_count); + + /* Verify values are correct */ + KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_only_val); + KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_val); + KUNIT_EXPECT_EQ(test, (rand1 / div_factor), exit_val); +} + +/* Test multiple fprobes hooking the same target function */ +static void test_fprobe_multi(struct kunit *test) +{ + struct fprobe fp1 = { + .entry_handler = fprobe_entry_multi_handler, + .exit_handler = fprobe_exit_multi_handler, + }; + struct fprobe fp2 = { + .entry_handler = entry_only_handler, + }; + + current_test = test; + + /* Test Case 1: Register in order 1 -> 2 */ + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL)); + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL)); + + check_fprobe_multi(test); + + /* Unregister all */ + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1)); + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2)); + + /* Test Case 2: Register in order 2 -> 1 */ + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL)); + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL)); + + check_fprobe_multi(test); + + /* Unregister all */ + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1)); + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2)); +} + +static unsigned long get_ftrace_location(void *func) +{ + unsigned long size, addr = (unsigned long)func; + + if (!kallsyms_lookup_size_offset(addr, &size, NULL) || !size) + return 0; + + return ftrace_location_range(addr, addr + size - 1); +} + +static int fprobe_test_init(struct kunit *test) +{ + rand1 = get_random_u32_above(div_factor); + target = fprobe_selftest_target; + target2 = fprobe_selftest_target2; + target_ip = get_ftrace_location(target); + target2_ip = get_ftrace_location(target2); + + return 0; +} + +static struct kunit_case fprobe_testcases[] = { + KUNIT_CASE(test_fprobe_entry), + KUNIT_CASE(test_fprobe), + KUNIT_CASE(test_fprobe_syms), + KUNIT_CASE(test_fprobe_data), + KUNIT_CASE(test_fprobe_skip), + KUNIT_CASE(test_fprobe_multi), + {} +}; + +static struct kunit_suite fprobe_test_suite = { + .name = "fprobe_test", + .init = fprobe_test_init, + .test_cases = fprobe_testcases, +}; + +kunit_test_suites(&fprobe_test_suite); + -- cgit v1.3.1