sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 232 insertions, 0 deletions

diff --git a/drivers/mtd/tests/stresstest.c b/drivers/mtd/tests/stresstest.c
new file mode 100644
index 000000000..944f960a4
--- /dev/null
+++ b/drivers/mtd/tests/stresstest.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2006-2008 Nokia Corporation
+ *
+ * Test random reads, writes and erases on MTD device.
+ *
+ * Author: Adrian Hunter <ext-adrian.hunter@nokia.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/err.h>
+#include <linux/mtd/mtd.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/vmalloc.h>
+#include <linux/random.h>
+
+#include "mtd_test.h"
+
+static int dev = -EINVAL;
+module_param(dev, int, S_IRUGO);
+MODULE_PARM_DESC(dev, "MTD device number to use");
+
+static int count = 10000;
+module_param(count, int, S_IRUGO);
+MODULE_PARM_DESC(count, "Number of operations to do (default is 10000)");
+
+static struct mtd_info *mtd;
+static unsigned char *writebuf;
+static unsigned char *readbuf;
+static unsigned char *bbt;
+static int *offsets;
+
+static int pgsize;
+static int bufsize;
+static int ebcnt;
+static int pgcnt;
+
+static int rand_eb(void)
+{
+	unsigned int eb;
+
+again:
+	/* Read or write up 2 eraseblocks at a time - hence 'ebcnt - 1' */
+	eb = get_random_u32_below(ebcnt - 1);
+	if (bbt[eb])
+		goto again;
+	return eb;
+}
+
+static int rand_offs(void)
+{
+	return get_random_u32_below(bufsize);
+}
+
+static int rand_len(int offs)
+{
+	return get_random_u32_below(bufsize - offs);
+}
+
+static int do_read(void)
+{
+	int eb = rand_eb();
+	int offs = rand_offs();
+	int len = rand_len(offs);
+	loff_t addr;
+
+	if (bbt[eb + 1]) {
+		if (offs >= mtd->erasesize)
+			offs -= mtd->erasesize;
+		if (offs + len > mtd->erasesize)
+			len = mtd->erasesize - offs;
+	}
+	addr = (loff_t)eb * mtd->erasesize + offs;
+	return mtdtest_read(mtd, addr, len, readbuf);
+}
+
+static int do_write(void)
+{
+	int eb = rand_eb(), offs, err, len;
+	loff_t addr;
+
+	offs = offsets[eb];
+	if (offs >= mtd->erasesize) {
+		err = mtdtest_erase_eraseblock(mtd, eb);
+		if (err)
+			return err;
+		offs = offsets[eb] = 0;
+	}
+	len = rand_len(offs);
+	len = ((len + pgsize - 1) / pgsize) * pgsize;
+	if (offs + len > mtd->erasesize) {
+		if (bbt[eb + 1])
+			len = mtd->erasesize - offs;
+		else {
+			err = mtdtest_erase_eraseblock(mtd, eb + 1);
+			if (err)
+				return err;
+			offsets[eb + 1] = 0;
+		}
+	}
+	addr = (loff_t)eb * mtd->erasesize + offs;
+	err = mtdtest_write(mtd, addr, len, writebuf);
+	if (unlikely(err))
+		return err;
+	offs += len;
+	while (offs > mtd->erasesize) {
+		offsets[eb++] = mtd->erasesize;
+		offs -= mtd->erasesize;
+	}
+	offsets[eb] = offs;
+	return 0;
+}
+
+static int do_operation(void)
+{
+	if (get_random_u32_below(2))
+		return do_read();
+	else
+		return do_write();
+}
+
+static int __init mtd_stresstest_init(void)
+{
+	int err;
+	int i, op;
+	uint64_t tmp;
+
+	printk(KERN_INFO "\n");
+	printk(KERN_INFO "=================================================\n");
+
+	if (dev < 0) {
+		pr_info("Please specify a valid mtd-device via module parameter\n");
+		pr_crit("CAREFUL: This test wipes all data on the specified MTD device!\n");
+		return -EINVAL;
+	}
+
+	pr_info("MTD device: %d\n", dev);
+
+	mtd = get_mtd_device(NULL, dev);
+	if (IS_ERR(mtd)) {
+		err = PTR_ERR(mtd);
+		pr_err("error: cannot get MTD device\n");
+		return err;
+	}
+
+	if (mtd->writesize == 1) {
+		pr_info("not NAND flash, assume page size is 512 "
+		       "bytes.\n");
+		pgsize = 512;
+	} else
+		pgsize = mtd->writesize;
+
+	tmp = mtd->size;
+	do_div(tmp, mtd->erasesize);
+	ebcnt = tmp;
+	pgcnt = mtd->erasesize / pgsize;
+
+	pr_info("MTD device size %llu, eraseblock size %u, "
+	       "page size %u, count of eraseblocks %u, pages per "
+	       "eraseblock %u, OOB size %u\n",
+	       (unsigned long long)mtd->size, mtd->erasesize,
+	       pgsize, ebcnt, pgcnt, mtd->oobsize);
+
+	if (ebcnt < 2) {
+		pr_err("error: need at least 2 eraseblocks\n");
+		err = -ENOSPC;
+		goto out_put_mtd;
+	}
+
+	/* Read or write up 2 eraseblocks at a time */
+	bufsize = mtd->erasesize * 2;
+
+	err = -ENOMEM;
+	readbuf = vmalloc(bufsize);
+	writebuf = vmalloc(bufsize);
+	offsets = kmalloc_objs(int, ebcnt);
+	if (!readbuf || !writebuf || !offsets)
+		goto out;
+	for (i = 0; i < ebcnt; i++)
+		offsets[i] = mtd->erasesize;
+	get_random_bytes(writebuf, bufsize);
+
+	bbt = kzalloc(ebcnt, GFP_KERNEL);
+	if (!bbt)
+		goto out;
+	err = mtdtest_scan_for_bad_eraseblocks(mtd, bbt, 0, ebcnt);
+	if (err)
+		goto out;
+
+	/* Do operations */
+	pr_info("doing operations\n");
+	for (op = 0; op < count; op++) {
+		if ((op & 1023) == 0)
+			pr_info("%d operations done\n", op);
+		err = do_operation();
+		if (err)
+			goto out;
+
+		err = mtdtest_relax();
+		if (err)
+			goto out;
+	}
+	pr_info("finished, %d operations done\n", op);
+
+out:
+	kfree(offsets);
+	kfree(bbt);
+	vfree(writebuf);
+	vfree(readbuf);
+out_put_mtd:
+	put_mtd_device(mtd);
+	if (err)
+		pr_info("error %d occurred\n", err);
+	printk(KERN_INFO "=================================================\n");
+	return err;
+}
+module_init(mtd_stresstest_init);
+
+static void __exit mtd_stresstest_exit(void)
+{
+	return;
+}
+module_exit(mtd_stresstest_exit);
+
+MODULE_DESCRIPTION("Stress test module");
+MODULE_AUTHOR("Adrian Hunter");
+MODULE_LICENSE("GPL");