sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 215 insertions, 0 deletions

diff --git a/drivers/mtd/tests/readtest.c b/drivers/mtd/tests/readtest.c
new file mode 100644
index 000000000..99670ef91
--- /dev/null
+++ b/drivers/mtd/tests/readtest.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2006-2008 Nokia Corporation
+ *
+ * Check MTD device read.
+ *
+ * Author: Adrian Hunter <ext-adrian.hunter@nokia.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/err.h>
+#include <linux/mtd/mtd.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+
+#include "mtd_test.h"
+
+static int dev = -EINVAL;
+module_param(dev, int, S_IRUGO);
+MODULE_PARM_DESC(dev, "MTD device number to use");
+
+static struct mtd_info *mtd;
+static unsigned char *iobuf;
+static unsigned char *iobuf1;
+static unsigned char *bbt;
+
+static int pgsize;
+static int ebcnt;
+static int pgcnt;
+
+static int read_eraseblock_by_page(int ebnum)
+{
+	int i, ret, err = 0;
+	loff_t addr = (loff_t)ebnum * mtd->erasesize;
+	void *buf = iobuf;
+	void *oobbuf = iobuf1;
+
+	for (i = 0; i < pgcnt; i++) {
+		memset(buf, 0 , pgsize);
+		ret = mtdtest_read(mtd, addr, pgsize, buf);
+		if (ret) {
+			if (!err)
+				err = ret;
+		}
+		if (mtd->oobsize) {
+			struct mtd_oob_ops ops = { };
+
+			ops.mode      = MTD_OPS_PLACE_OOB;
+			ops.len       = 0;
+			ops.retlen    = 0;
+			ops.ooblen    = mtd->oobsize;
+			ops.oobretlen = 0;
+			ops.ooboffs   = 0;
+			ops.datbuf    = NULL;
+			ops.oobbuf    = oobbuf;
+			ret = mtd_read_oob(mtd, addr, &ops);
+			if ((ret && !mtd_is_bitflip(ret)) ||
+					ops.oobretlen != mtd->oobsize) {
+				pr_err("error: read oob failed at "
+						  "%#llx\n", (long long)addr);
+				if (!err)
+					err = ret;
+				if (!err)
+					err = -EINVAL;
+			}
+			oobbuf += mtd->oobsize;
+		}
+		addr += pgsize;
+		buf += pgsize;
+	}
+
+	return err;
+}
+
+static void dump_eraseblock(int ebnum)
+{
+	int i, j, n;
+	char line[128];
+	int pg, oob;
+
+	pr_info("dumping eraseblock %d\n", ebnum);
+	n = mtd->erasesize;
+	for (i = 0; i < n;) {
+		char *p = line;
+
+		p += sprintf(p, "%05x: ", i);
+		for (j = 0; j < 32 && i < n; j++, i++)
+			p += sprintf(p, "%02x", (unsigned int)iobuf[i]);
+		printk(KERN_CRIT "%s\n", line);
+		cond_resched();
+	}
+	if (!mtd->oobsize)
+		return;
+	pr_info("dumping oob from eraseblock %d\n", ebnum);
+	n = mtd->oobsize;
+	for (pg = 0, i = 0; pg < pgcnt; pg++)
+		for (oob = 0; oob < n;) {
+			char *p = line;
+
+			p += sprintf(p, "%05x: ", i);
+			for (j = 0; j < 32 && oob < n; j++, oob++, i++)
+				p += sprintf(p, "%02x",
+					     (unsigned int)iobuf1[i]);
+			printk(KERN_CRIT "%s\n", line);
+			cond_resched();
+		}
+}
+
+static int __init mtd_readtest_init(void)
+{
+	uint64_t tmp;
+	int err, i;
+
+	printk(KERN_INFO "\n");
+	printk(KERN_INFO "=================================================\n");
+
+	if (dev < 0) {
+		pr_info("Please specify a valid mtd-device via module parameter\n");
+		return -EINVAL;
+	}
+
+	pr_info("MTD device: %d\n", dev);
+
+	mtd = get_mtd_device(NULL, dev);
+	if (IS_ERR(mtd)) {
+		err = PTR_ERR(mtd);
+		pr_err("error: Cannot get MTD device\n");
+		return err;
+	}
+
+	if (mtd->writesize == 1) {
+		pr_info("not NAND flash, assume page size is 512 "
+		       "bytes.\n");
+		pgsize = 512;
+	} else
+		pgsize = mtd->writesize;
+
+	tmp = mtd->size;
+	do_div(tmp, mtd->erasesize);
+	ebcnt = tmp;
+	pgcnt = mtd->erasesize / pgsize;
+
+	pr_info("MTD device size %llu, eraseblock size %u, "
+	       "page size %u, count of eraseblocks %u, pages per "
+	       "eraseblock %u, OOB size %u\n",
+	       (unsigned long long)mtd->size, mtd->erasesize,
+	       pgsize, ebcnt, pgcnt, mtd->oobsize);
+
+	err = -ENOMEM;
+	iobuf = kmalloc(mtd->erasesize, GFP_KERNEL);
+	if (!iobuf)
+		goto out;
+	iobuf1 = kmalloc(mtd->erasesize, GFP_KERNEL);
+	if (!iobuf1)
+		goto out;
+
+	bbt = kzalloc(ebcnt, GFP_KERNEL);
+	if (!bbt)
+		goto out;
+	err = mtdtest_scan_for_bad_eraseblocks(mtd, bbt, 0, ebcnt);
+	if (err)
+		goto out;
+
+	/* Read all eraseblocks 1 page at a time */
+	pr_info("testing page read\n");
+	for (i = 0; i < ebcnt; ++i) {
+		int ret;
+
+		if (bbt[i])
+			continue;
+		ret = read_eraseblock_by_page(i);
+		if (ret) {
+			dump_eraseblock(i);
+			if (!err)
+				err = ret;
+		}
+
+		ret = mtdtest_relax();
+		if (ret) {
+			err = ret;
+			goto out;
+		}
+	}
+
+	if (err)
+		pr_info("finished with errors\n");
+	else
+		pr_info("finished\n");
+
+out:
+
+	kfree(iobuf);
+	kfree(iobuf1);
+	kfree(bbt);
+	put_mtd_device(mtd);
+	if (err)
+		pr_info("error %d occurred\n", err);
+	printk(KERN_INFO "=================================================\n");
+	return err;
+}
+module_init(mtd_readtest_init);
+
+static void __exit mtd_readtest_exit(void)
+{
+	return;
+}
+module_exit(mtd_readtest_exit);
+
+MODULE_DESCRIPTION("Read test module");
+MODULE_AUTHOR("Adrian Hunter");
+MODULE_LICENSE("GPL");