sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 324 insertions, 0 deletions

diff --git a/drivers/mtd/maps/pci.c b/drivers/mtd/maps/pci.c
new file mode 100644
index 000000000..34041cc8f
--- /dev/null
+++ b/drivers/mtd/maps/pci.c
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  linux/drivers/mtd/maps/pci.c
+ *
+ *  Copyright (C) 2001 Russell King, All rights reserved.
+ *
+ * Generic PCI memory map driver.  We support the following boards:
+ *  - Intel IQ80310 ATU.
+ *  - Intel EBSA285 (blank rom programming mode). Tested working 27/09/2001
+ */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#include <linux/mtd/mtd.h>
+#include <linux/mtd/map.h>
+#include <linux/mtd/partitions.h>
+
+struct map_pci_info;
+
+struct mtd_pci_info {
+	int  (*init)(struct pci_dev *dev, struct map_pci_info *map);
+	void (*exit)(struct pci_dev *dev, struct map_pci_info *map);
+	unsigned long (*translate)(struct map_pci_info *map, unsigned long ofs);
+	const char *map_name;
+};
+
+struct map_pci_info {
+	struct map_info map;
+	void __iomem *base;
+	void (*exit)(struct pci_dev *dev, struct map_pci_info *map);
+	unsigned long (*translate)(struct map_pci_info *map, unsigned long ofs);
+	struct pci_dev *dev;
+};
+
+static map_word mtd_pci_read8(struct map_info *_map, unsigned long ofs)
+{
+	struct map_pci_info *map = (struct map_pci_info *)_map;
+	map_word val;
+	val.x[0]= readb(map->base + map->translate(map, ofs));
+	return val;
+}
+
+static map_word mtd_pci_read32(struct map_info *_map, unsigned long ofs)
+{
+	struct map_pci_info *map = (struct map_pci_info *)_map;
+	map_word val;
+	val.x[0] = readl(map->base + map->translate(map, ofs));
+	return val;
+}
+
+static void mtd_pci_copyfrom(struct map_info *_map, void *to, unsigned long from, ssize_t len)
+{
+	struct map_pci_info *map = (struct map_pci_info *)_map;
+	memcpy_fromio(to, map->base + map->translate(map, from), len);
+}
+
+static void mtd_pci_write8(struct map_info *_map, map_word val, unsigned long ofs)
+{
+	struct map_pci_info *map = (struct map_pci_info *)_map;
+	writeb(val.x[0], map->base + map->translate(map, ofs));
+}
+
+static void mtd_pci_write32(struct map_info *_map, map_word val, unsigned long ofs)
+{
+	struct map_pci_info *map = (struct map_pci_info *)_map;
+	writel(val.x[0], map->base + map->translate(map, ofs));
+}
+
+static void mtd_pci_copyto(struct map_info *_map, unsigned long to, const void *from, ssize_t len)
+{
+	struct map_pci_info *map = (struct map_pci_info *)_map;
+	memcpy_toio(map->base + map->translate(map, to), from, len);
+}
+
+static const struct map_info mtd_pci_map = {
+	.phys =		NO_XIP,
+	.copy_from =	mtd_pci_copyfrom,
+	.copy_to =	mtd_pci_copyto,
+};
+
+/*
+ * Intel IOP80310 Flash driver
+ */
+
+static int
+intel_iq80310_init(struct pci_dev *dev, struct map_pci_info *map)
+{
+	u32 win_base;
+
+	map->map.bankwidth = 1;
+	map->map.read = mtd_pci_read8;
+	map->map.write = mtd_pci_write8;
+
+	map->map.size     = 0x00800000;
+	map->base         = ioremap(pci_resource_start(dev, 0),
+					    pci_resource_len(dev, 0));
+
+	if (!map->base)
+		return -ENOMEM;
+
+	/*
+	 * We want to base the memory window at Xscale
+	 * bus address 0, not 0x1000.
+	 */
+	pci_read_config_dword(dev, 0x44, &win_base);
+	pci_write_config_dword(dev, 0x44, 0);
+
+	map->map.map_priv_2 = win_base;
+
+	return 0;
+}
+
+static void
+intel_iq80310_exit(struct pci_dev *dev, struct map_pci_info *map)
+{
+	if (map->base)
+		iounmap(map->base);
+	pci_write_config_dword(dev, 0x44, map->map.map_priv_2);
+}
+
+static unsigned long
+intel_iq80310_translate(struct map_pci_info *map, unsigned long ofs)
+{
+	unsigned long page_addr = ofs & 0x00400000;
+
+	/*
+	 * This mundges the flash location so we avoid
+	 * the first 80 bytes (they appear to read nonsense).
+	 */
+	if (page_addr) {
+		writel(0x00000008, map->base + 0x1558);
+		writel(0x00000000, map->base + 0x1550);
+	} else {
+		writel(0x00000007, map->base + 0x1558);
+		writel(0x00800000, map->base + 0x1550);
+		ofs += 0x00800000;
+	}
+
+	return ofs;
+}
+
+static struct mtd_pci_info intel_iq80310_info = {
+	.init =		intel_iq80310_init,
+	.exit =		intel_iq80310_exit,
+	.translate =	intel_iq80310_translate,
+	.map_name =	"cfi_probe",
+};
+
+/*
+ * Intel DC21285 driver
+ */
+
+static int
+intel_dc21285_init(struct pci_dev *dev, struct map_pci_info *map)
+{
+	unsigned long base, len;
+
+	base = pci_resource_start(dev, PCI_ROM_RESOURCE);
+	len  = pci_resource_len(dev, PCI_ROM_RESOURCE);
+
+	if (!len || !base) {
+		/*
+		 * No ROM resource
+		 */
+		base = pci_resource_start(dev, 2);
+		len  = pci_resource_len(dev, 2);
+
+		/*
+		 * We need to re-allocate PCI BAR2 address range to the
+		 * PCI ROM BAR, and disable PCI BAR2.
+		 */
+	} else {
+		/*
+		 * Hmm, if an address was allocated to the ROM resource, but
+		 * not enabled, should we be allocating a new resource for it
+		 * or simply enabling it?
+		 */
+		pci_enable_rom(dev);
+		printk("%s: enabling expansion ROM\n", pci_name(dev));
+	}
+
+	if (!len || !base)
+		return -ENXIO;
+
+	map->map.bankwidth = 4;
+	map->map.read = mtd_pci_read32;
+	map->map.write = mtd_pci_write32;
+	map->map.size     = len;
+	map->base         = ioremap(base, len);
+
+	if (!map->base)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void
+intel_dc21285_exit(struct pci_dev *dev, struct map_pci_info *map)
+{
+	if (map->base)
+		iounmap(map->base);
+
+	/*
+	 * We need to undo the PCI BAR2/PCI ROM BAR address alteration.
+	 */
+	pci_disable_rom(dev);
+}
+
+static unsigned long
+intel_dc21285_translate(struct map_pci_info *map, unsigned long ofs)
+{
+	return ofs & 0x00ffffc0 ? ofs : (ofs ^ (1 << 5));
+}
+
+static struct mtd_pci_info intel_dc21285_info = {
+	.init =		intel_dc21285_init,
+	.exit =		intel_dc21285_exit,
+	.translate =	intel_dc21285_translate,
+	.map_name =	"jedec_probe",
+};
+
+/*
+ * PCI device ID table
+ */
+
+static const struct pci_device_id mtd_pci_ids[] = {
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x530d),
+		.class =	PCI_CLASS_MEMORY_OTHER << 8,
+		.class_mask =	0xffff00,
+		.driver_data =	(unsigned long)&intel_iq80310_info,
+	}, {
+		/* DC21285 defaults to 0 for .subvendor and .subdevice on reset */
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_21285, 0, 0),
+		.driver_data =	(unsigned long)&intel_dc21285_info,
+	},
+	{ }
+};
+
+/*
+ * Generic code follows.
+ */
+
+static int mtd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	struct mtd_pci_info *info = (struct mtd_pci_info *)id->driver_data;
+	struct map_pci_info *map = NULL;
+	struct mtd_info *mtd = NULL;
+	int err;
+
+	err = pci_enable_device(dev);
+	if (err)
+		goto out;
+
+	err = pci_request_regions(dev, "pci mtd");
+	if (err)
+		goto out;
+
+	map = kmalloc_obj(*map);
+	err = -ENOMEM;
+	if (!map)
+		goto release;
+
+	map->map       = mtd_pci_map;
+	map->map.name  = pci_name(dev);
+	map->dev       = dev;
+	map->exit      = info->exit;
+	map->translate = info->translate;
+
+	err = info->init(dev, map);
+	if (err)
+		goto release;
+
+	mtd = do_map_probe(info->map_name, &map->map);
+	err = -ENODEV;
+	if (!mtd)
+		goto release;
+
+	mtd->owner = THIS_MODULE;
+	mtd_device_register(mtd, NULL, 0);
+
+	pci_set_drvdata(dev, mtd);
+
+	return 0;
+
+release:
+	if (map) {
+		map->exit(dev, map);
+		kfree(map);
+	}
+
+	pci_release_regions(dev);
+out:
+	return err;
+}
+
+static void mtd_pci_remove(struct pci_dev *dev)
+{
+	struct mtd_info *mtd = pci_get_drvdata(dev);
+	struct map_pci_info *map = mtd->priv;
+
+	mtd_device_unregister(mtd);
+	map_destroy(mtd);
+	map->exit(dev, map);
+	kfree(map);
+
+	pci_release_regions(dev);
+}
+
+static struct pci_driver mtd_pci_driver = {
+	.name =		"MTD PCI",
+	.probe =	mtd_pci_probe,
+	.remove =	mtd_pci_remove,
+	.id_table =	mtd_pci_ids,
+};
+
+module_pci_driver(mtd_pci_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Russell King <rmk@arm.linux.org.uk>");
+MODULE_DESCRIPTION("Generic PCI map driver");
+MODULE_DEVICE_TABLE(pci, mtd_pci_ids);