sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 145 insertions, 0 deletions

diff --git a/include/net/gre.h b/include/net/gre.h
new file mode 100644
index 000000000..b55f67ecd
--- /dev/null
+++ b/include/net/gre.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_GRE_H
+#define __LINUX_GRE_H
+
+#include <linux/skbuff.h>
+#include <net/ip_tunnels.h>
+
+struct gre_base_hdr {
+	__be16 flags;
+	__be16 protocol;
+} __packed;
+
+struct gre_full_hdr {
+	struct gre_base_hdr fixed_header;
+	__be16 csum;
+	__be16 reserved1;
+	__be32 key;
+	__be32 seq;
+} __packed;
+#define GRE_HEADER_SECTION 4
+
+#define GREPROTO_CISCO		0
+#define GREPROTO_PPTP		1
+#define GREPROTO_MAX		2
+#define GRE_IP_PROTO_MAX	2
+
+struct gre_protocol {
+	int  (*handler)(struct sk_buff *skb);
+	void (*err_handler)(struct sk_buff *skb, u32 info);
+};
+
+int gre_add_protocol(const struct gre_protocol *proto, u8 version);
+int gre_del_protocol(const struct gre_protocol *proto, u8 version);
+
+int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi,
+		     bool *csum_err, __be16 proto, int nhs);
+
+static inline bool netif_is_gretap(const struct net_device *dev)
+{
+	return dev->rtnl_link_ops &&
+	       !strcmp(dev->rtnl_link_ops->kind, "gretap");
+}
+
+static inline bool netif_is_ip6gretap(const struct net_device *dev)
+{
+	return dev->rtnl_link_ops &&
+	       !strcmp(dev->rtnl_link_ops->kind, "ip6gretap");
+}
+
+static inline int gre_calc_hlen(const unsigned long *o_flags)
+{
+	int addend = 4;
+
+	if (test_bit(IP_TUNNEL_CSUM_BIT, o_flags))
+		addend += 4;
+	if (test_bit(IP_TUNNEL_KEY_BIT, o_flags))
+		addend += 4;
+	if (test_bit(IP_TUNNEL_SEQ_BIT, o_flags))
+		addend += 4;
+	return addend;
+}
+
+static inline void gre_flags_to_tnl_flags(unsigned long *dst, __be16 flags)
+{
+	IP_TUNNEL_DECLARE_FLAGS(res) = { };
+
+	__assign_bit(IP_TUNNEL_CSUM_BIT, res, flags & GRE_CSUM);
+	__assign_bit(IP_TUNNEL_ROUTING_BIT, res, flags & GRE_ROUTING);
+	__assign_bit(IP_TUNNEL_KEY_BIT, res, flags & GRE_KEY);
+	__assign_bit(IP_TUNNEL_SEQ_BIT, res, flags & GRE_SEQ);
+	__assign_bit(IP_TUNNEL_STRICT_BIT, res, flags & GRE_STRICT);
+	__assign_bit(IP_TUNNEL_REC_BIT, res, flags & GRE_REC);
+	__assign_bit(IP_TUNNEL_VERSION_BIT, res, flags & GRE_VERSION);
+
+	ip_tunnel_flags_copy(dst, res);
+}
+
+static inline __be16 gre_tnl_flags_to_gre_flags(const unsigned long *tflags)
+{
+	__be16 flags = 0;
+
+	if (test_bit(IP_TUNNEL_CSUM_BIT, tflags))
+		flags |= GRE_CSUM;
+	if (test_bit(IP_TUNNEL_ROUTING_BIT, tflags))
+		flags |= GRE_ROUTING;
+	if (test_bit(IP_TUNNEL_KEY_BIT, tflags))
+		flags |= GRE_KEY;
+	if (test_bit(IP_TUNNEL_SEQ_BIT, tflags))
+		flags |= GRE_SEQ;
+	if (test_bit(IP_TUNNEL_STRICT_BIT, tflags))
+		flags |= GRE_STRICT;
+	if (test_bit(IP_TUNNEL_REC_BIT, tflags))
+		flags |= GRE_REC;
+	if (test_bit(IP_TUNNEL_VERSION_BIT, tflags))
+		flags |= GRE_VERSION;
+
+	return flags;
+}
+
+static inline void gre_build_header(struct sk_buff *skb, int hdr_len,
+				    const unsigned long *flags, __be16 proto,
+				    __be32 key, __be32 seq)
+{
+	IP_TUNNEL_DECLARE_FLAGS(cond) = { };
+	struct gre_base_hdr *greh;
+
+	skb_push(skb, hdr_len);
+
+	skb_set_inner_protocol(skb, proto);
+	skb_reset_transport_header(skb);
+	greh = (struct gre_base_hdr *)skb->data;
+	greh->flags = gre_tnl_flags_to_gre_flags(flags);
+	greh->protocol = proto;
+
+	__set_bit(IP_TUNNEL_KEY_BIT, cond);
+	__set_bit(IP_TUNNEL_CSUM_BIT, cond);
+	__set_bit(IP_TUNNEL_SEQ_BIT, cond);
+
+	if (ip_tunnel_flags_intersect(flags, cond)) {
+		__be32 *ptr = (__be32 *)(((u8 *)greh) + hdr_len - 4);
+
+		if (test_bit(IP_TUNNEL_SEQ_BIT, flags)) {
+			*ptr = seq;
+			ptr--;
+		}
+		if (test_bit(IP_TUNNEL_KEY_BIT, flags)) {
+			*ptr = key;
+			ptr--;
+		}
+		if (test_bit(IP_TUNNEL_CSUM_BIT, flags) &&
+		    !(skb_shinfo(skb)->gso_type &
+		      (SKB_GSO_GRE | SKB_GSO_GRE_CSUM))) {
+			*ptr = 0;
+			if (skb->ip_summed == CHECKSUM_PARTIAL) {
+				*(__sum16 *)ptr = csum_fold(lco_csum(skb));
+			} else {
+				skb->ip_summed = CHECKSUM_PARTIAL;
+				skb->csum_start = skb_transport_header(skb) - skb->head;
+				skb->csum_offset = sizeof(*greh);
+			}
+		}
+	}
+}
+
+#endif