sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 259 insertions, 0 deletions

diff --git a/include/drm/drm_fixed.h b/include/drm/drm_fixed.h
new file mode 100644
index 000000000..21d822aee
--- /dev/null
+++ b/include/drm/drm_fixed.h
@@ -0,0 +1,259 @@
+/*
+ * Copyright 2009 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Dave Airlie
+ *          Christian König
+ */
+#ifndef DRM_FIXED_H
+#define DRM_FIXED_H
+
+#include <linux/math64.h>
+#include <linux/types.h>
+#include <linux/wordpart.h>
+
+typedef union dfixed {
+	u32 full;
+} fixed20_12;
+
+
+#define dfixed_const(A) (u32)(((A) << 12))/*  + ((B + 0.000122)*4096)) */
+#define dfixed_const_half(A) (u32)(((A) << 12) + 2048)
+#define dfixed_const_666(A) (u32)(((A) << 12) + 2731)
+#define dfixed_const_8(A) (u32)(((A) << 12) + 3277)
+#define dfixed_mul(A, B) ((u64)((u64)(A).full * (B).full + 2048) >> 12)
+#define dfixed_init(A) { .full = dfixed_const((A)) }
+#define dfixed_init_half(A) { .full = dfixed_const_half((A)) }
+#define dfixed_trunc(A) ((A).full >> 12)
+#define dfixed_frac(A) ((A).full & ((1 << 12) - 1))
+
+static inline u32 dfixed_floor(fixed20_12 A)
+{
+	u32 non_frac = dfixed_trunc(A);
+
+	return dfixed_const(non_frac);
+}
+
+static inline u32 dfixed_ceil(fixed20_12 A)
+{
+	u32 non_frac = dfixed_trunc(A);
+
+	if (A.full > dfixed_const(non_frac))
+		return dfixed_const(non_frac + 1);
+	else
+		return dfixed_const(non_frac);
+}
+
+static inline u32 dfixed_div(fixed20_12 A, fixed20_12 B)
+{
+	u64 tmp = ((u64)A.full << 13);
+
+	do_div(tmp, B.full);
+	tmp += 1;
+	tmp /= 2;
+	return lower_32_bits(tmp);
+}
+
+#define DRM_FIXED_POINT		32
+#define DRM_FIXED_ONE		(1ULL << DRM_FIXED_POINT)
+#define DRM_FIXED_DECIMAL_MASK	(DRM_FIXED_ONE - 1)
+#define DRM_FIXED_DIGITS_MASK	(~DRM_FIXED_DECIMAL_MASK)
+#define DRM_FIXED_EPSILON	1LL
+#define DRM_FIXED_ALMOST_ONE	(DRM_FIXED_ONE - DRM_FIXED_EPSILON)
+
+/**
+ * drm_sm2fixp() - convert signed-magnitude to fixed point
+ * @a: 1.31.32 signed-magnitude fixed point
+ *
+ * Convert a 1.31.32 signed-magnitude fixed point to 32.32
+ * 2s-complement fixed point
+ *
+ * @return s64 2s-complement fixed point
+ */
+static inline s64 drm_sm2fixp(__u64 a)
+{
+	if ((a & (1LL << 63))) {
+		return -(a & 0x7fffffffffffffffll);
+	} else {
+		return a;
+	}
+}
+
+static inline s64 drm_int2fixp(int a)
+{
+	return ((s64)a) << DRM_FIXED_POINT;
+}
+
+static inline int drm_fixp2int(s64 a)
+{
+	return ((s64)a) >> DRM_FIXED_POINT;
+}
+
+static inline int drm_fixp2int_round(s64 a)
+{
+	return drm_fixp2int(a + DRM_FIXED_ONE / 2);
+}
+
+static inline int drm_fixp2int_ceil(s64 a)
+{
+	if (a >= 0)
+		return drm_fixp2int(a + DRM_FIXED_ALMOST_ONE);
+	else
+		return drm_fixp2int(a - DRM_FIXED_ALMOST_ONE);
+}
+
+static inline unsigned drm_fixp_msbset(s64 a)
+{
+	unsigned shift, sign = (a >> 63) & 1;
+
+	for (shift = 62; shift > 0; --shift)
+		if (((a >> shift) & 1) != sign)
+			return shift;
+
+	return 0;
+}
+
+static inline s64 drm_fixp_mul(s64 a, s64 b)
+{
+	unsigned shift = drm_fixp_msbset(a) + drm_fixp_msbset(b);
+	s64 result;
+
+	if (shift > 61) {
+		shift = shift - 61;
+		a >>= (shift >> 1) + (shift & 1);
+		b >>= shift >> 1;
+	} else
+		shift = 0;
+
+	result = a * b;
+
+	if (shift > DRM_FIXED_POINT)
+		return result << (shift - DRM_FIXED_POINT);
+
+	if (shift < DRM_FIXED_POINT)
+		return result >> (DRM_FIXED_POINT - shift);
+
+	return result;
+}
+
+static inline s64 drm_fixp_div(s64 a, s64 b)
+{
+	unsigned shift = 62 - drm_fixp_msbset(a);
+	s64 result;
+
+	a <<= shift;
+
+	if (shift < DRM_FIXED_POINT)
+		b >>= (DRM_FIXED_POINT - shift);
+
+	result = div64_s64(a, b);
+
+	if (shift > DRM_FIXED_POINT)
+		return result >> (shift - DRM_FIXED_POINT);
+
+	return result;
+}
+
+static inline s64 drm_fixp_from_fraction(s64 a, s64 b)
+{
+	s64 res;
+	bool a_neg = a < 0;
+	bool b_neg = b < 0;
+	u64 a_abs = a_neg ? -a : a;
+	u64 b_abs = b_neg ? -b : b;
+	u64 rem;
+
+	/* determine integer part */
+	u64 res_abs  = div64_u64_rem(a_abs, b_abs, &rem);
+
+	/* determine fractional part */
+	{
+		u32 i = DRM_FIXED_POINT;
+
+		do {
+			rem <<= 1;
+			res_abs <<= 1;
+			if (rem >= b_abs) {
+				res_abs |= 1;
+				rem -= b_abs;
+			}
+		} while (--i != 0);
+	}
+
+	/* round up LSB */
+	{
+		u64 summand = (rem << 1) >= b_abs;
+
+		res_abs += summand;
+	}
+
+	res = (s64) res_abs;
+	if (a_neg ^ b_neg)
+		res = -res;
+	return res;
+}
+
+static inline s64 drm_fixp_exp(s64 x)
+{
+	s64 tolerance = div64_s64(DRM_FIXED_ONE, 1000000);
+	s64 sum = DRM_FIXED_ONE, term, y = x;
+	u64 count = 1;
+
+	if (x < 0)
+		y = -1 * x;
+
+	term = y;
+
+	while (term >= tolerance) {
+		sum = sum + term;
+		count = count + 1;
+		term = drm_fixp_mul(term, div64_s64(y, count));
+	}
+
+	if (x < 0)
+		sum = drm_fixp_div(DRM_FIXED_ONE, sum);
+
+	return sum;
+}
+
+static inline int fxp_q4_from_int(int val_int)
+{
+	return val_int << 4;
+}
+
+static inline int fxp_q4_to_int(int val_q4)
+{
+	return val_q4 >> 4;
+}
+
+static inline int fxp_q4_to_int_roundup(int val_q4)
+{
+	return (val_q4 + 0xf) >> 4;
+}
+
+static inline int fxp_q4_to_frac(int val_q4)
+{
+	return val_q4 & 0xf;
+}
+
+#define FXP_Q4_FMT		"%d.%04d"
+#define FXP_Q4_ARGS(val_q4)	fxp_q4_to_int(val_q4), (fxp_q4_to_frac(val_q4) * 625)
+
+#endif