sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling buggrafted

alb_break_llc() decides whether to break LLC preference during active
load balance. It does so by testing that every runnable fair task on the
source rq prefers its LLC:

	env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable

But the two counters cover different sets. nr_pref_llc_running is updated
in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued,
so it follows queued tasks. h_nr_runnable is updated in set_delayed()/
clear_delayed() and drops delay-dequeued tasks.

So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted
in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks,
alb_break_llc() returns false, and active balance is free to pull a task
off its preferred LLC. Active balance only moves runnable tasks, and this
is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB
skips the per-task test in can_migrate_task(). The runnable set is the one
we want.

Fix it on the counter side. A task should be counted in
nr_pref_llc_running exactly while it is both queued on its preferred LLC
(pref_llc_queued) and runnable (!sched_delayed). Define that membership
once in task_pref_llc_runnable(), and adjust the counter only through
pref_llc_running_inc()/pref_llc_running_dec() from the four sites that
change either input: account_llc_enqueue(), account_llc_dequeue(),
set_delayed() and clear_delayed(). Gating every update on the same
predicate keeps the delay, wake and dequeue paths from double-counting
or underflowing; see the comments at those sites for the ordering.

nr_llc_running and sd->llc_counts are not touched and stay on queued
semantics.

Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC")
Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/
Reported-by: Zhan Xusheng <zhanxusheng@xiaomi.com>
Suggested-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Kayra Cizmeci <kayracizmeci@gmail.com>
Cc: <stable@kernel.org> # v7.2.x
Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com

1 files changed, 301 insertions, 0 deletions

diff --git a/include/trace/events/ceph.h b/include/trace/events/ceph.h
new file mode 100644
index 000000000..1b990632f
--- /dev/null
+++ b/include/trace/events/ceph.h
@@ -0,0 +1,301 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Ceph filesystem support module tracepoints
+ *
+ * Copyright (C) 2025 IONOS SE. All Rights Reserved.
+ * Written by Max Kellermann (max.kellermann@ionos.com)
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ceph
+
+#if !defined(_TRACE_CEPH_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_CEPH_H
+
+#include <linux/tracepoint.h>
+
+#define ceph_mdsc_suspend_reasons						\
+	EM(ceph_mdsc_suspend_reason_no_mdsmap,		"no-mdsmap")		\
+	EM(ceph_mdsc_suspend_reason_no_active_mds,	"no-active-mds")	\
+	EM(ceph_mdsc_suspend_reason_rejected,		"rejected")		\
+	E_(ceph_mdsc_suspend_reason_session,		"session")
+
+#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
+#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
+
+#undef EM
+#undef E_
+#define EM(a, b) a,
+#define E_(a, b) a
+
+enum ceph_mdsc_suspend_reason { ceph_mdsc_suspend_reasons } __mode(byte);
+
+#endif
+
+/*
+ * Export enum symbols via userspace.
+ */
+#undef EM
+#undef E_
+#define EM(a, b) TRACE_DEFINE_ENUM(a);
+#define E_(a, b) TRACE_DEFINE_ENUM(a);
+
+ceph_mdsc_suspend_reasons;
+
+/*
+ * Now redefine the EM() and E_() macros to map the enums to the strings that
+ * will be printed in the output.
+ */
+#undef EM
+#undef E_
+#define EM(a, b)	{ a, b },
+#define E_(a, b)	{ a, b }
+
+TRACE_EVENT(ceph_mdsc_submit_request,
+	TP_PROTO(struct ceph_mds_client *mdsc,
+		 struct ceph_mds_request *req),
+
+	TP_ARGS(mdsc, req),
+
+	TP_STRUCT__entry(
+		__field(u64,	tid)
+		__field(int,	op)
+		__field(u64,	ino)
+		__field(u64,	snap)
+	),
+
+	TP_fast_assign(
+		struct inode *inode;
+
+		__entry->tid = req->r_tid;
+		__entry->op = req->r_op;
+
+		inode = req->r_inode;
+		if (inode == NULL && req->r_dentry)
+			inode = d_inode(req->r_dentry);
+
+		if (inode) {
+			__entry->ino = ceph_ino(inode);
+			__entry->snap = ceph_snap(inode);
+		} else {
+			__entry->ino = __entry->snap = 0;
+		}
+	),
+
+	TP_printk("R=%llu op=%s ino=%llx,%llx",
+		  __entry->tid,
+		  ceph_mds_op_name(__entry->op),
+		  __entry->ino, __entry->snap)
+);
+
+TRACE_EVENT(ceph_mdsc_suspend_request,
+	TP_PROTO(struct ceph_mds_client *mdsc,
+		     struct ceph_mds_session *session,
+		     struct ceph_mds_request *req,
+		     enum ceph_mdsc_suspend_reason reason),
+
+	TP_ARGS(mdsc, session, req, reason),
+
+	TP_STRUCT__entry(
+		__field(u64,				tid)
+		__field(int,				op)
+		__field(int,				mds)
+		__field(enum ceph_mdsc_suspend_reason,	reason)
+	),
+
+	TP_fast_assign(
+		__entry->tid = req->r_tid;
+		__entry->op = req->r_op;
+		__entry->mds = session ? session->s_mds : -1;
+		__entry->reason = reason;
+	),
+
+	TP_printk("R=%llu op=%s reason=%s",
+		  __entry->tid,
+		  ceph_mds_op_name(__entry->op),
+		  __print_symbolic(__entry->reason, ceph_mdsc_suspend_reasons))
+);
+
+TRACE_EVENT(ceph_mdsc_resume_request,
+	TP_PROTO(struct ceph_mds_client *mdsc,
+		 struct ceph_mds_request *req),
+
+	TP_ARGS(mdsc, req),
+
+	TP_STRUCT__entry(
+		__field(u64,				tid)
+		__field(int,				op)
+	),
+
+	TP_fast_assign(
+		__entry->tid = req->r_tid;
+		__entry->op = req->r_op;
+	),
+
+	TP_printk("R=%llu op=%s",
+		  __entry->tid,
+		  ceph_mds_op_name(__entry->op))
+);
+
+TRACE_EVENT(ceph_mdsc_send_request,
+	TP_PROTO(struct ceph_mds_session *session,
+		 struct ceph_mds_request *req),
+
+	TP_ARGS(session, req),
+
+	TP_STRUCT__entry(
+		__field(u64,		tid)
+		__field(int,		op)
+		__field(int,		mds)
+	),
+
+	TP_fast_assign(
+		__entry->tid = req->r_tid;
+		__entry->op = req->r_op;
+		__entry->mds = session->s_mds;
+	),
+
+	TP_printk("R=%llu op=%s mds=%d",
+		  __entry->tid,
+		  ceph_mds_op_name(__entry->op),
+		  __entry->mds)
+);
+
+TRACE_EVENT(ceph_mdsc_complete_request,
+	TP_PROTO(struct ceph_mds_client *mdsc,
+		     struct ceph_mds_request *req),
+
+	TP_ARGS(mdsc, req),
+
+	TP_STRUCT__entry(
+		__field(u64,			tid)
+		__field(int,			op)
+		__field(int,			err)
+		__field(unsigned long,		latency_ns)
+	),
+
+	TP_fast_assign(
+		__entry->tid = req->r_tid;
+		__entry->op = req->r_op;
+		__entry->err = req->r_err;
+		__entry->latency_ns = req->r_end_latency - req->r_start_latency;
+	),
+
+	TP_printk("R=%llu op=%s err=%d latency_ns=%lu",
+		  __entry->tid,
+		  ceph_mds_op_name(__entry->op),
+		  __entry->err,
+		  __entry->latency_ns)
+);
+
+TRACE_EVENT(ceph_handle_caps,
+	TP_PROTO(struct ceph_mds_client *mdsc,
+		 struct ceph_mds_session *session,
+		 int op,
+		 const struct ceph_vino *vino,
+		 struct ceph_inode_info *inode,
+		 u32 seq, u32 mseq, u32 issue_seq),
+
+	TP_ARGS(mdsc, session, op, vino, inode, seq, mseq, issue_seq),
+
+	TP_STRUCT__entry(
+		__field(int,	mds)
+		__field(int,	op)
+		__field(u64,	ino)
+		__field(u64,	snap)
+		__field(u32,	seq)
+		__field(u32,	mseq)
+		__field(u32,	issue_seq)
+	),
+
+	TP_fast_assign(
+		__entry->mds = session->s_mds;
+		__entry->op = op;
+		__entry->ino = vino->ino;
+		__entry->snap = vino->snap;
+		__entry->seq = seq;
+		__entry->mseq = mseq;
+		__entry->issue_seq = issue_seq;
+	),
+
+	TP_printk("mds=%d op=%s vino=%llx.%llx seq=%u iseq=%u mseq=%u",
+		  __entry->mds,
+		  ceph_cap_op_name(__entry->op),
+		  __entry->ino,
+		  __entry->snap,
+		  __entry->seq,
+		  __entry->issue_seq,
+		  __entry->mseq)
+);
+
+/*
+ * Client reset tracepoints - identify the client by its monitor-
+ * assigned global_id so traces remain meaningful when kernel pointer
+ * hashing is enabled.
+ */
+TRACE_EVENT(ceph_client_reset_schedule,
+	TP_PROTO(const struct ceph_mds_client *mdsc, const char *reason),
+	TP_ARGS(mdsc, reason),
+	TP_STRUCT__entry(
+		__field(u64, client_id)
+		__string(reason, reason ? reason : "")
+	),
+	TP_fast_assign(
+		__entry->client_id = mdsc->fsc->client->monc.auth ?
+			mdsc->fsc->client->monc.auth->global_id : 0;
+		__assign_str(reason);
+	),
+	TP_printk("client_id=%llu reason=%s",
+		  __entry->client_id, __get_str(reason))
+);
+
+TRACE_EVENT(ceph_client_reset_complete,
+	TP_PROTO(const struct ceph_mds_client *mdsc, int ret),
+	TP_ARGS(mdsc, ret),
+	TP_STRUCT__entry(
+		__field(u64, client_id)
+		__field(int, ret)
+	),
+	TP_fast_assign(
+		__entry->client_id = mdsc->fsc->client->monc.auth ?
+			mdsc->fsc->client->monc.auth->global_id : 0;
+		__entry->ret = ret;
+	),
+	TP_printk("client_id=%llu ret=%d", __entry->client_id, __entry->ret)
+);
+
+TRACE_EVENT(ceph_client_reset_blocked,
+	TP_PROTO(const struct ceph_mds_client *mdsc, int blocked_count),
+	TP_ARGS(mdsc, blocked_count),
+	TP_STRUCT__entry(
+		__field(u64, client_id)
+		__field(int, blocked_count)
+	),
+	TP_fast_assign(
+		__entry->client_id = mdsc->fsc->client->monc.auth ?
+			mdsc->fsc->client->monc.auth->global_id : 0;
+		__entry->blocked_count = blocked_count;
+	),
+	TP_printk("client_id=%llu blocked_count=%d", __entry->client_id,
+		  __entry->blocked_count)
+);
+
+TRACE_EVENT(ceph_client_reset_unblocked,
+	TP_PROTO(const struct ceph_mds_client *mdsc, int ret),
+	TP_ARGS(mdsc, ret),
+	TP_STRUCT__entry(
+		__field(u64, client_id)
+		__field(int, ret)
+	),
+	TP_fast_assign(
+		__entry->client_id = mdsc->fsc->client->monc.auth ?
+			mdsc->fsc->client->monc.auth->global_id : 0;
+		__entry->ret = ret;
+	),
+	TP_printk("client_id=%llu ret=%d", __entry->client_id, __entry->ret)
+);
+
+#undef EM
+#undef E_
+#endif /* _TRACE_CEPH_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>