From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- arch/powerpc/lib/ldstfp.S | 237 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 arch/powerpc/lib/ldstfp.S (limited to 'arch/powerpc/lib/ldstfp.S') diff --git a/arch/powerpc/lib/ldstfp.S b/arch/powerpc/lib/ldstfp.S new file mode 100644 index 000000000..e00abeabc --- /dev/null +++ b/arch/powerpc/lib/ldstfp.S @@ -0,0 +1,237 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Floating-point, VMX/Altivec and VSX loads and stores + * for use in instruction emulation. + * + * Copyright 2010 Paul Mackerras, IBM Corp. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define STKFRM (PPC_MIN_STKFRM + 16) + +/* Get the contents of frN into *p; N is in r3 and p is in r4. */ +_GLOBAL(get_fpr) + mflr r0 + mfmsr r6 + ori r7, r6, MSR_FP + MTMSRD(r7) + isync + rlwinm r3,r3,3,0xf8 + bcl 20,31,1f +reg = 0 + .rept 32 + stfd reg, 0(r4) + b 2f +reg = reg + 1 + .endr +1: mflr r5 + add r5,r3,r5 + mtctr r5 + mtlr r0 + bctr +2: MTMSRD(r6) + isync + blr + +/* Put the contents of *p into frN; N is in r3 and p is in r4. */ +_GLOBAL(put_fpr) + mflr r0 + mfmsr r6 + ori r7, r6, MSR_FP + MTMSRD(r7) + isync + rlwinm r3,r3,3,0xf8 + bcl 20,31,1f +reg = 0 + .rept 32 + lfd reg, 0(r4) + b 2f +reg = reg + 1 + .endr +1: mflr r5 + add r5,r3,r5 + mtctr r5 + mtlr r0 + bctr +2: MTMSRD(r6) + isync + blr + +#ifdef CONFIG_ALTIVEC +/* Get the contents of vrN into *p; N is in r3 and p is in r4. */ +_GLOBAL(get_vr) + mflr r0 + mfmsr r6 + oris r7, r6, MSR_VEC@h + MTMSRD(r7) + isync + rlwinm r3,r3,3,0xf8 + bcl 20,31,1f +reg = 0 + .rept 32 + stvx reg, 0, r4 + b 2f +reg = reg + 1 + .endr +1: mflr r5 + add r5,r3,r5 + mtctr r5 + mtlr r0 + bctr +2: MTMSRD(r6) + isync + blr + +/* Put the contents of *p into vrN; N is in r3 and p is in r4. */ +_GLOBAL(put_vr) + mflr r0 + mfmsr r6 + oris r7, r6, MSR_VEC@h + MTMSRD(r7) + isync + rlwinm r3,r3,3,0xf8 + bcl 20,31,1f +reg = 0 + .rept 32 + lvx reg, 0, r4 + b 2f +reg = reg + 1 + .endr +1: mflr r5 + add r5,r3,r5 + mtctr r5 + mtlr r0 + bctr +2: MTMSRD(r6) + isync + blr +#endif /* CONFIG_ALTIVEC */ + +#ifdef CONFIG_VSX +/* Get the contents of vsN into vs0; N is in r3. */ +_GLOBAL(get_vsr) + mflr r0 + rlwinm r3,r3,3,0x1f8 + bcl 20,31,1f + blr /* vs0 is already in vs0 */ + nop +reg = 1 + .rept 63 + XXLOR(0,reg,reg) + blr +reg = reg + 1 + .endr +1: mflr r5 + add r5,r3,r5 + mtctr r5 + mtlr r0 + bctr + +/* Put the contents of vs0 into vsN; N is in r3. */ +_GLOBAL(put_vsr) + mflr r0 + rlwinm r3,r3,3,0x1f8 + bcl 20,31,1f + blr /* v0 is already in v0 */ + nop +reg = 1 + .rept 63 + XXLOR(reg,0,0) + blr +reg = reg + 1 + .endr +1: mflr r5 + add r5,r3,r5 + mtctr r5 + mtlr r0 + bctr + +/* Load VSX reg N from vector doubleword *p. N is in r3, p in r4. */ +_GLOBAL(load_vsrn) + PPC_STLU r1,-STKFRM(r1) + mflr r0 + PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1) + mfmsr r6 + oris r7,r6,MSR_VSX@h + cmpwi cr7,r3,0 + li r8,STKFRM-16 + MTMSRD(r7) + isync + beq cr7,1f + STXVD2X(0,R1,R8) +1: LXVD2X(0,R0,R4) +#ifdef __LITTLE_ENDIAN__ + XXSWAPD(0,0) +#endif + beq cr7,4f + bl put_vsr + LXVD2X(0,R1,R8) +4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1) + mtlr r0 + MTMSRD(r6) + isync + addi r1,r1,STKFRM + blr + +/* Store VSX reg N to vector doubleword *p. N is in r3, p in r4. */ +_GLOBAL(store_vsrn) + PPC_STLU r1,-STKFRM(r1) + mflr r0 + PPC_STL r0,STKFRM+PPC_LR_STKOFF(r1) + mfmsr r6 + oris r7,r6,MSR_VSX@h + li r8,STKFRM-16 + MTMSRD(r7) + isync + STXVD2X(0,R1,R8) + bl get_vsr +#ifdef __LITTLE_ENDIAN__ + XXSWAPD(0,0) +#endif + STXVD2X(0,R0,R4) + LXVD2X(0,R1,R8) + PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1) + mtlr r0 + MTMSRD(r6) + isync + mr r3,r9 + addi r1,r1,STKFRM + blr +#endif /* CONFIG_VSX */ + +/* Convert single-precision to double, without disturbing FPRs. */ +/* conv_sp_to_dp(float *sp, double *dp) */ +_GLOBAL(conv_sp_to_dp) + mfmsr r6 + ori r7, r6, MSR_FP + MTMSRD(r7) + isync + stfd fr0, -16(r1) + lfs fr0, 0(r3) + stfd fr0, 0(r4) + lfd fr0, -16(r1) + MTMSRD(r6) + isync + blr + +/* Convert single-precision to double, without disturbing FPRs. */ +/* conv_sp_to_dp(double *dp, float *sp) */ +_GLOBAL(conv_dp_to_sp) + mfmsr r6 + ori r7, r6, MSR_FP + MTMSRD(r7) + isync + stfd fr0, -16(r1) + lfd fr0, 0(r3) + stfs fr0, 0(r4) + lfd fr0, -16(r1) + MTMSRD(r6) + isync + blr -- cgit v1.3.1