From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- arch/alpha/lib/memcpy.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 arch/alpha/lib/memcpy.c (limited to 'arch/alpha/lib/memcpy.c') diff --git a/arch/alpha/lib/memcpy.c b/arch/alpha/lib/memcpy.c new file mode 100644 index 000000000..78b6850a9 --- /dev/null +++ b/arch/alpha/lib/memcpy.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/alpha/lib/memcpy.c + * + * Copyright (C) 1995 Linus Torvalds + */ + +/* + * This is a reasonably optimized memcpy() routine. + */ + +/* + * Note that the C code is written to be optimized into good assembly. However, + * at this point gcc is unable to sanely compile "if (n >= 0)", resulting in a + * explicit compare against 0 (instead of just using the proper "blt reg, xx" or + * "bge reg, xx"). I hope alpha-gcc will be fixed to notice this eventually.. + */ + +#include +#include +#include + +/* + * This should be done in one go with ldq_u*2/mask/stq_u. Do it + * with a macro so that we can fix it up later.. + */ +#define ALIGN_DEST_TO8_UP(d,s,n) \ + while (d & 7) { \ + if (n <= 0) return; \ + n--; \ + *(char *) d = *(char *) s; \ + d++; s++; \ + } +#define ALIGN_DEST_TO8_DN(d,s,n) \ + while (d & 7) { \ + if (n <= 0) return; \ + n--; \ + d--; s--; \ + *(char *) d = *(char *) s; \ + } + +/* + * This should similarly be done with ldq_u*2/mask/stq. The destination + * is aligned, but we don't fill in a full quad-word + */ +#define DO_REST_UP(d,s,n) \ + while (n > 0) { \ + n--; \ + *(char *) d = *(char *) s; \ + d++; s++; \ + } +#define DO_REST_DN(d,s,n) \ + while (n > 0) { \ + n--; \ + d--; s--; \ + *(char *) d = *(char *) s; \ + } + +/* + * This should be done with ldq/mask/stq. The source and destination are + * aligned, but we don't fill in a full quad-word + */ +#define DO_REST_ALIGNED_UP(d,s,n) DO_REST_UP(d,s,n) +#define DO_REST_ALIGNED_DN(d,s,n) DO_REST_DN(d,s,n) + +/* + * This does unaligned memory copies. We want to avoid storing to + * an unaligned address, as that would do a read-modify-write cycle. + * We also want to avoid double-reading the unaligned reads. + * + * Note the ordering to try to avoid load (and address generation) latencies. + */ +static inline void __memcpy_unaligned_up (unsigned long d, unsigned long s, + long n) +{ + ALIGN_DEST_TO8_UP(d,s,n); + n -= 8; /* to avoid compare against 8 in the loop */ + if (n >= 0) { + unsigned long low_word, high_word; + __asm__("ldq_u %0,%1":"=r" (low_word):"m" (*(unsigned long *) s)); + do { + unsigned long tmp; + __asm__("ldq_u %0,%1":"=r" (high_word):"m" (*(unsigned long *)(s+8))); + n -= 8; + __asm__("extql %1,%2,%0" + :"=r" (low_word) + :"r" (low_word), "r" (s)); + __asm__("extqh %1,%2,%0" + :"=r" (tmp) + :"r" (high_word), "r" (s)); + s += 8; + *(unsigned long *) d = low_word | tmp; + d += 8; + low_word = high_word; + } while (n >= 0); + } + n += 8; + DO_REST_UP(d,s,n); +} + +static inline void __memcpy_unaligned_dn (unsigned long d, unsigned long s, + long n) +{ + /* I don't understand AXP assembler well enough for this. -Tim */ + s += n; + d += n; + while (n--) + * (char *) --d = * (char *) --s; +} + +/* + * Hmm.. Strange. The __asm__ here is there to make gcc use an integer register + * for the load-store. I don't know why, but it would seem that using a floating + * point register for the move seems to slow things down (very small difference, + * though). + * + * Note the ordering to try to avoid load (and address generation) latencies. + */ +static inline void __memcpy_aligned_up (unsigned long d, unsigned long s, + long n) +{ + ALIGN_DEST_TO8_UP(d,s,n); + n -= 8; + while (n >= 0) { + unsigned long tmp; + __asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s)); + n -= 8; + s += 8; + *(unsigned long *) d = tmp; + d += 8; + } + n += 8; + DO_REST_ALIGNED_UP(d,s,n); +} +static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s, + long n) +{ + s += n; + d += n; + ALIGN_DEST_TO8_DN(d,s,n); + n -= 8; + while (n >= 0) { + unsigned long tmp; + s -= 8; + __asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s)); + n -= 8; + d -= 8; + *(unsigned long *) d = tmp; + } + n += 8; + DO_REST_ALIGNED_DN(d,s,n); +} + +#undef memcpy + +void * memcpy(void * dest, const void *src, size_t n) +{ + if (!(((unsigned long) dest ^ (unsigned long) src) & 7)) { + __memcpy_aligned_up ((unsigned long) dest, (unsigned long) src, + n); + return dest; + } + __memcpy_unaligned_up ((unsigned long) dest, (unsigned long) src, n); + return dest; +} +EXPORT_SYMBOL(memcpy); -- cgit v1.3.1