From 9cc63f8bcd560c760d0b12e15bba8f81c86237cf Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 14 Aug 2026 15:43:51 -0700 Subject: Merge branches 'expcb.2026.07.24a', 'misc.2026.07.30a', 'rcu-tasks.2026.07.30a', 'srcu.2026.08.11a' and 'torture.2026.08.14a' into HEAD Changes: Make expedited grace periods expedite normal RCU callbacks Miscellaneous fixes: * Improve diagnostic output with character task states. * Mark accesses to inform KCSAN of concurrency design. * Move from kmalloc() to kmalloc_obj(). * Documentation updates. * Improve handling of RCU deferred quiescent states. * Clean up unused function arguments and structure fields. * Reduce show_rcu_gp_kthreads() stack space. Tasks RCU updates: * Clean up after SRCU re-implementation of Tasks Trace RCU. * Mark accesses to inform KCSAN of concurrency design. * Add ->lazy_timer status to diagnostic output. * Remove an unnecessary memory barrier. * Fix a data race, courtesy of KCSAN. * Documentation updates. * Convert cond_resched_tasks_rcu_qs() from macro to static inline function. SRCU updates: * Add Rust helpers for SRCU. * Avoid losing queued work at cleanup_srcu_struct() time. Torture-test updates: * Preparation work for immediate RCU priority deboosting. * Test RCU readers from real interrupt handlers (as opposed to softirq). * Simplify code through use of cpumask_next_wrap(). * Improve diagnostic output with character task states. * Add rcutorture.nwriters parameter to allow lightweight stall testing, and rcutorture.stall_only to make doing so easier. * Test an RCU Tasks Trace grace period implying an RCU grace period. * Make RCU Tasks Trace torturing track reader batches. * Fix a data race, courtesy of KCSAN. * Plug a shuffle_tmp_mask memory leak on kthread spawn failure. --- lib/cpumask.c | 168 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 lib/cpumask.c (limited to 'lib/cpumask.c') diff --git a/lib/cpumask.c b/lib/cpumask.c new file mode 100644 index 000000000..5adb9874f --- /dev/null +++ b/lib/cpumask.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +/* These are not inline because of header tangles. */ +#ifdef CONFIG_CPUMASK_OFFSTACK +/** + * alloc_cpumask_var_node - allocate a struct cpumask on a given node + * @mask: pointer to cpumask_var_t where the cpumask is returned + * @flags: GFP_ flags + * @node: memory node from which to allocate or %NUMA_NO_NODE + * + * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is + * a nop returning a constant 1 (in ). + * + * Return: TRUE if memory allocation succeeded, FALSE otherwise. + * + * In addition, mask will be NULL if this fails. Note that gcc is + * usually smart enough to know that mask can never be NULL if + * CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case + * too. + */ +bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) +{ + *mask = kmalloc_node(cpumask_size(), flags, node); + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS + if (!*mask) { + printk(KERN_ERR "=> alloc_cpumask_var: failed!\n"); + dump_stack(); + } +#endif + + return *mask != NULL; +} +EXPORT_SYMBOL(alloc_cpumask_var_node); + +/** + * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena. + * @mask: pointer to cpumask_var_t where the cpumask is returned + * + * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is + * a nop (in ). + * Either returns an allocated (zero-filled) cpumask, or causes the + * system to panic. + */ +void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask) +{ + *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES); +} + +/** + * free_cpumask_var - frees memory allocated for a struct cpumask. + * @mask: cpumask to free + * + * This is safe on a NULL mask. + */ +void free_cpumask_var(cpumask_var_t mask) +{ + kfree(mask); +} +EXPORT_SYMBOL(free_cpumask_var); + +/** + * free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var + * @mask: cpumask to free + */ +void __init free_bootmem_cpumask_var(cpumask_var_t mask) +{ + memblock_free(mask, cpumask_size()); +} +#endif + +/** + * cpumask_local_spread - select the i'th cpu based on NUMA distances + * @i: index number + * @node: local numa_node + * + * Return: online CPU according to a numa aware policy; local cpus are returned + * first, followed by non-local ones, then it wraps around. + * + * For those who wants to enumerate all CPUs based on their NUMA distances, + * i.e. call this function in a loop, like: + * + * for (i = 0; i < num_online_cpus(); i++) { + * cpu = cpumask_local_spread(i, node); + * do_something(cpu); + * } + * + * There's a better alternative based on for_each()-like iterators: + * + * for_each_numa_hop_mask(mask, node) { + * for_each_cpu_andnot(cpu, mask, prev) + * do_something(cpu); + * prev = mask; + * } + * + * It's simpler and more verbose than above. Complexity of iterator-based + * enumeration is O(sched_domains_numa_levels * nr_cpu_ids), while + * cpumask_local_spread() when called for each cpu is + * O(sched_domains_numa_levels * nr_cpu_ids * log(nr_cpu_ids)). + */ +unsigned int cpumask_local_spread(unsigned int i, int node) +{ + unsigned int cpu; + + /* Wrap: we always want a cpu. */ + i %= num_online_cpus(); + + cpu = sched_numa_find_nth_cpu(cpu_online_mask, i, node); + + WARN_ON(cpu >= nr_cpu_ids); + return cpu; +} +EXPORT_SYMBOL(cpumask_local_spread); + +static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); + +/** + * cpumask_any_and_distribute - Return an arbitrary cpu within src1p & src2p. + * @src1p: first &cpumask for intersection + * @src2p: second &cpumask for intersection + * + * Iterated calls using the same srcp1 and srcp2 will be distributed within + * their intersection. + * + * Return: >= nr_cpu_ids if the intersection is empty. + */ +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, + const struct cpumask *src2p) +{ + unsigned int next, prev; + + /* NOTE: our first selection will skip 0. */ + prev = __this_cpu_read(distribute_cpu_mask_prev); + + next = cpumask_next_and_wrap(prev, src1p, src2p); + if (next < nr_cpu_ids) + __this_cpu_write(distribute_cpu_mask_prev, next); + + return next; +} +EXPORT_SYMBOL(cpumask_any_and_distribute); + +/** + * cpumask_any_distribute - Return an arbitrary cpu from srcp + * @srcp: &cpumask for selection + * + * Return: >= nr_cpu_ids if the intersection is empty. + */ +unsigned int cpumask_any_distribute(const struct cpumask *srcp) +{ + unsigned int next, prev; + + /* NOTE: our first selection will skip 0. */ + prev = __this_cpu_read(distribute_cpu_mask_prev); + next = cpumask_next_wrap(prev, srcp); + if (next < nr_cpu_ids) + __this_cpu_write(distribute_cpu_mask_prev, next); + + return next; +} +EXPORT_SYMBOL(cpumask_any_distribute); -- cgit v1.3.1