From a155ac8f0c523bd53f412196dcbb104ad1f4595f Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Sat, 29 Aug 2026 14:34:12 -0700 Subject: interrupt: Disable interrupt before modifying hardirq_disable counter Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between hardirq_disable_enter() and _local_interrupt_disable() in local_interrupt_disable(): local_interrupt_disable(): hardirq_disable_enter(); ... __irq_exit_rcu(): // false because hardirq_disable_count() is not 0 if (.. && !hardirq_disable_count() && ..) { invoke_softirq(); } _local_interrupt_disable(); , it'll defer the softirq to the next interrupt which can be forever. The order between hardirq_disable_enter() and _local_interrupt_disable() is to optimize re-disabling interrupts if they are already disabled, but as 1) local_interrupt_disable() is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue. Because of this fix, the part of saving the current state is separated from irq disabling, and the logic of local_interrupt_disable() becomes: local_irq_save(flags); if (counter++ == 0) { this_cpu(local_interrupt_disable_state) = flags; } Therefore change the helper function _local_interrupt_disable() to _local_interrupt_save_state() which only saves the current irqflags (when interrupts get disabled the first time). Fixes: e901c1510e24 ("irq,spin_lock: Add counted interrupt disabling/enabling") Reported-by: Thomas Gleixner Signed-off-by: Boqun Feng Signed-off-by: Thomas Gleixner Reviewed-by: Bradley Morgan Link: https://patch.msgid.link/20260829213412.14303-1-boqun@kernel.org Link: https://lore.kernel.org/lkml/87v78wezid.ffs@fw13/ [1] Closes: https://lore.kernel.org/lkml/87jypbfu1t.ffs@fw13/ --- include/linux/interrupt_rc.h | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h index b9a7f05ec..e68e1bedb 100644 --- a/include/linux/interrupt_rc.h +++ b/include/linux/interrupt_rc.h @@ -20,11 +20,8 @@ /* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); -static __always_inline void __local_interrupt_disable(void) +static __always_inline void __local_interrupt_save_state(unsigned long flags) { - unsigned long flags; - - local_irq_save(flags); raw_cpu_write(local_interrupt_disable_state, flags); } @@ -36,9 +33,9 @@ static __always_inline void __local_interrupt_enable(void) } #ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE -static __always_inline void _local_interrupt_disable(void) +static __always_inline void _local_interrupt_save_state(unsigned long flags) { - __local_interrupt_disable(); + __local_interrupt_save_state(flags); } static __always_inline void _local_interrupt_enable(void) @@ -46,27 +43,27 @@ static __always_inline void _local_interrupt_enable(void) __local_interrupt_enable(); } #else -extern void _local_interrupt_disable(void); +extern void _local_interrupt_save_state(unsigned long flags); extern void _local_interrupt_enable(void); #endif #else /* !MODULE */ -extern void _local_interrupt_disable(void); +extern void _local_interrupt_save_state(unsigned long flags); extern void _local_interrupt_enable(void); #endif /* !MODULE */ static inline void local_interrupt_disable(void) { int new_count; + unsigned long flags; WARN_ON_ONCE(in_nmi()); + local_irq_save(flags); new_count = hardirq_disable_enter(); - /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ - if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) - _local_interrupt_disable(); + _local_interrupt_save_state(flags); } static inline void local_interrupt_enable(void) -- cgit v1.3.1 From 2af470916a208b576ac9975d221d9a378cf8ace9 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Thu, 27 Aug 2026 12:48:35 -0700 Subject: preempt: Remove hardirq_disable_count() It turns out the previous usage of hardirq_disable_count() in __irq_exit_rcu() would cause softirq pending issues. Without that usage, hardirq_disable_count() doesn't need to exist, so remove it. Also move hardirq_disable_enter/exit() into the Rust specific interrupt_rc header. [ tglx: Move the helpers over ] Signed-off-by: Boqun Feng Signed-off-by: Thomas Gleixner Link: https://patch.msgid.link/20260827194835.38968-1-boqun@kernel.org --- include/linux/interrupt_rc.h | 3 +++ include/linux/preempt.h | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h index e68e1bedb..a9ed937a8 100644 --- a/include/linux/interrupt_rc.h +++ b/include/linux/interrupt_rc.h @@ -52,6 +52,9 @@ extern void _local_interrupt_save_state(unsigned long flags); extern void _local_interrupt_enable(void); #endif /* !MODULE */ +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) + static inline void local_interrupt_disable(void) { int new_count; diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 8299657f0..2e689de7b 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -168,10 +168,6 @@ static __always_inline unsigned char interrupt_context_level(void) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) -#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT) -#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) -#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) - /* * The preempt_count offset after preempt_disable(); */ -- cgit v1.3.1 From 912edebe8501a36c6bedcef03bd238ab90a7e060 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 1 Sep 2026 15:54:51 +0200 Subject: futex: Provide rt_mutex_.*_schedule() equivalents for futex scheduling There is rt_mutex_{pre|post}_schedule() around rt_mutex_wait_proxy_lock() to ensure that sched_submit_work()/ sched_update_worker() is invoked before we schedule out and block on rt_mutex while waiting for it become available. The reason is that blocking on rt_mutex assigns a pi_waiter for the PI chain and sched_submit_work() will also assign a pi_waiter if it blocks on lock but a this point we already have a waiter assigned. We can't skip sched_submit_work() entirely because I/O relies on the fact that I/O queue is flushed while it blocks on a sleeping lock. Therefore sched_submit_work() is moved before we block on the lock. Sleeping lock in this context means mutex or rw_semaphore not spinlock_t on PREEMPT_RT. Because the mutex abstraction on PREEMPT_RT uses the same abstraction as the futex proxy lock, the futex code ended up using rt_mutex_{pre|post}_schedule(), too. Using it is/ was just to keep the task_struct::sched_rt_mutex assertion happy. Futex proxy lock is used only in the syscall context of a task. At this point it never got any I/O that needs to be flushed and it can't be a workqueue that needs to notify that it will be scheduled out. Therefore sched_submit_work() does nothing here. By mistake futex_wait_requeue_pi() -> rt_mutex_wait_proxy_lock() did not get the rt_mutex_{pre|post}_schedule() annotation. This was not noticed because in this callchain the lock is (usually) not contended and so rt_mutex_slowlock_block() does not schedule, triggering the assert. Adding rt_mutex_pre_schedule() here looks wrong (as noted by PeterZ) because at this point there is a pi_waiter recorded and invoking sched_submit_work() with a possible lock contention would be wrong. Add rt_mutex_futex_{pre|post}_schedule() which toggles the sched_rt_mutex assert and does not involve sched_submit_work(). Add asserts here to ensure that sched_submit_work() would do nothing. Use it only in futex proxy lock case which is rt_mutex_wait_proxy_lock(). Remove it from futex_lock_pi(). Fixes: d14f9e930b90 ("locking/rtmutex: Use rt_mutex specific scheduler helpers") Reported-by: Yao Kai Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260901135453.3121948-2-bigeasy@linutronix.de Closes: https://lore.kernel.org/all/20260717084922.4153317-2-yaokai34@huawei.com --- include/linux/sched/rt.h | 2 ++ kernel/futex/pi.c | 16 +++------------- kernel/locking/rtmutex_api.c | 2 ++ kernel/sched/core.c | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 13 deletions(-) (limited to 'include') diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index 4e3338103..922935cc3 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -52,8 +52,10 @@ static inline bool rt_or_dl_task_policy(struct task_struct *tsk) #ifdef CONFIG_RT_MUTEXES extern void rt_mutex_pre_schedule(void); +extern void rt_mutex_futex_pre_schedule(void); extern void rt_mutex_schedule(void); extern void rt_mutex_post_schedule(void); +extern void rt_mutex_futex_post_schedule(void); /* * Must hold either p->pi_lock or task_rq(p)->lock. diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 88788e584..98f1b962e 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -1070,17 +1070,11 @@ retry_private: * Caution; releasing @hb in-scope. The hb->lock is still locked * while the reference is dropped. The reference can not be dropped * after the unlock because if a user initiated resize is in progress - * then we might need to wake him. This can not be done after the - * rt_mutex_pre_schedule() invocation. The hb will remain valid because - * the thread, performing resize, will block on hb->lock during - * the requeue. + * then we might need to wake him. The hb will remain valid + * because the thread, performing resize, will block on + * hb->lock during the requeue. */ futex_private_hash_put(no_free_ptr(hbr.fph)); - /* - * Must be done before we enqueue the waiter, here is unfortunately - * under the hb lock, but that *should* work because it does nothing. - */ - rt_mutex_pre_schedule(); rt_mutex_init_waiter(&rt_waiter); @@ -1146,10 +1140,6 @@ cleanup: * the */ futex_q_lockptr_lock(&q); - /* - * Waiter is unqueued. - */ - rt_mutex_post_schedule(); no_block: /* * Fixup the pi_state owner and possibly acquire the lock if we diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 5d48d6472..eb18b0944 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -423,6 +423,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, { int ret; + rt_mutex_futex_pre_schedule(); raw_spin_lock_irq(&lock->wait_lock); /* sleep on the mutex */ set_current_state(TASK_INTERRUPTIBLE); @@ -433,6 +434,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, */ fixup_rt_mutex_waiters(lock, true); raw_spin_unlock_irq(&lock->wait_lock); + rt_mutex_futex_post_schedule(); return ret; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f78275192..449ccd871 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7637,6 +7637,17 @@ void rt_mutex_pre_schedule(void) sched_submit_work(current); } +/* + * Used within the futex syscall context, skips sched_submit_work() because none + * its work will be done. Asserts ensure that it is indeed the case. + */ +void rt_mutex_futex_pre_schedule(void) +{ + lockdep_assert(!(current->flags & (PF_WQ_WORKER | PF_IO_WORKER))); + lockdep_assert(!current->plug); + lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); +} + void rt_mutex_schedule(void) { lockdep_assert(current->sched_rt_mutex); @@ -7649,6 +7660,11 @@ void rt_mutex_post_schedule(void) lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); } +void rt_mutex_futex_post_schedule(void) +{ + lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); +} + /* * rt_mutex_setprio - set the current priority of a task * @p: task to boost -- cgit v1.3.1