diff options
| author | Tejun Heo <tj@kernel.org> | 2026-09-15 12:34:44 -1000 |
|---|---|---|
| committer | Tejun Heo <tj@kernel.org> | 2026-09-15 12:34:44 -1000 |
| commit | 7de9a6fb44eae4f05e68c805d58b9c618815adfa (patch) | |
| tree | b97f1f711082e3cbd8302005449d7fdc619796e1 | |
| parent | a9e3760b0838299649c0d57cca44daaf40ba3c33 (diff) | |
| download | linux-stable-7de9a6fb44eae4f05e68c805d58b9c618815adfa.tar.gz linux-stable-7de9a6fb44eae4f05e68c805d58b9c618815adfa.zip | |
sched_ext: Wait for SCX_OPSS_DISPATCHING before reenqueueing a task
ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics") moved the final
ops_state store in scx_dispatch_enqueue() after the DSQ unlock so that the
custody update and ops.dequeue() precede it. A task can thus be found on a
DSQ while still SCX_OPSS_DISPATCHING.
The dequeue and core-sched pick paths wait for the state to clear in
ops_dequeue() but the reenqueue paths don't. A reenqueue in that window runs
ops.enqueue() and sets SCX_OPSS_QUEUED before the dispatch has completed.
The dispatcher's final store then overwrites it with SCX_OPSS_NONE and
finish_dispatch() drops every later dispatch of the task.
Wait for SCX_OPSS_DISPATCHING to clear before dequeueing a task for
reenqueue, the same way ops_dequeue() does.
Fixes: ebf1ccff79c4 ("sched_ext: Fix ops.dequeue() semantics")
Cc: stable@vger.kernel.org # v7.1+
Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
| -rw-r--r-- | kernel/sched/ext/ext.c | 13 | ||||
| -rw-r--r-- | kernel/sched/ext/internal.h | 1 | ||||
| -rw-r--r-- | kernel/sched/ext/sub.c | 1 |
3 files changed, 15 insertions, 0 deletions
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 70b711c4d..725b890fc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4404,6 +4404,17 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, return *reenq_flags & SCX_REENQ_ANY; } +/* + * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p + * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p + * before that store lands would have it clobber the new %SCX_OPSS_QUEUED. + */ +void scx_reenq_wait_dispatching(struct task_struct *p) +{ + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); +} + static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) { LIST_HEAD(tasks); @@ -4447,6 +4458,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) @@ -4570,6 +4582,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ + scx_reenq_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 3464e0f11..115c96fbf 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2078,6 +2078,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); +void scx_reenq_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index f7aeb1488..a30c965b1 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -801,6 +801,7 @@ void scx_reenq_reject(struct rq *rq) if (WARN_ON_ONCE(p->migration_pending)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) |