Merge tag 'sched_ext-for-7.4-rc2-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Pull sched_ext fixes from Tejun Heo:

 - With proxy execution or core scheduling, a CPU could run a task that
   had been requeued on another CPU while dispatch had the rq lock
   dropped.

 - Unregistering a scheduler could return while its teardown was still
   running, letting the teardown call into BPF code that was already
   freed.

 - A CPU coming online could call into the BPF scheduler before the
   scheduler was told that the CPU is online.

 - A parent scheduler could be asked to attach a sub-scheduler to a
   cgroup that was already removed.

 - Hardening: more kfuncs that take a queue lock are rejected in tracing
   programs, where they could deadlock the CPU, and a queue iterator no
   longer restarts after reaching the end, which the BPF verifier
   assumes can't happen.

 - Smaller fixes: tasks needlessly sent back to the BPF scheduler on
   idle CPUs, stale scheduling state carried into a task's next
   scheduler, and uneven CPU shards with partially populated SMT cores.

* tag 'sched_ext-for-7.4-rc2-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext:
  sched_ext: Reject DSQ iterator kfuncs in tracing programs
  sched_ext: make DSQ iterator stay drained after NULL
  sched_ext: Fix partial-SMT core counting in calc_shard_layout()
  sched_ext: Reset task fields on exit of an SCX_TASK_READY task
  sched_ext: Read exit_kind before flushing in scx_flush_disable_work()
  sched_ext: Reject scx_bpf_destroy_dsq() in tracing programs
  sched_ext: Fix ops.enqueue/dispatch() calls before ops.cpu_online()
  sched_ext: Factor out drain_offline_rq()
  sched_ext: Fix stale pick when the donor changes during dispatch
  sched_ext: Call ops.sub_attach() only while the cgroup is online
  sched_ext: Fix scx_bpf_dsq_reenq() kerneldoc claiming NMI calls are allowed
  sched_ext: Avoid unnecessary IMMED reenqueues on idle CPUs
This commit is contained in:
Linus Torvalds
2026-11-15 14:20:40 -08:00
6 changed files with 183 additions and 67 deletions
+7 -1
View File
@@ -64,7 +64,13 @@ static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size,
for_each_cpu(cpu, llc_cpus) {
nr_cpus++;
if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu)
/*
* scx_cid_init() forms each core from the siblings in
* @llc_cpus. The sibling mask may include cpus outside
* @llc_cpus, so count a core at its first sibling in @llc_cpus.
*/
if (cpumask_first_and(topology_sibling_cpumask(cpu),
llc_cpus) == cpu)
nr_cores++;
}
+111 -49
View File
@@ -461,6 +461,14 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
* @rq is either picking, in transition to, or running an SCX task.
*/
/*
* A local insertion can raise next_class to ext while the donor is
* still idle. Keep the rq open so that the local DSQ head is not
* unnecessarily reenqueued before the CPU can pick it.
*/
if (rq->donor == rq->idle)
return true;
/*
* If we're in the dispatch path holding rq lock, $donor may or may not
* be ready depending on whether the on-going dispatch decides to extend
@@ -580,12 +588,14 @@ enum scx_dsq_iter_flags {
/* iterate in the reverse dispatch order */
SCX_DSQ_ITER_REV = 1U << 16,
__SCX_DSQ_ITER_DRAINED = 1U << 28,
__SCX_DSQ_ITER_HAS_REENQ = 1U << 29,
__SCX_DSQ_ITER_HAS_SLICE = 1U << 30,
__SCX_DSQ_ITER_HAS_VTIME = 1U << 31,
__SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV,
__SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS |
__SCX_DSQ_ITER_DRAINED |
__SCX_DSQ_ITER_HAS_REENQ |
__SCX_DSQ_ITER_HAS_SLICE |
__SCX_DSQ_ITER_HAS_VTIME,
@@ -2225,11 +2235,11 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
bool scx_rq_online(struct rq *rq)
{
/*
* Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE follows the
* core's rq online state. cpu_active() test guarantees that, if this
* function returns %true, %SCX_RQ_ONLINE will stay set until the
* current scheduling operation is complete even if we aren't locking
* @rq.
* Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE is set
* after the root's ops.cpu_online() and cleared when the cpu goes down.
* cpu_active() test guarantees that, if this function returns %true,
* %SCX_RQ_ONLINE will stay set until the current scheduling operation
* is complete even if we aren't locking @rq.
*/
return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
}
@@ -2420,7 +2430,7 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p)
}
/*
* list_add_tail() must be used. scx_bypass() and rq_offline_scx()
* list_add_tail() must be used. scx_bypass() and drain_offline_rq()
* depend on tasks being appended to the runnable_list.
*/
list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
@@ -3399,6 +3409,12 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
rq->scx.cpu_released = false;
}
/* the ecaps sync may have dropped the rq lock */
if (scx_dsp_prev_stale(rq, prev)) {
verdict = SCX_DSP_RETRY;
goto has_tasks;
}
if (prev->sched_class == &ext_sched_class) {
update_curr_scx(rq);
@@ -3441,6 +3457,10 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
}
verdict = scx_dispatch_sched(root_sch, rq, prev, false);
/* scx_dispatch_sched() may have dropped the rq lock */
if (scx_dsp_prev_stale(rq, prev))
verdict = SCX_DSP_RETRY;
if (verdict != SCX_DSP_NONE)
goto has_tasks;
@@ -4296,12 +4316,37 @@ static void set_cpus_allowed_scx(struct task_struct *p,
scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
}
static void drain_offline_rq(struct rq *rq)
{
struct task_struct *p, *n;
scx_rescue_flush(rq);
/*
* An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
* onto the local DSQ so that they run here and balance_push() moves
* them off.
*/
list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
if (p->scx.dsq == &rq->scx.local_dsq)
continue;
guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
}
}
static void handle_hotplug(struct rq *rq, bool online)
{
struct scx_sched *sch = scx_root_protected();
s32 cpu = cpu_of(rq);
s32 cpu_or_cid = cpu;
if (!online) {
guard(rq_lock_irqsave)(rq);
update_rq_clock(rq);
rq->scx.flags &= ~SCX_RQ_ONLINE;
drain_offline_rq(rq);
}
atomic_long_inc(&scx_hotplug_seq);
scx_update_frozen_cpus(cpu, online);
@@ -4310,8 +4355,14 @@ static void handle_hotplug(struct rq *rq, bool online)
* stable here. Note that we can't depend on scx_enabled() test as the
* hotplug ops need to be enabled before __scx_enabled is set.
*/
if (unlikely(!sch))
if (unlikely(!sch)) {
/* no root to notify, set %SCX_RQ_ONLINE right away */
if (online) {
guard(rq_lock_irqsave)(rq);
rq->scx.flags |= SCX_RQ_ONLINE;
}
return;
}
if (scx_enabled())
scx_idle_update_selcpu_topology(&sch->ops);
@@ -4362,8 +4413,22 @@ static void handle_hotplug(struct rq *rq, bool online)
"cpu %d going %s, exiting scheduler", cpu,
online ? "online" : "offline");
if (online)
if (online) {
/*
* The cpu must not call ops.enqueue() or ops.dispatch() before
* the root's ops.cpu_online(). The core's rq online state can't
* gate them, as the sched domain rebuild in
* sched_cpu_activate() can put @rq online before this function
* runs. Set %SCX_RQ_ONLINE here.
*
* scx_online_ecaps() queues a reseed whose ecaps ops run in
* dispatch context on the cpu and can insert into its local
* DSQ. Set the flag first.
*/
scoped_guard (rq_lock_irqsave, rq)
rq->scx.flags |= SCX_RQ_ONLINE;
scx_online_ecaps(rq);
}
}
void scx_rq_activate(struct rq *rq)
@@ -4376,35 +4441,6 @@ void scx_rq_deactivate(struct rq *rq)
handle_hotplug(rq, false);
}
static void rq_online_scx(struct rq *rq)
{
rq->scx.flags |= SCX_RQ_ONLINE;
}
static void rq_offline_scx(struct rq *rq)
{
struct task_struct *p, *n;
rq->scx.flags &= ~SCX_RQ_ONLINE;
/* sched domain rebuilds call rq_offline with the CPU staying alive */
if (cpu_active(cpu_of(rq)))
return;
scx_rescue_flush(rq);
/*
* An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
* onto the local DSQ so that they run here and balance_push() moves
* them off.
*/
list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
if (p->scx.dsq == &rq->scx.local_dsq)
continue;
guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
}
}
static bool check_rq_for_timeouts(struct rq *rq)
{
struct scx_sched *sch;
@@ -4728,6 +4764,7 @@ void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)
args.cancelled = true;
break;
case SCX_TASK_READY:
scx_reset_task_fields(task_rq(p), p);
break;
case SCX_TASK_ENABLED:
scx_disable_task(sch, p);
@@ -5876,9 +5913,6 @@ DEFINE_SCHED_CLASS(ext) = {
.task_woken = task_woken_scx,
.set_cpus_allowed = set_cpus_allowed_scx,
.rq_online = rq_online_scx,
.rq_offline = rq_offline_scx,
.task_tick = task_tick_scx,
.switching_to = switching_to_scx,
@@ -7779,10 +7813,17 @@ void scx_flush_disable_work(struct scx_sched *sch)
{
int kind;
/*
* scx_disable_workfn() sets SCX_EXIT_DONE before tearing down. Read
* exit_kind with acquire ordering (paired with atomic_try_cmpxchg() in
* scx_disable_workfn()) before syncing and flushing so that, once DONE
* is seen, the following flush observes the queued work and waits for
* the teardown to finish.
*/
do {
kind = atomic_read_acquire(&sch->exit_kind);
irq_work_sync(&sch->disable_irq_work);
kthread_flush_work(&sch->disable_work);
kind = atomic_read(&sch->exit_kind);
} while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
}
@@ -11028,13 +11069,24 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
{
struct bpf_iter_scx_dsq_kern *kit = (void *)it;
struct task_struct *p;
if (!kit->dsq)
if (!kit->dsq || (kit->cursor.flags & __SCX_DSQ_ITER_DRAINED))
return NULL;
guard(raw_spinlock_irqsave)(&kit->dsq->lock);
return nldsq_cursor_next_task(&kit->cursor, kit->dsq);
p = nldsq_cursor_next_task(&kit->cursor, kit->dsq);
/*
* nldsq_cursor_next_task() unlinks the cursor and starts from the DSQ
* head when it is called again. The verifier relies on an iterator
* which returned NULL to keep returning NULL (see
* process_iter_next_call()), so remember the drained state.
*/
if (!p)
kit->cursor.flags |= __SCX_DSQ_ITER_DRAINED;
return p;
}
/**
@@ -11107,7 +11159,7 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
/**
* scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ
* @dsq_id: DSQ to re-enqueue
* @reenq_flags: %SCX_RENQ_*
* @reenq_flags: %SCX_REENQ_*
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Iterate over all of the tasks currently enqueued on the DSQ identified by
@@ -11117,8 +11169,9 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
* - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu)
* - User DSQs
*
* Re-enqueues are performed asynchronously. Can be called from anywhere. A
* re-enqueue of an inactive cpu's local DSQ is ignored.
* Re-enqueues are performed asynchronously. Can be called from any context
* except NMI. A call from NMI aborts the scheduler. A re-enqueue of an inactive
* cpu's local DSQ is ignored.
*
* %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
* operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
@@ -11215,9 +11268,10 @@ __bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter,
* scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
*
* Iterate over all of the tasks currently enqueued on the local DSQ of the
* caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from
* anywhere.
* Iterate over all of the tasks currently enqueued on the local DSQ of the rq
* the current scheduler operation is locked to, or the calling CPU's when no rq
* is locked, and re-enqueue them in the BPF scheduler. Can be called from any
* context except NMI. A call from NMI aborts the scheduler.
*
* This is now a special case of scx_bpf_dsq_reenq() and may be removed in the
* future.
@@ -12134,6 +12188,11 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = {
* when scx_locked_rq() returns NULL. Taking it while the context already holds
* an rq lock, the same one or another cpu's, can deadlock.
*
* scx_bpf_destroy_dsq() takes a user DSQ lock. So do
* bpf_iter_scx_dsq_next() and bpf_iter_scx_dsq_destroy(), the latter only
* after a successful next() has linked the cursor. A tracing program can
* run while that lock is already held and recursively acquire it.
*
* Before returning, scx_bpf_sub_grant() and scx_bpf_sub_revoke() invoke
* ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops
* run in the tracing program's context. No SCX op expects to run from an
@@ -12148,6 +12207,9 @@ BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
BTF_KFUNCS_END(scx_kfunc_ids_no_tracing)
/*
+34 -5
View File
@@ -18,9 +18,32 @@ enum scx_dsp_verdict {
SCX_DSP_NONE, /* nothing to run */
SCX_DSP_LOCAL, /* local DSQ has tasks */
SCX_DSP_PREV, /* keep running @prev */
SCX_DSP_RETRY, /* pick helpers only: restart the pick */
SCX_DSP_RETRY, /* restart the pick */
};
/**
* scx_dsp_prev_stale - Test whether dispatch's @prev went stale
* @rq: rq being dispatched for
* @prev: @rq's donor when dispatch started
*
* @prev can stop being @rq's donor while dispatch has the rq lock dropped. With
* proxy execution, @prev may not be the running task, and another CPU can reset
* the donor to rq->curr. With core scheduling, the core pick runs dispatch for
* the sibling rqs too, and a sibling can switch @prev out. Dispatch restarts
* the pick when that happened and doesn't hand the old donor to the BPF
* scheduler.
*
* With proxy execution alone, the donor only ever changes to rq->curr during a
* pick. Only __schedule() on this CPU sets a different donor, after the pick,
* so the restarted pick takes rq->curr as @prev and can't go stale again. Under
* core scheduling, the restart goes through the core pick, which already
* retries whenever dispatch dropped the lock.
*/
static inline bool scx_dsp_prev_stale(struct rq *rq, struct task_struct *prev)
{
return (sched_proxy_exec() || sched_core_enabled(rq)) && unlikely(prev != rq->donor);
}
/*
* One user of this function is scx_bpf_sub_dispatch() which can be called
* recursively as sub-sched dispatches nest. Always inline to reduce stack usage
@@ -33,8 +56,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
int nr_loops = SCX_DSP_MAX_LOOPS;
s32 cpu = cpu_of(rq);
bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
scx_task_on_sched(sch, prev);
bool prev_on_sch;
if (scx_consume_global_dsq(sch, rq))
return SCX_DSP_LOCAL;
@@ -82,7 +104,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
/*
* scx_rq_online() can't be used. Its cpu_active() test goes false
* before CPU hotplug waits for an RCU grace period, and
* rq_offline_scx() moves this CPU's tasks to the local DSQ only after
* handle_hotplug() moves this CPU's tasks to the local DSQ only after
* the wait. The grace period can depend on those tasks running.
*/
if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE))
@@ -106,6 +128,11 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
rq->scx.sub_dispatch_prev = prev;
#endif
/* the dispatch so far may have dropped the rq lock */
prev_on_sch = !scx_dsp_prev_stale(rq, prev) &&
prev->sched_class == &ext_sched_class &&
scx_task_on_sched(sch, prev);
SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
prev_on_sch ? prev : NULL);
@@ -116,7 +143,9 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
scx_flush_dispatch_buf(sch, rq);
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
/* ops.dispatch() and the flush may have dropped the rq lock */
if (!scx_dsp_prev_stale(rq, prev) &&
(prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
return SCX_DSP_PREV;
if (rq->scx.local_dsq.nr)
return SCX_DSP_LOCAL;
+4
View File
@@ -869,6 +869,10 @@ struct sched_ext_ops {
* @sub_attach: Attach a sub-scheduler
* @args: argument container, see the struct definition
*
* The sub-scheduler's cgroup is online for the whole call. If it goes
* offline after the sub-scheduler is accepted, ops.sub_detach()
* follows.
*
* Return 0 to accept the sub-scheduler. -errno to reject.
*/
s32 (*sub_attach)(struct scx_sub_attach_args *args);
+18 -4
View File
@@ -918,10 +918,11 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq
return &rq->scx.local_dsq;
/*
* The task must run on this CPU regardless of caps: the rq is draining
* offline (BPF scheduler bypassed), the task is migration-disabled, or a
* migration is pending. Admit despite the missing caps and count it.
* Refuse preemptions.
* The task must run on this CPU regardless of caps: ops.enqueue() is
* skipped while the CPU is going down or before the root's
* ops.cpu_online(), the task is migration-disabled, or a migration is
* pending. Admit despite the missing caps and count it. Refuse
* preemptions.
*/
if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
p->migration_pending)) {
@@ -2340,8 +2341,21 @@ void scx_sub_enable_workfn(struct kthread_work *work)
.cgroup_path = sch->cgrp_path,
};
/*
* The cgroup stays online across ops.sub_attach(), so the parent never
* sees an attach for a cgroup that rmdir already removed.
*/
cgroup_lock();
if (!(cgrp->self.flags & CSS_ONLINE)) {
cgroup_unlock();
scx_error(sch, "cgroup is not online");
ret = -ENODEV;
goto err_disable;
}
ret = SCX_CALL_OP_RET(parent, sub_attach, NULL,
&sub_attach_args);
cgroup_unlock();
if (ret) {
ret = scx_ops_sanitize_err(sch, "sub_attach", ret);
scx_error(sch, "parent rejected (%d)", ret);
+9 -8
View File
@@ -781,19 +781,20 @@ struct cfs_rq {
/* scx_rq->flags, protected by the rq lock */
enum scx_rq_flags {
/*
* Set while the core has the rq online, see rq_online_scx() and
* rq_offline_scx(). A hotplugged cpu starts scheduling before
* rq_online_scx(). ops.enqueue() and ops.dispatch() are called for the
* cpu only while it is set, see scx_rq_online() and
* scx_dispatch_sched().
* Set right after the root's ops.cpu_online() when a cpu comes up and
* cleared when the cpu goes down, see handle_hotplug(). A hotplugged
* cpu starts scheduling before %SCX_RQ_ONLINE is set. ops.enqueue() and
* ops.dispatch() are called for the cpu only while the flag is set, see
* scx_rq_online() and scx_dispatch_sched().
*/
SCX_RQ_ONLINE = 1 << 0,
SCX_RQ_CAN_STOP_TICK = 1 << 1,
/*
* Set while the root considers the cpu online, from its
* ops.cpu_online() until its ops.cpu_offline(). ecaps are delivered
* only while it is set. %SCX_RQ_ONLINE follows the core's rq state
* instead, which changes ahead of the root's view on both edges.
* only while it is set. %SCX_RQ_ONLINE can't gate them, as a cpu going
* down clears it before the ecaps loss is reported, see
* scx_offline_ecaps().
*/
SCX_RQ_OPS_ONLINE = 1 << 2,
SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */
@@ -4308,7 +4309,7 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
* Typically this must be called while holding task_rq_lock, since most/all
* properties are serialized under those locks. There are currently two
* exceptions to this rule in sched/ext which only hold rq->lock: scx_bypass()
* and rq_offline_scx().
* and drain_offline_rq().
*/
/*