mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'sched_ext-for-7.4-rc2-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext
Pull sched_ext fixes from Tejun Heo: - With proxy execution or core scheduling, a CPU could run a task that had been requeued on another CPU while dispatch had the rq lock dropped. - Unregistering a scheduler could return while its teardown was still running, letting the teardown call into BPF code that was already freed. - A CPU coming online could call into the BPF scheduler before the scheduler was told that the CPU is online. - A parent scheduler could be asked to attach a sub-scheduler to a cgroup that was already removed. - Hardening: more kfuncs that take a queue lock are rejected in tracing programs, where they could deadlock the CPU, and a queue iterator no longer restarts after reaching the end, which the BPF verifier assumes can't happen. - Smaller fixes: tasks needlessly sent back to the BPF scheduler on idle CPUs, stale scheduling state carried into a task's next scheduler, and uneven CPU shards with partially populated SMT cores. * tag 'sched_ext-for-7.4-rc2-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: sched_ext: Reject DSQ iterator kfuncs in tracing programs sched_ext: make DSQ iterator stay drained after NULL sched_ext: Fix partial-SMT core counting in calc_shard_layout() sched_ext: Reset task fields on exit of an SCX_TASK_READY task sched_ext: Read exit_kind before flushing in scx_flush_disable_work() sched_ext: Reject scx_bpf_destroy_dsq() in tracing programs sched_ext: Fix ops.enqueue/dispatch() calls before ops.cpu_online() sched_ext: Factor out drain_offline_rq() sched_ext: Fix stale pick when the donor changes during dispatch sched_ext: Call ops.sub_attach() only while the cgroup is online sched_ext: Fix scx_bpf_dsq_reenq() kerneldoc claiming NMI calls are allowed sched_ext: Avoid unnecessary IMMED reenqueues on idle CPUs
This commit is contained in:
@@ -64,7 +64,13 @@ static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size,
|
||||
|
||||
for_each_cpu(cpu, llc_cpus) {
|
||||
nr_cpus++;
|
||||
if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu)
|
||||
/*
|
||||
* scx_cid_init() forms each core from the siblings in
|
||||
* @llc_cpus. The sibling mask may include cpus outside
|
||||
* @llc_cpus, so count a core at its first sibling in @llc_cpus.
|
||||
*/
|
||||
if (cpumask_first_and(topology_sibling_cpumask(cpu),
|
||||
llc_cpus) == cpu)
|
||||
nr_cores++;
|
||||
}
|
||||
|
||||
|
||||
+111
-49
@@ -461,6 +461,14 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
|
||||
* @rq is either picking, in transition to, or running an SCX task.
|
||||
*/
|
||||
|
||||
/*
|
||||
* A local insertion can raise next_class to ext while the donor is
|
||||
* still idle. Keep the rq open so that the local DSQ head is not
|
||||
* unnecessarily reenqueued before the CPU can pick it.
|
||||
*/
|
||||
if (rq->donor == rq->idle)
|
||||
return true;
|
||||
|
||||
/*
|
||||
* If we're in the dispatch path holding rq lock, $donor may or may not
|
||||
* be ready depending on whether the on-going dispatch decides to extend
|
||||
@@ -580,12 +588,14 @@ enum scx_dsq_iter_flags {
|
||||
/* iterate in the reverse dispatch order */
|
||||
SCX_DSQ_ITER_REV = 1U << 16,
|
||||
|
||||
__SCX_DSQ_ITER_DRAINED = 1U << 28,
|
||||
__SCX_DSQ_ITER_HAS_REENQ = 1U << 29,
|
||||
__SCX_DSQ_ITER_HAS_SLICE = 1U << 30,
|
||||
__SCX_DSQ_ITER_HAS_VTIME = 1U << 31,
|
||||
|
||||
__SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV,
|
||||
__SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS |
|
||||
__SCX_DSQ_ITER_DRAINED |
|
||||
__SCX_DSQ_ITER_HAS_REENQ |
|
||||
__SCX_DSQ_ITER_HAS_SLICE |
|
||||
__SCX_DSQ_ITER_HAS_VTIME,
|
||||
@@ -2225,11 +2235,11 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
|
||||
bool scx_rq_online(struct rq *rq)
|
||||
{
|
||||
/*
|
||||
* Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE follows the
|
||||
* core's rq online state. cpu_active() test guarantees that, if this
|
||||
* function returns %true, %SCX_RQ_ONLINE will stay set until the
|
||||
* current scheduling operation is complete even if we aren't locking
|
||||
* @rq.
|
||||
* Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE is set
|
||||
* after the root's ops.cpu_online() and cleared when the cpu goes down.
|
||||
* cpu_active() test guarantees that, if this function returns %true,
|
||||
* %SCX_RQ_ONLINE will stay set until the current scheduling operation
|
||||
* is complete even if we aren't locking @rq.
|
||||
*/
|
||||
return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
|
||||
}
|
||||
@@ -2420,7 +2430,7 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p)
|
||||
}
|
||||
|
||||
/*
|
||||
* list_add_tail() must be used. scx_bypass() and rq_offline_scx()
|
||||
* list_add_tail() must be used. scx_bypass() and drain_offline_rq()
|
||||
* depend on tasks being appended to the runnable_list.
|
||||
*/
|
||||
list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
|
||||
@@ -3399,6 +3409,12 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
|
||||
rq->scx.cpu_released = false;
|
||||
}
|
||||
|
||||
/* the ecaps sync may have dropped the rq lock */
|
||||
if (scx_dsp_prev_stale(rq, prev)) {
|
||||
verdict = SCX_DSP_RETRY;
|
||||
goto has_tasks;
|
||||
}
|
||||
|
||||
if (prev->sched_class == &ext_sched_class) {
|
||||
update_curr_scx(rq);
|
||||
|
||||
@@ -3441,6 +3457,10 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
|
||||
}
|
||||
|
||||
verdict = scx_dispatch_sched(root_sch, rq, prev, false);
|
||||
|
||||
/* scx_dispatch_sched() may have dropped the rq lock */
|
||||
if (scx_dsp_prev_stale(rq, prev))
|
||||
verdict = SCX_DSP_RETRY;
|
||||
if (verdict != SCX_DSP_NONE)
|
||||
goto has_tasks;
|
||||
|
||||
@@ -4296,12 +4316,37 @@ static void set_cpus_allowed_scx(struct task_struct *p,
|
||||
scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
|
||||
}
|
||||
|
||||
static void drain_offline_rq(struct rq *rq)
|
||||
{
|
||||
struct task_struct *p, *n;
|
||||
|
||||
scx_rescue_flush(rq);
|
||||
|
||||
/*
|
||||
* An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
|
||||
* onto the local DSQ so that they run here and balance_push() moves
|
||||
* them off.
|
||||
*/
|
||||
list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
|
||||
if (p->scx.dsq == &rq->scx.local_dsq)
|
||||
continue;
|
||||
guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
|
||||
}
|
||||
}
|
||||
|
||||
static void handle_hotplug(struct rq *rq, bool online)
|
||||
{
|
||||
struct scx_sched *sch = scx_root_protected();
|
||||
s32 cpu = cpu_of(rq);
|
||||
s32 cpu_or_cid = cpu;
|
||||
|
||||
if (!online) {
|
||||
guard(rq_lock_irqsave)(rq);
|
||||
update_rq_clock(rq);
|
||||
rq->scx.flags &= ~SCX_RQ_ONLINE;
|
||||
drain_offline_rq(rq);
|
||||
}
|
||||
|
||||
atomic_long_inc(&scx_hotplug_seq);
|
||||
scx_update_frozen_cpus(cpu, online);
|
||||
|
||||
@@ -4310,8 +4355,14 @@ static void handle_hotplug(struct rq *rq, bool online)
|
||||
* stable here. Note that we can't depend on scx_enabled() test as the
|
||||
* hotplug ops need to be enabled before __scx_enabled is set.
|
||||
*/
|
||||
if (unlikely(!sch))
|
||||
if (unlikely(!sch)) {
|
||||
/* no root to notify, set %SCX_RQ_ONLINE right away */
|
||||
if (online) {
|
||||
guard(rq_lock_irqsave)(rq);
|
||||
rq->scx.flags |= SCX_RQ_ONLINE;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (scx_enabled())
|
||||
scx_idle_update_selcpu_topology(&sch->ops);
|
||||
@@ -4362,8 +4413,22 @@ static void handle_hotplug(struct rq *rq, bool online)
|
||||
"cpu %d going %s, exiting scheduler", cpu,
|
||||
online ? "online" : "offline");
|
||||
|
||||
if (online)
|
||||
if (online) {
|
||||
/*
|
||||
* The cpu must not call ops.enqueue() or ops.dispatch() before
|
||||
* the root's ops.cpu_online(). The core's rq online state can't
|
||||
* gate them, as the sched domain rebuild in
|
||||
* sched_cpu_activate() can put @rq online before this function
|
||||
* runs. Set %SCX_RQ_ONLINE here.
|
||||
*
|
||||
* scx_online_ecaps() queues a reseed whose ecaps ops run in
|
||||
* dispatch context on the cpu and can insert into its local
|
||||
* DSQ. Set the flag first.
|
||||
*/
|
||||
scoped_guard (rq_lock_irqsave, rq)
|
||||
rq->scx.flags |= SCX_RQ_ONLINE;
|
||||
scx_online_ecaps(rq);
|
||||
}
|
||||
}
|
||||
|
||||
void scx_rq_activate(struct rq *rq)
|
||||
@@ -4376,35 +4441,6 @@ void scx_rq_deactivate(struct rq *rq)
|
||||
handle_hotplug(rq, false);
|
||||
}
|
||||
|
||||
static void rq_online_scx(struct rq *rq)
|
||||
{
|
||||
rq->scx.flags |= SCX_RQ_ONLINE;
|
||||
}
|
||||
|
||||
static void rq_offline_scx(struct rq *rq)
|
||||
{
|
||||
struct task_struct *p, *n;
|
||||
|
||||
rq->scx.flags &= ~SCX_RQ_ONLINE;
|
||||
|
||||
/* sched domain rebuilds call rq_offline with the CPU staying alive */
|
||||
if (cpu_active(cpu_of(rq)))
|
||||
return;
|
||||
|
||||
scx_rescue_flush(rq);
|
||||
|
||||
/*
|
||||
* An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
|
||||
* onto the local DSQ so that they run here and balance_push() moves
|
||||
* them off.
|
||||
*/
|
||||
list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
|
||||
if (p->scx.dsq == &rq->scx.local_dsq)
|
||||
continue;
|
||||
guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
|
||||
}
|
||||
}
|
||||
|
||||
static bool check_rq_for_timeouts(struct rq *rq)
|
||||
{
|
||||
struct scx_sched *sch;
|
||||
@@ -4728,6 +4764,7 @@ void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)
|
||||
args.cancelled = true;
|
||||
break;
|
||||
case SCX_TASK_READY:
|
||||
scx_reset_task_fields(task_rq(p), p);
|
||||
break;
|
||||
case SCX_TASK_ENABLED:
|
||||
scx_disable_task(sch, p);
|
||||
@@ -5876,9 +5913,6 @@ DEFINE_SCHED_CLASS(ext) = {
|
||||
.task_woken = task_woken_scx,
|
||||
.set_cpus_allowed = set_cpus_allowed_scx,
|
||||
|
||||
.rq_online = rq_online_scx,
|
||||
.rq_offline = rq_offline_scx,
|
||||
|
||||
.task_tick = task_tick_scx,
|
||||
|
||||
.switching_to = switching_to_scx,
|
||||
@@ -7779,10 +7813,17 @@ void scx_flush_disable_work(struct scx_sched *sch)
|
||||
{
|
||||
int kind;
|
||||
|
||||
/*
|
||||
* scx_disable_workfn() sets SCX_EXIT_DONE before tearing down. Read
|
||||
* exit_kind with acquire ordering (paired with atomic_try_cmpxchg() in
|
||||
* scx_disable_workfn()) before syncing and flushing so that, once DONE
|
||||
* is seen, the following flush observes the queued work and waits for
|
||||
* the teardown to finish.
|
||||
*/
|
||||
do {
|
||||
kind = atomic_read_acquire(&sch->exit_kind);
|
||||
irq_work_sync(&sch->disable_irq_work);
|
||||
kthread_flush_work(&sch->disable_work);
|
||||
kind = atomic_read(&sch->exit_kind);
|
||||
} while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
|
||||
}
|
||||
|
||||
@@ -11028,13 +11069,24 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
|
||||
__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
|
||||
{
|
||||
struct bpf_iter_scx_dsq_kern *kit = (void *)it;
|
||||
struct task_struct *p;
|
||||
|
||||
if (!kit->dsq)
|
||||
if (!kit->dsq || (kit->cursor.flags & __SCX_DSQ_ITER_DRAINED))
|
||||
return NULL;
|
||||
|
||||
guard(raw_spinlock_irqsave)(&kit->dsq->lock);
|
||||
|
||||
return nldsq_cursor_next_task(&kit->cursor, kit->dsq);
|
||||
p = nldsq_cursor_next_task(&kit->cursor, kit->dsq);
|
||||
/*
|
||||
* nldsq_cursor_next_task() unlinks the cursor and starts from the DSQ
|
||||
* head when it is called again. The verifier relies on an iterator
|
||||
* which returned NULL to keep returning NULL (see
|
||||
* process_iter_next_call()), so remember the drained state.
|
||||
*/
|
||||
if (!p)
|
||||
kit->cursor.flags |= __SCX_DSQ_ITER_DRAINED;
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -11107,7 +11159,7 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
|
||||
/**
|
||||
* scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ
|
||||
* @dsq_id: DSQ to re-enqueue
|
||||
* @reenq_flags: %SCX_RENQ_*
|
||||
* @reenq_flags: %SCX_REENQ_*
|
||||
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
|
||||
*
|
||||
* Iterate over all of the tasks currently enqueued on the DSQ identified by
|
||||
@@ -11117,8 +11169,9 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
|
||||
* - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu)
|
||||
* - User DSQs
|
||||
*
|
||||
* Re-enqueues are performed asynchronously. Can be called from anywhere. A
|
||||
* re-enqueue of an inactive cpu's local DSQ is ignored.
|
||||
* Re-enqueues are performed asynchronously. Can be called from any context
|
||||
* except NMI. A call from NMI aborts the scheduler. A re-enqueue of an inactive
|
||||
* cpu's local DSQ is ignored.
|
||||
*
|
||||
* %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
|
||||
* operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
|
||||
@@ -11215,9 +11268,10 @@ __bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter,
|
||||
* scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ
|
||||
* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
|
||||
*
|
||||
* Iterate over all of the tasks currently enqueued on the local DSQ of the
|
||||
* caller's CPU, and re-enqueue them in the BPF scheduler. Can be called from
|
||||
* anywhere.
|
||||
* Iterate over all of the tasks currently enqueued on the local DSQ of the rq
|
||||
* the current scheduler operation is locked to, or the calling CPU's when no rq
|
||||
* is locked, and re-enqueue them in the BPF scheduler. Can be called from any
|
||||
* context except NMI. A call from NMI aborts the scheduler.
|
||||
*
|
||||
* This is now a special case of scx_bpf_dsq_reenq() and may be removed in the
|
||||
* future.
|
||||
@@ -12134,6 +12188,11 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = {
|
||||
* when scx_locked_rq() returns NULL. Taking it while the context already holds
|
||||
* an rq lock, the same one or another cpu's, can deadlock.
|
||||
*
|
||||
* scx_bpf_destroy_dsq() takes a user DSQ lock. So do
|
||||
* bpf_iter_scx_dsq_next() and bpf_iter_scx_dsq_destroy(), the latter only
|
||||
* after a successful next() has linked the cursor. A tracing program can
|
||||
* run while that lock is already held and recursively acquire it.
|
||||
*
|
||||
* Before returning, scx_bpf_sub_grant() and scx_bpf_sub_revoke() invoke
|
||||
* ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops
|
||||
* run in the tracing program's context. No SCX op expects to run from an
|
||||
@@ -12148,6 +12207,9 @@ BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
|
||||
BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)
|
||||
BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
|
||||
BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
|
||||
BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
|
||||
BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
|
||||
BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
|
||||
BTF_KFUNCS_END(scx_kfunc_ids_no_tracing)
|
||||
|
||||
/*
|
||||
|
||||
@@ -18,9 +18,32 @@ enum scx_dsp_verdict {
|
||||
SCX_DSP_NONE, /* nothing to run */
|
||||
SCX_DSP_LOCAL, /* local DSQ has tasks */
|
||||
SCX_DSP_PREV, /* keep running @prev */
|
||||
SCX_DSP_RETRY, /* pick helpers only: restart the pick */
|
||||
SCX_DSP_RETRY, /* restart the pick */
|
||||
};
|
||||
|
||||
/**
|
||||
* scx_dsp_prev_stale - Test whether dispatch's @prev went stale
|
||||
* @rq: rq being dispatched for
|
||||
* @prev: @rq's donor when dispatch started
|
||||
*
|
||||
* @prev can stop being @rq's donor while dispatch has the rq lock dropped. With
|
||||
* proxy execution, @prev may not be the running task, and another CPU can reset
|
||||
* the donor to rq->curr. With core scheduling, the core pick runs dispatch for
|
||||
* the sibling rqs too, and a sibling can switch @prev out. Dispatch restarts
|
||||
* the pick when that happened and doesn't hand the old donor to the BPF
|
||||
* scheduler.
|
||||
*
|
||||
* With proxy execution alone, the donor only ever changes to rq->curr during a
|
||||
* pick. Only __schedule() on this CPU sets a different donor, after the pick,
|
||||
* so the restarted pick takes rq->curr as @prev and can't go stale again. Under
|
||||
* core scheduling, the restart goes through the core pick, which already
|
||||
* retries whenever dispatch dropped the lock.
|
||||
*/
|
||||
static inline bool scx_dsp_prev_stale(struct rq *rq, struct task_struct *prev)
|
||||
{
|
||||
return (sched_proxy_exec() || sched_core_enabled(rq)) && unlikely(prev != rq->donor);
|
||||
}
|
||||
|
||||
/*
|
||||
* One user of this function is scx_bpf_sub_dispatch() which can be called
|
||||
* recursively as sub-sched dispatches nest. Always inline to reduce stack usage
|
||||
@@ -33,8 +56,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
||||
struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
|
||||
int nr_loops = SCX_DSP_MAX_LOOPS;
|
||||
s32 cpu = cpu_of(rq);
|
||||
bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
|
||||
scx_task_on_sched(sch, prev);
|
||||
bool prev_on_sch;
|
||||
|
||||
if (scx_consume_global_dsq(sch, rq))
|
||||
return SCX_DSP_LOCAL;
|
||||
@@ -82,7 +104,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
||||
/*
|
||||
* scx_rq_online() can't be used. Its cpu_active() test goes false
|
||||
* before CPU hotplug waits for an RCU grace period, and
|
||||
* rq_offline_scx() moves this CPU's tasks to the local DSQ only after
|
||||
* handle_hotplug() moves this CPU's tasks to the local DSQ only after
|
||||
* the wait. The grace period can depend on those tasks running.
|
||||
*/
|
||||
if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE))
|
||||
@@ -106,6 +128,11 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
||||
rq->scx.sub_dispatch_prev = prev;
|
||||
#endif
|
||||
|
||||
/* the dispatch so far may have dropped the rq lock */
|
||||
prev_on_sch = !scx_dsp_prev_stale(rq, prev) &&
|
||||
prev->sched_class == &ext_sched_class &&
|
||||
scx_task_on_sched(sch, prev);
|
||||
|
||||
SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
|
||||
prev_on_sch ? prev : NULL);
|
||||
|
||||
@@ -116,7 +143,9 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
||||
|
||||
scx_flush_dispatch_buf(sch, rq);
|
||||
|
||||
if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
|
||||
/* ops.dispatch() and the flush may have dropped the rq lock */
|
||||
if (!scx_dsp_prev_stale(rq, prev) &&
|
||||
(prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice)
|
||||
return SCX_DSP_PREV;
|
||||
if (rq->scx.local_dsq.nr)
|
||||
return SCX_DSP_LOCAL;
|
||||
|
||||
@@ -869,6 +869,10 @@ struct sched_ext_ops {
|
||||
* @sub_attach: Attach a sub-scheduler
|
||||
* @args: argument container, see the struct definition
|
||||
*
|
||||
* The sub-scheduler's cgroup is online for the whole call. If it goes
|
||||
* offline after the sub-scheduler is accepted, ops.sub_detach()
|
||||
* follows.
|
||||
*
|
||||
* Return 0 to accept the sub-scheduler. -errno to reject.
|
||||
*/
|
||||
s32 (*sub_attach)(struct scx_sub_attach_args *args);
|
||||
|
||||
+18
-4
@@ -918,10 +918,11 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq
|
||||
return &rq->scx.local_dsq;
|
||||
|
||||
/*
|
||||
* The task must run on this CPU regardless of caps: the rq is draining
|
||||
* offline (BPF scheduler bypassed), the task is migration-disabled, or a
|
||||
* migration is pending. Admit despite the missing caps and count it.
|
||||
* Refuse preemptions.
|
||||
* The task must run on this CPU regardless of caps: ops.enqueue() is
|
||||
* skipped while the CPU is going down or before the root's
|
||||
* ops.cpu_online(), the task is migration-disabled, or a migration is
|
||||
* pending. Admit despite the missing caps and count it. Refuse
|
||||
* preemptions.
|
||||
*/
|
||||
if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
|
||||
p->migration_pending)) {
|
||||
@@ -2340,8 +2341,21 @@ void scx_sub_enable_workfn(struct kthread_work *work)
|
||||
.cgroup_path = sch->cgrp_path,
|
||||
};
|
||||
|
||||
/*
|
||||
* The cgroup stays online across ops.sub_attach(), so the parent never
|
||||
* sees an attach for a cgroup that rmdir already removed.
|
||||
*/
|
||||
cgroup_lock();
|
||||
if (!(cgrp->self.flags & CSS_ONLINE)) {
|
||||
cgroup_unlock();
|
||||
scx_error(sch, "cgroup is not online");
|
||||
ret = -ENODEV;
|
||||
goto err_disable;
|
||||
}
|
||||
ret = SCX_CALL_OP_RET(parent, sub_attach, NULL,
|
||||
&sub_attach_args);
|
||||
cgroup_unlock();
|
||||
|
||||
if (ret) {
|
||||
ret = scx_ops_sanitize_err(sch, "sub_attach", ret);
|
||||
scx_error(sch, "parent rejected (%d)", ret);
|
||||
|
||||
@@ -781,19 +781,20 @@ struct cfs_rq {
|
||||
/* scx_rq->flags, protected by the rq lock */
|
||||
enum scx_rq_flags {
|
||||
/*
|
||||
* Set while the core has the rq online, see rq_online_scx() and
|
||||
* rq_offline_scx(). A hotplugged cpu starts scheduling before
|
||||
* rq_online_scx(). ops.enqueue() and ops.dispatch() are called for the
|
||||
* cpu only while it is set, see scx_rq_online() and
|
||||
* scx_dispatch_sched().
|
||||
* Set right after the root's ops.cpu_online() when a cpu comes up and
|
||||
* cleared when the cpu goes down, see handle_hotplug(). A hotplugged
|
||||
* cpu starts scheduling before %SCX_RQ_ONLINE is set. ops.enqueue() and
|
||||
* ops.dispatch() are called for the cpu only while the flag is set, see
|
||||
* scx_rq_online() and scx_dispatch_sched().
|
||||
*/
|
||||
SCX_RQ_ONLINE = 1 << 0,
|
||||
SCX_RQ_CAN_STOP_TICK = 1 << 1,
|
||||
/*
|
||||
* Set while the root considers the cpu online, from its
|
||||
* ops.cpu_online() until its ops.cpu_offline(). ecaps are delivered
|
||||
* only while it is set. %SCX_RQ_ONLINE follows the core's rq state
|
||||
* instead, which changes ahead of the root's view on both edges.
|
||||
* only while it is set. %SCX_RQ_ONLINE can't gate them, as a cpu going
|
||||
* down clears it before the ecaps loss is reported, see
|
||||
* scx_offline_ecaps().
|
||||
*/
|
||||
SCX_RQ_OPS_ONLINE = 1 << 2,
|
||||
SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */
|
||||
@@ -4308,7 +4309,7 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
|
||||
* Typically this must be called while holding task_rq_lock, since most/all
|
||||
* properties are serialized under those locks. There are currently two
|
||||
* exceptions to this rule in sched/ext which only hold rq->lock: scx_bypass()
|
||||
* and rq_offline_scx().
|
||||
* and drain_offline_rq().
|
||||
*/
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user