From 0c93e25ef11a9dee43f6076bd1f09771dc2b167e Mon Sep 17 00:00:00 2001 From: Nickolai Zeldovich Date: Sun, 25 Oct 2026 08:46:57 -0400 Subject: [PATCH 1/6] sched/deadline: Fix this_bw on class change of a dying task Moving a SCHED_DEADLINE task out of the class with sched_setscheduler(2) after it stored TASK_DEAD in do_task_dead(), but before __schedule() dequeued it, subtracts its bandwidth from the runqueue's this_bw twice. this_bw then stays short until reboot: __sub_rq_bw() warns about the underflow or about running_bw > this_bw, and GRUB reclaim for SCHED_FLAG_RECLAIM tasks on that CPU is miscomputed. The class change dequeues the still queued task with DEQUEUE_SAVE, which removes its bandwidth from running_bw and this_bw. switched_from_dl() then calls task_non_contending(), which, if the 0-lag time has passed, removes it from this_bw again because the task is TASK_DEAD. That removal is meant for the dying task's own DEQUEUE_SLEEP dequeue, which has removed nothing before it. Do the TASK_DEAD removal only on that dequeue path. Reproduced under QEMU on x86_64 and riscv64 by moving exiting deadline tasks to SCHED_OTHER in a loop; with this patch there is no warning. Fixes: 8fd27231c330 ("sched/deadline: Track the "total rq utilization" too") Signed-off-by: Nickolai Zeldovich Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Acked-by: Juri Lelli Tested-by: Juri Lelli Link: https://patch.msgid.link/20261025124657.836229-1-nickolai@csail.mit.edu --- kernel/sched/deadline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index c0ebdcde5fe5..dfae08625342 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -444,7 +444,7 @@ static void task_non_contending(struct sched_dl_entity *dl_se, bool dl_task) if (!dl_task || READ_ONCE(p->__state) == TASK_DEAD) { struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); - if (READ_ONCE(p->__state) == TASK_DEAD) + if (dl_task && READ_ONCE(p->__state) == TASK_DEAD) sub_rq_bw(dl_se, &rq->dl); raw_spin_lock(&dl_b->lock); __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); From 0965587d00b84f0a3b0fc94930d4a5f8ff011eda Mon Sep 17 00:00:00 2001 From: Christian Loehle Date: Thu, 15 Oct 2026 12:37:54 +0100 Subject: [PATCH 2/6] sched/core: Account I/O waits independently of delayed dequeue Starting blkio delay accounting and incrementing rq->nr_iowait in __block_task() ties both to removal from the runqueue. With EEVDF delayed dequeue, this misses the initial part of an I/O sleep, or the entire sleep if the task wakes before it is dequeued. Such tasks are also missing from procs_blocked while their dequeue is delayed. Final dequeue can run in another task's context, causing delayacct_blkio_start() to timestamp the wrong task. Start both forms of accounting in try_to_block_task(), when the current task commits to blocking, before delayed dequeue or proxy execution can retain it on the runqueue. End both when clearing is_blocked, including queued wakeups and the proxy execution paths that let the current task continue. Checking is_blocked excludes wakeups that precede blocking. Save the CPU charged at the start of the wait. Delayed tasks can migrate while still queued, and proxy execution can also move blocked tasks. Decrement the saved CPU's counter on unblock, regardless of the task's current placement, and remove the old dequeue-based accounting. CPU iowait continues to classify idle time using the counter on the charged CPU; it does not accumulate the full per-task blocked duration. Closes: https://lore.kernel.org/r/20261014081731.1068794-1-zhanxusheng@xiaomi.com Fixes: 152e11f6df29 ("sched/fair: Implement delayed dequeue") Reported-by: Zhan Xusheng Signed-off-by: Christian Loehle Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Zhan Xusheng Reviewed-by: Kayra Cizmeci Link: https://patch.msgid.link/fd8578a1-e60b-4993-ab51-2b5d63abcf77@arm.com --- include/linux/sched.h | 2 ++ kernel/sched/core.c | 33 ++++++++++++++++++++------------- kernel/sched/sched.h | 5 ----- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 1c93da75226e..e4f6323d7855 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -881,6 +881,8 @@ struct task_struct { */ int recent_used_cpu; int wake_cpu; + /* CPU charged while is_blocked && in_iowait. */ + int iowait_cpu; int prio; int static_prio; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 04b62c16fa8c..1a088d4576f5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -636,7 +636,7 @@ int task_llc(const struct task_struct *p) * * p->is_blocked <- { 0, 1 }: * - * is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks + * is set by try_to_block_task() and cleared by unblock_task() and tracks * if the task is blocked. Traditionally this would mirror p->on_rq, however * due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge. * @@ -3773,12 +3773,22 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) __schedstat_inc(p->stats.nr_wakeups_sync); } +static inline void unblock_task(struct task_struct *p) +{ + if (p->is_blocked && p->in_iowait) { + delayacct_blkio_end(p); + atomic_dec(&cpu_rq(p->iowait_cpu)->nr_iowait); + } + + p->is_blocked = 0; +} + /* * Mark the task runnable. */ static inline void ttwu_do_wakeup(struct task_struct *p) { - p->is_blocked = 0; + unblock_task(p); WRITE_ONCE(p->__state, TASK_RUNNING); trace_sched_wakeup(p); } @@ -3889,10 +3899,6 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, en_flags |= ENQUEUE_RQ_SELECTED; if (wake_flags & WF_MIGRATED) en_flags |= ENQUEUE_MIGRATED; - else if (p->in_iowait) { - delayacct_blkio_end(p); - atomic_dec(&task_rq(p)->nr_iowait); - } activate_task(rq, p, en_flags); wakeup_preempt(rq, p, wake_flags); @@ -4465,11 +4471,6 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) cpu = select_task_rq(p, p->wake_cpu, &wake_flags); if (task_cpu(p) != cpu) { - if (p->in_iowait) { - delayacct_blkio_end(p); - atomic_dec(&task_rq(p)->nr_iowait); - } - wake_flags |= WF_MIGRATED; psi_ttwu_dequeue(p); set_task_cpu(p, cpu); @@ -6813,6 +6814,12 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, p->is_blocked = 1; + if (p->in_iowait) { + p->iowait_cpu = cpu_of(rq); + atomic_inc(&rq->nr_iowait); + delayacct_blkio_start(); + } + /* * We check should_block after signal_pending because we * will want to wake the task in that case. But if @@ -7020,7 +7027,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) if (!mutex) { clear_task_blocked_on(p, mutex); if (task_current(rq, p)) { - p->is_blocked = 0; + unblock_task(p); return p; } goto deactivate; @@ -7056,7 +7063,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) */ __clear_task_blocked_on(p, NULL); if (task_current(rq, p)) { - p->is_blocked = 0; + unblock_task(p); return p; } goto deactivate; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6c7c5633524a..1d76950dbb0a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3124,11 +3124,6 @@ static inline void __block_task(struct rq *rq, struct task_struct *p) if (p->sched_contributes_to_load) rq->nr_uninterruptible++; - if (p->in_iowait) { - atomic_inc(&rq->nr_iowait); - delayacct_blkio_start(); - } - ASSERT_EXCLUSIVE_WRITER(p->on_rq); /* From c282051fc2cae2ea54770ebe435dbea090272c25 Mon Sep 17 00:00:00 2001 From: Daehyeon Ko <4ncienth@gmail.com> Date: Fri, 23 Oct 2026 08:52:33 +0900 Subject: [PATCH 3/6] sched/fair: Clear throttle state when leaving the fair class Task-based CFS bandwidth throttling removes a task's fair sched_entity while the scheduler core still considers the task queued. p->throttled carries the fair-specific state across dequeue/enqueue operations. DEQUEUE_CLASS removes the task from cfs_rq->throttled_limbo_list but retains the bit. If the last finite CFS quota is disabled while the task runs in another class, it is absent from the unthrottle walk and the bit survives. Returning to fair while cfs_bandwidth_used() is false then performs a real fair enqueue with p->throttled still set. After bandwidth is re-enabled, a class-change dequeue treats the stale bit as limbo state and skips __dequeue_task(). p->se.on_rq therefore remains set, and put_prev_task_fair() places the entity in the EEVDF tree while the task moves to RT. On exit, the fair picker can select the TASK_DEAD task. __schedule() then returns to do_task_dead(), which hits BUG(). Keep p->throttled across migrations and group moves, but clear it on DEQUEUE_CLASS because fair-specific state must not outlive fair-class membership. A later return to fair uses the normal enqueue path. That path still respects a throttled cfs_rq. A KASAN, LOCKDEP and DEBUG_LIST v7.3 kernel reproduced the do_task_dead() BUG in 3/3 root and 3/3 delegated uid-1000 runs with CapPrm=CapEff=0 and RLIMIT_RTPRIO=1. The fix passed both matrices plus a 3/3 class-switch control with bandwidth continuously enabled, without a scheduler warning, KASAN report, lockdep splat or crash. An AI coding assistant analyzed the state mismatch and drafted the initial reproducer, fix candidate and this changelog. The source paths, trigger, controls and results were rechecked against the cited revisions. Fixes: e1fad12dcb66 ("sched/fair: Switch to task based throttle model") Signed-off-by: Daehyeon Ko <4ncienth@gmail.com> Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Aaron Lu Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20261022235234.150957-1-4ncienth@gmail.com --- kernel/sched/fair.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index cfc982205b98..ad414539bc4f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6964,8 +6964,8 @@ static void dequeue_throttled_task(struct task_struct *p, int flags) WARN_ON_ONCE(p->se.on_rq); list_del_init(&p->throttle_node); - /* task blocked after throttled */ - if (flags & DEQUEUE_SLEEP) { + /* Task blocked after throttling or is leaving the fair class: */ + if (flags & (DEQUEUE_SLEEP | DEQUEUE_CLASS)) { p->throttled = false; return; } From a0c755e574e37fddb59edf175f27dc3877e7fe23 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Fri, 23 Oct 2026 11:52:07 +0900 Subject: [PATCH 4/6] sched/pelt: Fix u32 truncation of the PELT period count accumulate_sum() computes the number of elapsed PELT periods in a u64 but returns u32. When the count is a multiple of 2^32, it is truncated to zero even though the PELT sums have been updated. The caller interprets zero as no period boundary crossed and skips recomputing the averages, leaving *_avg stale relative to *_sum. Return the period count as u64 so the caller can test it without truncation. Fixes: a481db34b9be ("sched/fair: Optimize ___update_sched_avg()") Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Vincent Guittot Reviewed-by: Kayra Cizmeci Link: https://patch.msgid.link/20261023025207.2967903-1-sh_def@163.com --- kernel/sched/pelt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 779eb58a4261..d10d57b6b8ec 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -99,7 +99,7 @@ static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3) * d1 y^p + 1024 \Sum y^n + d3 y^0 (Step 2) * n=1 */ -static __always_inline u32 +static __always_inline u64 accumulate_sum(u64 delta, struct sched_avg *sa, unsigned long load, unsigned long runnable, int running) { From 5ebdfe12ab96368d0339593129d4ce0bc2bd06f5 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 21 Oct 2026 13:38:34 +0900 Subject: [PATCH 5/6] sched/topology: Balance cluster static-key updates build_sched_domains() increments sched_cluster_active once per newly built partition that contains an SD_CLUSTER domain. detach_destroy_domains() instead tested the global static key before decrementing it, so destroying a cluster-less partition could drop a reference owned by another partition and disable cluster-aware scheduling while SD_CLUSTER domains remained installed. Check the installed domains before detaching them, only while the key is enabled, and decrement only when the removed partition contains an SD_CLUSTER domain. With two cluster-bearing partitions installed, removing a singleton partition left the baseline count at 1 where 2 partitions still carried SD_CLUSTER; the fixed kernel correctly reports 2. On an asymmetric cluster topology where the parent partition gains no cluster domain, the baseline count drops to 0 while a cluster partition remains installed; the fixed kernel keeps it at 1. Fixes: 8881e1639f1f ("sched/fair: Scan cluster before scanning LLC in wake-up path") Signed-off-by: Hui Su Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: K Prateek Nayak Reviewed-by: Kayra Cizmeci Link: https://patch.msgid.link/20261021043834.1140951-1-sh_def@163.com --- kernel/sched/topology.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 919d0fb00bd9..9d3a1b8dd393 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -3332,13 +3332,25 @@ int __init sched_init_domains(const struct cpumask *cpu_map) static void detach_destroy_domains(const struct cpumask *cpu_map) { unsigned int cpu = cpumask_any(cpu_map); + bool has_cluster = false; int i; if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu))) static_branch_dec_cpuslocked(&sched_asym_cpucapacity); - if (static_branch_unlikely(&sched_cluster_active)) - static_branch_dec_cpuslocked(&sched_cluster_active); + if (static_branch_unlikely(&sched_cluster_active)) { + rcu_read_lock(); + for_each_cpu(i, cpu_map) { + if (lowest_flag_domain(i, SD_CLUSTER)) { + has_cluster = true; + break; + } + } + rcu_read_unlock(); + + if (has_cluster) + static_branch_dec_cpuslocked(&sched_cluster_active); + } rcu_read_lock(); for_each_cpu(i, cpu_map) From 006991aa89974378ee579416dba87adc22cd60fd Mon Sep 17 00:00:00 2001 From: Kayra Cizmeci Date: Tue, 20 Oct 2026 21:54:01 +0300 Subject: [PATCH 6/6] sched/cache: Ignore non-fair current in alb_break_llc() Commit d6b9afab44e23 ("sched/cache: Fix checking active load balance by only considering the CFS task") stopped using task_util() for non-fair cur. __active_load_balance_cpu_stop() calls detach_one_task(), that only looks to cfs_tasks list. So if the rq->curr is not fair, doing a decision according to it is wrong, since we only migrate tasks that is fair. Commit f0d243a96f26 ("sched/fair: Avoid creating misfits during cache-aware balancing") passes cur to task_misfits_asym_cpu() without checking if the cur is in fair class or not. Fix this. Fixes: f0d243a96f26 ("sched/fair: Avoid creating misfits during cache-aware balancing") Signed-off-by: Kayra Cizmeci Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Tim Chen Link: https://patch.msgid.link/20261020185402.57017-1-kayracizmeci@gmail.com --- kernel/sched/fair.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ad414539bc4f..79f6b3a4fd7f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11029,7 +11029,14 @@ alb_break_llc(struct lb_env *env) return true; cur = rcu_dereference_all(env->src_rq->curr); - if (cur && cur->sched_class == &fair_sched_class) + /* + * Active balance only detaches CFS tasks, so a non-fair + * curr is never migrated. Ignore it and avoid using its + * state util in the misfit and LLC checks. + */ + if (cur && cur->sched_class != &fair_sched_class) + cur = NULL; + if (cur) util = task_util(cur); if (task_misfits_asym_cpu(env, cur) ||