diff --git a/include/linux/sched.h b/include/linux/sched.h index 1c93da75226e..e4f6323d7855 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -881,6 +881,8 @@ struct task_struct { */ int recent_used_cpu; int wake_cpu; + /* CPU charged while is_blocked && in_iowait. */ + int iowait_cpu; int prio; int static_prio; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 04b62c16fa8c..1a088d4576f5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -636,7 +636,7 @@ int task_llc(const struct task_struct *p) * * p->is_blocked <- { 0, 1 }: * - * is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks + * is set by try_to_block_task() and cleared by unblock_task() and tracks * if the task is blocked. Traditionally this would mirror p->on_rq, however * due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge. * @@ -3773,12 +3773,22 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) __schedstat_inc(p->stats.nr_wakeups_sync); } +static inline void unblock_task(struct task_struct *p) +{ + if (p->is_blocked && p->in_iowait) { + delayacct_blkio_end(p); + atomic_dec(&cpu_rq(p->iowait_cpu)->nr_iowait); + } + + p->is_blocked = 0; +} + /* * Mark the task runnable. */ static inline void ttwu_do_wakeup(struct task_struct *p) { - p->is_blocked = 0; + unblock_task(p); WRITE_ONCE(p->__state, TASK_RUNNING); trace_sched_wakeup(p); } @@ -3889,10 +3899,6 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, en_flags |= ENQUEUE_RQ_SELECTED; if (wake_flags & WF_MIGRATED) en_flags |= ENQUEUE_MIGRATED; - else if (p->in_iowait) { - delayacct_blkio_end(p); - atomic_dec(&task_rq(p)->nr_iowait); - } activate_task(rq, p, en_flags); wakeup_preempt(rq, p, wake_flags); @@ -4465,11 +4471,6 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) cpu = select_task_rq(p, p->wake_cpu, &wake_flags); if (task_cpu(p) != cpu) { - if (p->in_iowait) { - delayacct_blkio_end(p); - atomic_dec(&task_rq(p)->nr_iowait); - } - wake_flags |= WF_MIGRATED; psi_ttwu_dequeue(p); set_task_cpu(p, cpu); @@ -6813,6 +6814,12 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p, p->is_blocked = 1; + if (p->in_iowait) { + p->iowait_cpu = cpu_of(rq); + atomic_inc(&rq->nr_iowait); + delayacct_blkio_start(); + } + /* * We check should_block after signal_pending because we * will want to wake the task in that case. But if @@ -7020,7 +7027,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) if (!mutex) { clear_task_blocked_on(p, mutex); if (task_current(rq, p)) { - p->is_blocked = 0; + unblock_task(p); return p; } goto deactivate; @@ -7056,7 +7063,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) */ __clear_task_blocked_on(p, NULL); if (task_current(rq, p)) { - p->is_blocked = 0; + unblock_task(p); return p; } goto deactivate; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index c0ebdcde5fe5..dfae08625342 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -444,7 +444,7 @@ static void task_non_contending(struct sched_dl_entity *dl_se, bool dl_task) if (!dl_task || READ_ONCE(p->__state) == TASK_DEAD) { struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); - if (READ_ONCE(p->__state) == TASK_DEAD) + if (dl_task && READ_ONCE(p->__state) == TASK_DEAD) sub_rq_bw(dl_se, &rq->dl); raw_spin_lock(&dl_b->lock); __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index cfc982205b98..79f6b3a4fd7f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6964,8 +6964,8 @@ static void dequeue_throttled_task(struct task_struct *p, int flags) WARN_ON_ONCE(p->se.on_rq); list_del_init(&p->throttle_node); - /* task blocked after throttled */ - if (flags & DEQUEUE_SLEEP) { + /* Task blocked after throttling or is leaving the fair class: */ + if (flags & (DEQUEUE_SLEEP | DEQUEUE_CLASS)) { p->throttled = false; return; } @@ -11029,7 +11029,14 @@ alb_break_llc(struct lb_env *env) return true; cur = rcu_dereference_all(env->src_rq->curr); - if (cur && cur->sched_class == &fair_sched_class) + /* + * Active balance only detaches CFS tasks, so a non-fair + * curr is never migrated. Ignore it and avoid using its + * state util in the misfit and LLC checks. + */ + if (cur && cur->sched_class != &fair_sched_class) + cur = NULL; + if (cur) util = task_util(cur); if (task_misfits_asym_cpu(env, cur) || diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 779eb58a4261..d10d57b6b8ec 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -99,7 +99,7 @@ static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3) * d1 y^p + 1024 \Sum y^n + d3 y^0 (Step 2) * n=1 */ -static __always_inline u32 +static __always_inline u64 accumulate_sum(u64 delta, struct sched_avg *sa, unsigned long load, unsigned long runnable, int running) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6c7c5633524a..1d76950dbb0a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3124,11 +3124,6 @@ static inline void __block_task(struct rq *rq, struct task_struct *p) if (p->sched_contributes_to_load) rq->nr_uninterruptible++; - if (p->in_iowait) { - atomic_inc(&rq->nr_iowait); - delayacct_blkio_start(); - } - ASSERT_EXCLUSIVE_WRITER(p->on_rq); /* diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 919d0fb00bd9..9d3a1b8dd393 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -3332,13 +3332,25 @@ int __init sched_init_domains(const struct cpumask *cpu_map) static void detach_destroy_domains(const struct cpumask *cpu_map) { unsigned int cpu = cpumask_any(cpu_map); + bool has_cluster = false; int i; if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu))) static_branch_dec_cpuslocked(&sched_asym_cpucapacity); - if (static_branch_unlikely(&sched_cluster_active)) - static_branch_dec_cpuslocked(&sched_cluster_active); + if (static_branch_unlikely(&sched_cluster_active)) { + rcu_read_lock(); + for_each_cpu(i, cpu_map) { + if (lowest_flag_domain(i, SD_CLUSTER)) { + has_cluster = true; + break; + } + } + rcu_read_unlock(); + + if (has_cluster) + static_branch_dec_cpuslocked(&sched_cluster_active); + } rcu_read_lock(); for_each_cpu(i, cpu_map)