diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1a088d4576f5..5ed660f099a5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6297,7 +6297,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { struct task_struct *next, *p, *max; - const struct cpumask *smt_mask; bool fi_before = false; bool core_clock_updated = (rq == rq->core); unsigned long cookie; @@ -6344,8 +6343,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf) prev_balance(rq, rf); - smt_mask = cpu_smt_mask(cpu); - restart: need_sync |= !!rq->core->core_cookie; @@ -6413,7 +6410,7 @@ restart: * Tie-break prio towards the current CPU */ max = NULL; - for_each_cpu_wrap(i, smt_mask, cpu) { + for_each_core_cpu_wrap(i, rq) { rq_i = cpu_rq(i); /* @@ -6446,7 +6443,7 @@ restart: * For each thread: try and find a runnable task that matches @max or * force idle. */ - for_each_cpu(i, smt_mask) { + for_each_core_cpu(i, rq) { rq_i = cpu_rq(i); p = rq_i->core_pick; @@ -6491,7 +6488,7 @@ restart: * their task. This ensures there is no inter-sibling overlap between * non-matching user state. */ - for_each_cpu(i, smt_mask) { + for_each_core_cpu(i, rq) { rq_i = cpu_rq(i); /* diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c index 43e0bde3038e..d56be1c31435 100644 --- a/kernel/sched/core_sched.c +++ b/kernel/sched/core_sched.c @@ -241,7 +241,6 @@ out: /* REQUIRES: rq->core's clock recently updated. */ void __sched_core_account_forceidle(struct rq *rq) { - const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); u64 delta, now = rq_clock(rq->core); struct rq *rq_i; struct task_struct *p; @@ -273,7 +272,7 @@ void __sched_core_account_forceidle(struct rq *rq) delta = div_u64(delta, rq->core->core_forceidle_occupation); } - for_each_cpu(i, smt_mask) { + for_each_core_cpu(i, rq) { rq_i = cpu_rq(i); p = rq_i->core_pick ?: rq_i->curr; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 79f6b3a4fd7f..0c8af1cac2d5 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1469,6 +1469,12 @@ static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu) return false; } +/* Are the running threads (in SMT siblings) below ~80% of the LLC's CPUs? */ +static inline bool fits_llc_nr(u64 nr_threads, unsigned int nr_cpus) +{ + return nr_threads * 5 < (u64)nr_cpus * 4; +} + static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, int cpu) { @@ -1485,8 +1491,8 @@ static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, if (scale == INT_MAX) return false; - return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), - (scale * per_cpu(sd_llc_size, cpu))); + return !fits_llc_nr(READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads, + scale * per_cpu(sd_llc_size, cpu)); } /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1d76950dbb0a..92134ce326a4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1534,6 +1534,25 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq) return &rq->__lock; } +/* + * A CPU coming online is set in cpu_smt_mask() before + * sched_core_cpu_starting() links its rq->core to the core leader. + * Until then it does not share the core-wide lock, so it must not be + * touched as part of the core. + */ +static inline bool sched_core_sibling(struct rq *rq, struct rq *rq_i) +{ + return rq_i->core == rq->core; +} + +#define for_each_core_cpu(i, rq) \ + for_each_cpu(i, cpu_smt_mask(rq->cpu)) \ + if (sched_core_sibling(rq, cpu_rq(i))) + +#define for_each_core_cpu_wrap(i, rq) \ + for_each_cpu_wrap(i, cpu_smt_mask(rq->cpu), rq->cpu) \ + if (sched_core_sibling(rq, cpu_rq(i))) + extern bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, bool fi); diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index 45a127deeadd..0ec401af80d1 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -318,8 +318,8 @@ static inline void __setscheduler_dl_pi(int newprio, int policy, static int uclamp_validate(struct task_struct *p, const struct sched_attr *attr) { - int util_min = p->uclamp_req[UCLAMP_MIN].value; - int util_max = p->uclamp_req[UCLAMP_MAX].value; + unsigned int util_min = p->uclamp_req[UCLAMP_MIN].value; + unsigned int util_max = p->uclamp_req[UCLAMP_MAX].value; if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP_MIN) { util_min = attr->sched_util_min; @@ -628,7 +628,7 @@ change: * the entire root_domain to become SCHED_DEADLINE. We * will also fail if there's no bandwidth available. */ - if (!cpumask_subset(span, p->cpus_ptr) || + if (!cpumask_subset(span, &p->cpus_mask) || rq->rd->dl_bw.bw == 0) { retval = -EPERM; goto unlock; @@ -640,8 +640,10 @@ change: if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { policy = oldpolicy = -1; task_rq_unlock(rq, p, &rf); - if (cpuset_locked) + if (cpuset_locked) { cpuset_unlock(); + cpuset_locked = false; + } goto recheck; }