Messages in this thread Patch in this message | | | From | Vincent Guittot <> | Subject | [PATCH 6/7 v2] sched/fair: trigger the update of blocked load on newly idle cpu | Date | Fri, 12 Feb 2021 14:28:48 +0100 |
| |
Instead of waking up a random and already idle CPU, we can take advantage of this_cpu being about to enter idle to run the ILB and update the blocked load.
Signed-off-by: Vincent Guittot <vincent.guittot@linaro.org> --- kernel/sched/fair.c | 20 +++++++++++++++++--- kernel/sched/idle.c | 6 ++++++ kernel/sched/sched.h | 3 +++ 3 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5d285d93e433..d11a632ea276 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10453,6 +10453,20 @@ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) return true; } +void nohz_run_idle_balance(int cpu) +{ + unsigned int flags; + + flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); + + if (flags && !need_resched()) { + struct rq *rq = cpu_rq(cpu); + + rq->nohz_idle_balance = flags; + nohz_idle_balance(rq, CPU_IDLE); + } +} + static void nohz_newidle_balance(struct rq *this_rq) { int this_cpu = this_rq->cpu; @@ -10474,10 +10488,10 @@ static void nohz_newidle_balance(struct rq *this_rq) return; /* - * Blocked load of idle CPUs need to be updated. - * Kick an ILB to update statistics. + * Set the need to trigger ILB in order to update blocked load + * before entering idle state. */ - kick_ilb(NOHZ_STATS_KICK); + atomic_or(NOHZ_STATS_KICK, nohz_flags(this_cpu)); } #else /* !CONFIG_NO_HZ_COMMON */ diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 305727ea0677..52a4e9ce2f9b 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -261,6 +261,12 @@ static void cpuidle_idle_call(void) static void do_idle(void) { int cpu = smp_processor_id(); + + /* + * Check if we need to update some blocked load + */ + nohz_run_idle_balance(cpu); + /* * If the arch has a polling bit, we maintain an invariant: * diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6edc67df3554..a11ec9110992 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2370,8 +2370,11 @@ extern void cfs_bandwidth_usage_dec(void); #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) extern void nohz_balance_exit_idle(struct rq *rq); +extern void nohz_run_idle_balance(int cpu); #else + static inline void nohz_balance_exit_idle(struct rq *rq) { } +static inline void nohz_run_idle_balance(int cpu) { } #endif -- 2.17.1
| |