lkml.org 
[lkml]   [2017]   [Dec]   [21]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[RFC PATCH 2/5] sched: Add NOHZ_STATS_KICK

    Suggested-by: Vincent Guittot <vincent.guittot@linaro.org>
    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    ---
    kernel/sched/core.c | 4 +--
    kernel/sched/fair.c | 52 ++++++++++++++++++++++++++++++++++-----------------
    kernel/sched/sched.h | 4 +++
    3 files changed, 41 insertions(+), 19 deletions(-)

    --- a/kernel/sched/core.c
    +++ b/kernel/sched/core.c
    @@ -608,7 +608,7 @@ static inline bool got_nohz_idle_kick(vo
    {
    int cpu = smp_processor_id();

    - if (!(atomic_read(nohz_flags(cpu)) & NOHZ_BALANCE_KICK))
    + if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
    return false;

    if (idle_cpu(cpu) && !need_resched())
    @@ -618,7 +618,7 @@ static inline bool got_nohz_idle_kick(vo
    * We can't run Idle Load Balance on this CPU for this time so we
    * cancel it and clear NOHZ_BALANCE_KICK
    */
    - atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(cpu));
    + atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
    return false;
    }

    --- a/kernel/sched/fair.c
    +++ b/kernel/sched/fair.c
    @@ -9001,8 +9001,8 @@ static void nohz_balancer_kick(void)
    if (ilb_cpu >= nr_cpu_ids)
    return;

    - flags = atomic_fetch_or(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu));
    - if (flags & NOHZ_BALANCE_KICK)
    + flags = atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(ilb_cpu));
    + if (flags & NOHZ_KICK_MASK)
    return;
    /*
    * Use smp_send_reschedule() instead of resched_cpu().
    @@ -9125,8 +9125,6 @@ static void rebalance_domains(struct rq
    int need_serialize, need_decay = 0;
    u64 max_cost = 0;

    - update_blocked_averages(cpu);
    -
    rcu_read_lock();
    for_each_domain(cpu, sd) {
    /*
    @@ -9221,20 +9219,27 @@ static void rebalance_domains(struct rq
    * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
    * rebalancing for all the cpus for whom scheduler ticks are stopped.
    */
    -static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
    +static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
    {
    - int this_cpu = this_rq->cpu;
    - struct rq *rq;
    - int balance_cpu;
    /* Earliest time when we have to do rebalance again */
    unsigned long next_balance = jiffies + 60*HZ;
    int update_next_balance = 0;
    + int this_cpu = this_rq->cpu;
    + unsigned int flags;
    + int balance_cpu;
    + struct rq *rq;

    - if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_BALANCE_KICK))
    - return;
    + if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK))
    + return false;

    - if (idle != CPU_IDLE)
    - goto end;
    + if (idle != CPU_IDLE) {
    + atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
    + return false;
    + }
    +
    + flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
    +
    + SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);

    for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
    if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
    @@ -9262,7 +9267,9 @@ static void nohz_idle_balance(struct rq
    cpu_load_update_idle(rq);
    rq_unlock_irq(rq, &rf);

    - rebalance_domains(rq, CPU_IDLE);
    + update_blocked_averages(rq->cpu);
    + if (flags & NOHZ_BALANCE_KICK)
    + rebalance_domains(rq, CPU_IDLE);
    }

    if (time_after(next_balance, rq->next_balance)) {
    @@ -9271,6 +9278,10 @@ static void nohz_idle_balance(struct rq
    }
    }

    + update_blocked_averages(this_cpu);
    + if (flags & NOHZ_BALANCE_KICK)
    + rebalance_domains(this_rq, CPU_IDLE);
    +
    /*
    * next_balance will be updated only when there is a need.
    * When the CPU is attached to null domain for ex, it will not be
    @@ -9278,8 +9289,8 @@ static void nohz_idle_balance(struct rq
    */
    if (likely(update_next_balance))
    nohz.next_balance = next_balance;
    -end:
    - atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));
    +
    + return true;
    }

    /*
    @@ -9366,7 +9377,10 @@ static inline bool nohz_kick_needed(stru
    return kick;
    }
    #else
    -static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { }
    +static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
    +{
    + return false;
    +}
    #endif

    /*
    @@ -9387,7 +9401,11 @@ static __latent_entropy void run_rebalan
    * load balance only within the local sched_domain hierarchy
    * and abort nohz_idle_balance altogether if we pull some load.
    */
    - nohz_idle_balance(this_rq, idle);
    + if (nohz_idle_balance(this_rq, idle))
    + return;
    +
    + /* normal load balance */
    + update_blocked_averages(this_rq->cpu);
    rebalance_domains(this_rq, idle);
    }

    --- a/kernel/sched/sched.h
    +++ b/kernel/sched/sched.h
    @@ -2005,9 +2005,13 @@ extern void cfs_bandwidth_usage_dec(void
    #ifdef CONFIG_NO_HZ_COMMON
    #define NOHZ_TICK_STOPPED_BIT 0
    #define NOHZ_BALANCE_KICK_BIT 1
    +#define NOHZ_STATS_KICK_BIT 2

    #define NOHZ_TICK_STOPPED BIT(NOHZ_TICK_STOPPED_BIT)
    #define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
    +#define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT)
    +
    +#define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)

    #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)


    \
     
     \ /
      Last update: 2017-12-21 11:25    [W:3.358 / U:0.008 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site