lkml.org 
[lkml]   [2011]   [Mar]   [2]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[PATCH 16/22] sched: Remove rq->lock from the first half of ttwu()
    Currently ttwu() does two rq->lock acquisitions, once on the task's
    old rq, holding it over the p->state fiddling and load-balance pass.
    Then it drops the old rq->lock to acquire the new rq->lock.

    By having serialized ttwu(), p->sched_class, p->cpus_allowed with
    p->pi_lock, we can now drop the whole first rq->lock acquisition.

    The p->pi_lock serializing concurrent ttwu() calls protects p->state,
    which we will set to TASK_WAKING to bridge possible p->pi_lock to
    rq->lock gaps and serialize set_task_cpu() calls against
    task_rq_lock().

    The p->pi_lock serialization of p->sched_class allows us to call
    scheduling class methods without holding the rq->lock, and the
    serialization of p->cpus_allowed allows us to do the load-balancing
    bits without races.

    Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
    LKML-Reference: <new-submission>
    ---
    kernel/sched.c | 63 +++++++++++++++++++++++++++++++--------------------------
    1 file changed, 35 insertions(+), 28 deletions(-)

    Index: linux-2.6/kernel/sched.c
    ===================================================================
    --- linux-2.6.orig/kernel/sched.c
    +++ linux-2.6/kernel/sched.c
    @@ -2486,69 +2486,76 @@ ttwu_post_activation(struct task_struct
    * Returns %true if @p was woken up, %false if it was already running
    * or @state didn't match @p's state.
    */
    -static int try_to_wake_up(struct task_struct *p, unsigned int state,
    - int wake_flags)
    +static int
    +try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
    {
    - int cpu, orig_cpu, this_cpu, success = 0;
    + int cpu, this_cpu, success = 0;
    unsigned long flags;
    - unsigned long en_flags = ENQUEUE_WAKEUP;
    struct rq *rq;

    this_cpu = get_cpu();

    smp_wmb();
    raw_spin_lock_irqsave(&p->pi_lock, flags);
    - rq = __task_rq_lock(p);
    if (!(p->state & state))
    goto out;

    cpu = task_cpu(p);

    - if (p->on_rq)
    - goto out_running;
    + if (p->on_rq) {
    + rq = __task_rq_lock(p);
    + if (p->on_rq)
    + goto out_running;
    + __task_rq_unlock(rq);
    + }

    - orig_cpu = cpu;
    #ifdef CONFIG_SMP
    - if (unlikely(task_running(rq, p)))
    - goto out_activate;
    + while (p->on_cpu) {
    +#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
    + /*
    + * If called from interrupt context we could have landed in the
    + * middle of schedule(), in this case we should take care not
    + * to spin on ->on_cpu if p is current, since that would
    + * deadlock.
    + */
    + if (p == current)
    + goto out_activate;
    +#endif
    + cpu_relax();
    + }
    + /*
    + * Pairs with the smp_wmb() in finish_lock_switch().
    + */
    + smp_rmb();

    p->sched_contributes_to_load = !!task_contributes_to_load(p);
    p->state = TASK_WAKING;

    - if (p->sched_class->task_waking) {
    + if (p->sched_class->task_waking)
    p->sched_class->task_waking(p);
    - en_flags |= ENQUEUE_WAKING;
    - }

    cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
    - if (cpu != orig_cpu)
    - set_task_cpu(p, cpu);
    - __task_rq_unlock(rq);
    +out_activate:
    +#endif /* CONFIG_SMP */

    rq = cpu_rq(cpu);
    raw_spin_lock(&rq->lock);

    - /*
    - * We migrated the task without holding either rq->lock, however
    - * since the task is not on the task list itself, nobody else
    - * will try and migrate the task, hence the rq should match the
    - * cpu we just moved it to.
    - */
    - WARN_ON(task_cpu(p) != cpu);
    - WARN_ON(p->state != TASK_WAKING);
    +#ifdef CONFIG_SMP
    + if (cpu != task_cpu(p))
    + set_task_cpu(p, cpu);

    if (p->sched_contributes_to_load)
    rq->nr_uninterruptible--;
    +#endif

    -out_activate:
    -#endif /* CONFIG_SMP */
    - activate_task(rq, p, en_flags);
    + activate_task(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
    out_running:
    ttwu_post_activation(p, rq, wake_flags);
    ttwu_stat(rq, p, cpu, wake_flags);
    success = 1;
    -out:
    __task_rq_unlock(rq);
    +out:
    raw_spin_unlock_irqrestore(&p->pi_lock, flags);
    put_cpu();




    \
     
     \ /
      Last update: 2011-03-02 18:53    [W:0.026 / U:32.424 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site