lkml.org 
[lkml]   [2010]   [Sep]   [18]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
Date
From
Subject[100/123] sched: Optimize task_rq_lock()
From: Peter Zijlstra <a.p.zijlstra@chello.nl>

commit 65cc8e4859ff29a9ddc989c88557d6059834c2a2 upstream

Now that we hold the rq->lock over set_task_cpu() again, we can do
away with most of the TASK_WAKING checks and reduce them again to
set_cpus_allowed_ptr().

Removes some conditionals from scheduling hot-paths.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Mike Galbraith <efault@gmx.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
---
kernel/sched.c | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)

--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -942,8 +942,8 @@ static inline void finish_lock_switch(st
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */

/*
- * Check whether the task is waking, we use this to synchronize against
- * ttwu() so that task_cpu() reports a stable number.
+ * Check whether the task is waking, we use this to synchronize ->cpus_allowed
+ * against ttwu().
*/
static inline int task_is_waking(struct task_struct *p)
{
@@ -960,11 +960,9 @@ static inline struct rq *__task_rq_lock(
struct rq *rq;

for (;;) {
- while (task_is_waking(p))
- cpu_relax();
rq = task_rq(p);
spin_lock(&rq->lock);
- if (likely(rq == task_rq(p) && !task_is_waking(p)))
+ if (likely(rq == task_rq(p)))
return rq;
spin_unlock(&rq->lock);
}
@@ -981,12 +979,10 @@ static struct rq *task_rq_lock(struct ta
struct rq *rq;

for (;;) {
- while (task_is_waking(p))
- cpu_relax();
local_irq_save(*flags);
rq = task_rq(p);
spin_lock(&rq->lock);
- if (likely(rq == task_rq(p) && !task_is_waking(p)))
+ if (likely(rq == task_rq(p)))
return rq;
spin_unlock_irqrestore(&rq->lock, *flags);
}
@@ -7213,7 +7209,18 @@ int set_cpus_allowed_ptr(struct task_str
struct rq *rq;
int ret = 0;

+ /*
+ * Serialize against TASK_WAKING so that ttwu() and wunt() can
+ * drop the rq->lock and still rely on ->cpus_allowed.
+ */
+again:
+ while (task_is_waking(p))
+ cpu_relax();
rq = task_rq_lock(p, &flags);
+ if (task_is_waking(p)) {
+ task_rq_unlock(rq, &flags);
+ goto again;
+ }

if (!cpumask_intersects(new_mask, cpu_active_mask)) {
ret = -EINVAL;



\
 
 \ /
  Last update: 2010-09-18 21:13    [W:0.262 / U:0.128 seconds]
©2003-2018 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site