lkml.org 
[lkml]   [2009]   [Nov]   [22]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
Subject[patch] sched: fix set_task_cpu() and provide an unlocked runqueue variant
From
Date

sched: fix set_task_cpu() and provide an unlocked runqueue variant.

set_task_cpu() falsifies migration stats by unconditionally generating migration
stats whether a task's cpu actually changed or not. As used in copy_process(),
the runqueue is unlocked, so we need to provide an unlocked variant which does
the locking to provide a write barrier.

Signed-off-by: Mike Galbraith <efault@gmx.de>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>

---
include/linux/sched.h | 5 +++++
kernel/fork.c | 2 +-
kernel/sched.c | 38 ++++++++++++++++++++++++--------------
3 files changed, 30 insertions(+), 15 deletions(-)

Index: linux-2.6/kernel/sched.c
===================================================================
--- linux-2.6.orig/kernel/sched.c
+++ linux-2.6/kernel/sched.c
@@ -2056,7 +2056,6 @@ task_hot(struct task_struct *p, u64 now,
return delta < (s64)sysctl_sched_migration_cost;
}

-
void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
{
int old_cpu = task_cpu(p);
@@ -2065,10 +2064,10 @@ void set_task_cpu(struct task_struct *p,
*new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
u64 clock_offset;

- clock_offset = old_rq->clock - new_rq->clock;
-
- trace_sched_migrate_task(p, new_cpu);
+ if (unlikely(old_cpu == new_cpu))
+ goto out;

+ clock_offset = old_rq->clock - new_rq->clock;
#ifdef CONFIG_SCHEDSTATS
if (p->se.wait_start)
p->se.wait_start -= clock_offset;
@@ -2076,22 +2075,33 @@ void set_task_cpu(struct task_struct *p,
p->se.sleep_start -= clock_offset;
if (p->se.block_start)
p->se.block_start -= clock_offset;
+ if (task_hot(p, old_rq->clock, NULL))
+ schedstat_inc(p, se.nr_forced2_migrations);
#endif
- if (old_cpu != new_cpu) {
- p->se.nr_migrations++;
-#ifdef CONFIG_SCHEDSTATS
- if (task_hot(p, old_rq->clock, NULL))
- schedstat_inc(p, se.nr_forced2_migrations);
-#endif
- perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS,
- 1, 1, NULL, 0);
- }
p->se.vruntime -= old_cfsrq->min_vruntime -
- new_cfsrq->min_vruntime;
+ new_cfsrq->min_vruntime;
+ p->se.nr_migrations++;
+ trace_sched_migrate_task(p, new_cpu);
+ perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);

+out:
__set_task_cpu(p, new_cpu);
}

+void set_task_cpu_unlocked(struct task_struct *p, unsigned int new_cpu)
+{
+ unsigned long flags;
+ struct rq *rq, *new_rq = cpu_rq(new_cpu);
+
+ smp_wmb();
+ rq = task_rq_lock(p, &flags);
+ update_rq_clock(rq);
+ if (rq != new_rq)
+ update_rq_clock(new_rq);
+ set_task_cpu(p, new_cpu);
+ task_rq_unlock(rq, &flags);
+}
+
struct migration_req {
struct list_head list;

Index: linux-2.6/include/linux/sched.h
===================================================================
--- linux-2.6.orig/include/linux/sched.h
+++ linux-2.6/include/linux/sched.h
@@ -2457,6 +2457,7 @@ static inline unsigned int task_cpu(cons
}

extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
+extern void set_task_cpu_unlocked(struct task_struct *p, unsigned int cpu);

#else

@@ -2469,6 +2470,10 @@ static inline void set_task_cpu(struct t
{
}

+static inline void set_task_cpu_unlocked(struct task_struct *p, unsigned int cpu)
+{
+}
+
#endif /* CONFIG_SMP */

extern void arch_pick_mmap_layout(struct mm_struct *mm);
Index: linux-2.6/kernel/fork.c
===================================================================
--- linux-2.6.orig/kernel/fork.c
+++ linux-2.6/kernel/fork.c
@@ -1242,7 +1242,7 @@ static struct task_struct *copy_process(
p->rt.nr_cpus_allowed = current->rt.nr_cpus_allowed;
if (unlikely(!cpu_isset(task_cpu(p), p->cpus_allowed) ||
!cpu_online(task_cpu(p))))
- set_task_cpu(p, smp_processor_id());
+ set_task_cpu_unlocked(p, smp_processor_id());

/* CLONE_PARENT re-uses the old parent */
if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {



\
 
 \ /
  Last update: 2009-11-22 13:11    [W:0.066 / U:0.244 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site