Messages in this thread Patch in this message | | | From | Ingo Molnar <> | Subject | [PATCH 09/89] sched/wake_q: Move the wake-queue types and interfaces from sched.h into <linux/sched/wake_q.h> | Date | Mon, 6 Feb 2017 14:28:12 +0100 |
| |
Note that this requires the basic task->wake_q type to be an opaque void *.
Cc: Peter Zijlstra <peterz@infradead.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org> --- include/linux/sched.h | 52 +--------------------------------------------------- include/linux/sched/wake_q.h | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ ipc/mqueue.c | 1 + ipc/msg.c | 1 + ipc/sem.c | 1 + kernel/fork.c | 2 +- kernel/futex.c | 1 + kernel/locking/mutex.c | 1 + kernel/locking/rtmutex.c | 1 + kernel/locking/rtmutex_common.h | 1 + kernel/locking/rwsem-xadd.c | 3 ++- kernel/sched/core.c | 6 +++--- kernel/sched/sched.h | 1 + 13 files changed, 69 insertions(+), 56 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h index ebc11bb42c49..e973a0f5550c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -961,56 +961,6 @@ void force_schedstat_enabled(void); # define SCHED_FIXEDPOINT_SHIFT 10 # define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT) -/* - * Wake-queues are lists of tasks with a pending wakeup, whose - * callers have already marked the task as woken internally, - * and can thus carry on. A common use case is being able to - * do the wakeups once the corresponding user lock as been - * released. - * - * We hold reference to each task in the list across the wakeup, - * thus guaranteeing that the memory is still valid by the time - * the actual wakeups are performed in wake_up_q(). - * - * One per task suffices, because there's never a need for a task to be - * in two wake queues simultaneously; it is forbidden to abandon a task - * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is - * already in a wake queue, the wakeup will happen soon and the second - * waker can just skip it. - * - * The DEFINE_WAKE_Q macro declares and initializes the list head. - * wake_up_q() does NOT reinitialize the list; it's expected to be - * called near the end of a function. Otherwise, the list can be - * re-initialized for later re-use by wake_q_init(). - * - * Note that this can cause spurious wakeups. schedule() callers - * must ensure the call is done inside a loop, confirming that the - * wakeup condition has in fact occurred. - */ -struct wake_q_node { - struct wake_q_node *next; -}; - -struct wake_q_head { - struct wake_q_node *first; - struct wake_q_node **lastp; -}; - -#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) - -#define DEFINE_WAKE_Q(name) \ - struct wake_q_head name = { WAKE_Q_TAIL, &name.first } - -static inline void wake_q_init(struct wake_q_head *head) -{ - head->first = WAKE_Q_TAIL; - head->lastp = &head->first; -} - -extern void wake_q_add(struct wake_q_head *head, - struct task_struct *task); -extern void wake_up_q(struct wake_q_head *head); - struct io_context; /* See blkdev.h */ @@ -1507,7 +1457,7 @@ struct task_struct { /* Protection of the PI data structures: */ raw_spinlock_t pi_lock; - struct wake_q_node wake_q; + void *wake_q; #ifdef CONFIG_RT_MUTEXES /* PI waiters blocked on a rt_mutex held by this task */ diff --git a/include/linux/sched/wake_q.h b/include/linux/sched/wake_q.h new file mode 100644 index 000000000000..9a32f17b6a5e --- /dev/null +++ b/include/linux/sched/wake_q.h @@ -0,0 +1,54 @@ +#ifndef _LINUX_SCHED_WAKE_Q_H +#define _LINUX_SCHED_WAKE_Q_H + +/* + * Wake-queues are lists of tasks with a pending wakeup, whose + * callers have already marked the task as woken internally, + * and can thus carry on. A common use case is being able to + * do the wakeups once the corresponding user lock as been + * released. + * + * We hold reference to each task in the list across the wakeup, + * thus guaranteeing that the memory is still valid by the time + * the actual wakeups are performed in wake_up_q(). + * + * One per task suffices, because there's never a need for a task to be + * in two wake queues simultaneously; it is forbidden to abandon a task + * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is + * already in a wake queue, the wakeup will happen soon and the second + * waker can just skip it. + * + * The DEFINE_WAKE_Q macro declares and initializes the list head. + * wake_up_q() does NOT reinitialize the list; it's expected to be + * called near the end of a function. Otherwise, the list can be + * re-initialized for later re-use by wake_q_init(). + * + * Note that this can cause spurious wakeups. schedule() callers + * must ensure the call is done inside a loop, confirming that the + * wakeup condition has in fact occurred. + */ +struct wake_q_node { + struct wake_q_node *next; +}; + +struct wake_q_head { + struct wake_q_node *first; + struct wake_q_node **lastp; +}; + +#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) + +#define DEFINE_WAKE_Q(name) \ + struct wake_q_head name = { WAKE_Q_TAIL, &name.first } + +static inline void wake_q_init(struct wake_q_head *head) +{ + head->first = WAKE_Q_TAIL; + head->lastp = &head->first; +} + +extern void wake_q_add(struct wake_q_head *head, + struct task_struct *task); +extern void wake_up_q(struct wake_q_head *head); + +#endif /* _LINUX_SCHED_WAKE_Q_H */ diff --git a/ipc/mqueue.c b/ipc/mqueue.c index 7a2d8f0c8ae5..a604f1036067 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -35,6 +35,7 @@ #include <linux/ipc_namespace.h> #include <linux/user_namespace.h> #include <linux/slab.h> +#include <linux/sched/wake_q.h> #include <net/sock.h> #include "util.h" diff --git a/ipc/msg.c b/ipc/msg.c index e3e52ce01123..ecc387e573f6 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -31,6 +31,7 @@ #include <linux/list.h> #include <linux/security.h> #include <linux/sched.h> +#include <linux/sched/wake_q.h> #include <linux/syscalls.h> #include <linux/audit.h> #include <linux/seq_file.h> diff --git a/ipc/sem.c b/ipc/sem.c index 3ec5742b5640..c65dff526a9f 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -82,6 +82,7 @@ #include <linux/rwsem.h> #include <linux/nsproxy.h> #include <linux/ipc_namespace.h> +#include <linux/sched/wake_q.h> #include <linux/uaccess.h> #include "util.h" diff --git a/kernel/fork.c b/kernel/fork.c index 077ce3b4c0ef..9aa2ca28a76c 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -538,7 +538,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif tsk->splice_pipe = NULL; tsk->task_frag.page = NULL; - tsk->wake_q.next = NULL; + tsk->wake_q = NULL; account_kernel_stack(tsk, 1); diff --git a/kernel/futex.c b/kernel/futex.c index 0842c8ca534b..e41472f33c73 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -61,6 +61,7 @@ #include <linux/nsproxy.h> #include <linux/ptrace.h> #include <linux/sched/rt.h> +#include <linux/sched/wake_q.h> #include <linux/hugetlb.h> #include <linux/freezer.h> #include <linux/bootmem.h> diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index ad2d9e22697b..57f6311e2405 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -21,6 +21,7 @@ #include <linux/ww_mutex.h> #include <linux/sched.h> #include <linux/sched/rt.h> +#include <linux/sched/wake_q.h> #include <linux/export.h> #include <linux/spinlock.h> #include <linux/interrupt.h> diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index d340be3a488f..d4f798491361 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -15,6 +15,7 @@ #include <linux/sched.h> #include <linux/sched/rt.h> #include <linux/sched/deadline.h> +#include <linux/sched/wake_q.h> #include <linux/timer.h> #include "rtmutex_common.h" diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h index 990134617b4c..856dfff5c33a 100644 --- a/kernel/locking/rtmutex_common.h +++ b/kernel/locking/rtmutex_common.h @@ -13,6 +13,7 @@ #define __KERNEL_RTMUTEX_COMMON_H #include <linux/rtmutex.h> +#include <linux/sched/wake_q.h> /* * This is the control structure for tasks blocked on a rt_mutex, diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c index 2ad8d8dc3bb1..4fe8d8ad4396 100644 --- a/kernel/locking/rwsem-xadd.c +++ b/kernel/locking/rwsem-xadd.c @@ -10,10 +10,11 @@ * and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes. */ #include <linux/rwsem.h> -#include <linux/sched.h> #include <linux/init.h> #include <linux/export.h> +#include <linux/sched.h> #include <linux/sched/rt.h> +#include <linux/sched/wake_q.h> #include <linux/osq_lock.h> #include "rwsem.h" diff --git a/kernel/sched/core.c b/kernel/sched/core.c index e4aa470ed454..fed9625fa954 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -426,7 +426,7 @@ static bool set_nr_if_polling(struct task_struct *p) void wake_q_add(struct wake_q_head *head, struct task_struct *task) { - struct wake_q_node *node = &task->wake_q; + struct wake_q_node *node = (void *)&task->wake_q; /* * Atomically grab the task, if ->wake_q is !nil already it means @@ -455,11 +455,11 @@ void wake_up_q(struct wake_q_head *head) while (node != WAKE_Q_TAIL) { struct task_struct *task; - task = container_of(node, struct task_struct, wake_q); + task = container_of((void *)node, struct task_struct, wake_q); BUG_ON(!task); /* Task can safely be re-inserted now: */ node = node->next; - task->wake_q.next = NULL; + task->wake_q = NULL; /* * wake_up_process() implies a wmb() to pair with the queueing diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 319fcf80930c..ebb66cdeff70 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3,6 +3,7 @@ #include <linux/sched/sysctl.h> #include <linux/sched/topology.h> #include <linux/sched/rt.h> +#include <linux/sched/wake_q.h> #include <linux/u64_stats_sync.h> #include <linux/sched/deadline.h> #include <linux/kernel_stat.h> -- 2.7.4
| |