Messages in this thread Patch in this message | | | From | NeilBrown <> | Date | Fri, 22 Dec 2017 14:11:04 +1100 | Subject | [PATCH 2/2] sched/wait: add wait_event_idle_exclusive_lifo() |
| |
wait_event_*_exclusive() adds new waiters to the end of the quest, while non-exclusive wait_event adds to the head.
This ensures that a wake_up will wake all non-exclusive waiters and at most one exclusive wait, but it means that exclusive waiters are woken in a FIFO order, so the task woken is the one least likely to have data in the CPU cache.
When simple interaction with non-exclusive waiters is not important, and when choosing a cache-hot task is, the new
wait_event_idle_exclusive_lifo() and wait_event_idle_exclusive_lifo_timeout()
can be used. To implement these we introduce a new WQ_FLAG_LIFO which causes prepare_to_wait_event() to add to the head of the queue.
This will be used to allow lustre's l_wait_event() to be replaced with more standard wait.h macros.
Signed-off-by: NeilBrown <neilb@suse.com> --- include/linux/wait.h | 95 +++++++++++++++++++++++++++++++++++++++++++++++--- kernel/sched/wait.c | 3 +- 2 files changed, 91 insertions(+), 7 deletions(-)
diff --git a/include/linux/wait.h b/include/linux/wait.h index 3aea0780c9d0..49cb393c53d5 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -20,6 +20,9 @@ int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int #define WQ_FLAG_EXCLUSIVE 0x01 #define WQ_FLAG_WOKEN 0x02 #define WQ_FLAG_BOOKMARK 0x04 +#define WQ_FLAG_LIFO 0x08 /* used with WQ_FLAG_EXCLUSIVE to force + * LIFO scheduling in prepare_to_wait_event(). + */ /* * A single wait-queue entry structure: @@ -247,7 +250,7 @@ extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags); struct wait_queue_entry __wq_entry; \ long __ret = ret; /* explicit shadow */ \ \ - init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \ + init_wait_entry(&__wq_entry, exclusive); \ for (;;) { \ long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\ \ @@ -381,7 +384,8 @@ do { \ }) #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \ - (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \ + (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, \ + WQ_FLAG_EXCLUSIVE, 0, \ cmd1; schedule(); cmd2) /* * Just like wait_event_cmd(), except it sets exclusive flag @@ -558,7 +562,7 @@ do { \ }) #define __wait_event_interruptible_exclusive(wq, condition) \ - ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \ + ___wait_event(wq, condition, TASK_INTERRUPTIBLE, WQ_FLAG_EXCLUSIVE, 0, \ schedule()) #define wait_event_interruptible_exclusive(wq, condition) \ @@ -571,7 +575,7 @@ do { \ }) #define __wait_event_killable_exclusive(wq, condition) \ - ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \ + ___wait_event(wq, condition, TASK_KILLABLE, WQ_FLAG_EXCLUSIVE, 0, \ schedule()) #define wait_event_killable_exclusive(wq, condition) \ @@ -585,7 +589,7 @@ do { \ #define __wait_event_freezable_exclusive(wq, condition) \ - ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \ + ___wait_event(wq, condition, TASK_INTERRUPTIBLE, WQ_FLAG_EXCLUSIVE, 0, \ schedule(); try_to_freeze()) #define wait_event_freezable_exclusive(wq, condition) \ @@ -638,9 +642,88 @@ do { \ do { \ might_sleep(); \ if (!(condition)) \ - ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \ + ___wait_event(wq_head, condition, TASK_IDLE, WQ_FLAG_EXCLUSIVE, \ + 0, schedule()); \ } while (0) +/** + * wait_event_idle_exclusive_lifo - wait for a condition without contributing to system load + * @wq_head: the waitqueue to wait on + * @condition: a C expression for the event to wait for + * + * The process is put to sleep (TASK_IDLE) until the + * @condition evaluates to true. + * The @condition is checked each time the waitqueue @wq_head is woken up. + * + * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag + * set thus when other process waits process on the list if this + * process is awaken further processes are not considered. + * + * Contrary to the usual practice with exclusive wait, this call adds + * the task to the head of the queue so that tasks are woken in a + * LIFO (rather than FIFO) order. This means that if both exclusive and + * non-exclusive waiter are waiting on the same queue, the non-exclusive + * waiters may *not* be woken on the next wakeup event. The benefit + * of using LIFO waits is that when multiple worker threads are + * available, the one with the warmest cache will preferentially + * be woken. + * + * wake_up() has to be called after changing any variable that could + * change the result of the wait condition. + * + */ +#define wait_event_idle_exclusive_lifo(wq_head, condition) \ +do { \ + might_sleep(); \ + if (!(condition)) \ + ___wait_event(wq_head, condition, TASK_IDLE, \ + WQ_FLAG_EXCLUSIVE | WQ_FLAG_LIFO, \ + 0, schedule()); \ +} while (0) + +/** + * wait_event_idle_exclusive_lifo_timeout - wait for a condition with timeout, without contributing to system load + * @wq_head: the waitqueue to wait on + * @condition: a C expression for the event to wait for + * @timeout: timeout, in jiffies + * + * The process is put to sleep (TASK_IDLE) until the + * @condition evaluates to true. + * The @condition is checked each time the waitqueue @wq_head is woken up. + * + * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag + * set thus when other process waits process on the list if this + * process is awaken further processes are not considered. + * + * Contrary to the usual practice with exclusive wait, this call adds + * the task to the head of the queue so that tasks are woken in a + * LIFO (rather than FIFO) order. This means that if both exclusive and + * non-exclusive waiter are waiting on the same queue, the non-exclusive + * waiters may *not* be woken on the next wakeup event. The benefit + * of using LIFO waits is that when multiple worker threads are + * available, the one with the warmest cache will preferentially + * be woken. + * + * wake_up() has to be called after changing any variable that could + * change the result of the wait condition. + * + * Returns: + * 0 if the @condition evaluated to %false after the @timeout elapsed, + * 1 if the @condition evaluated to %true after the @timeout elapsed, + * or the remaining jiffies (at least 1) if the @condition evaluated + * to %true before the @timeout elapsed. + */ +#define wait_event_idle_exclusive_lifo_timeout(wq_head, condition, timeout) \ +({ \ + long __ret = timeout; \ + might_sleep(); \ + if (!___wait_cond_timeout(condition)) \ + __ret = ___wait_event(wq_head, ___wait_cond_timeout(condition), TASK_IDLE, \ + WQ_FLAG_EXCLUSIVE | WQ_FLAG_LIFO, \ + timeout, __ret = schedule_timeout(__ret)); \ + __ret; \ +}) + #define __wait_event_idle_timeout(wq_head, condition, timeout) \ ___wait_event(wq_head, ___wait_cond_timeout(condition), \ TASK_IDLE, 0, timeout, \ diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 929ecb7d6b78..a92f368acbb0 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -285,7 +285,8 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_en ret = -ERESTARTSYS; } else { if (list_empty(&wq_entry->entry)) { - if (wq_entry->flags & WQ_FLAG_EXCLUSIVE) + if ((wq_entry->flags & (WQ_FLAG_EXCLUSIVE | WQ_FLAG_LIFO)) == + WQ_FLAG_EXCLUSIVE) __add_wait_queue_entry_tail(wq_head, wq_entry); else __add_wait_queue(wq_head, wq_entry);
| |