lkml.org 
[lkml]   [2012]   [Mar]   [27]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[PATCH 1/2] Avoid lock contention on page draining
Date
From: Andi Kleen <ak@linux.intel.com>

drain_all_pages asks all CPUs to drain their PCP lists. This causes a lot
of lock contention because they try to free into the same zones in lock
step.

Make half of the CPUs go through the zones forwards and the other half
backwards. This should lower the contention to half.

I opencoded the backwards walk: there were no macros for it, but it seemed
to obscure to create some extra for this.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
mm/page_alloc.c | 56 +++++++++++++++++++++++++++++++++++++++++-------------
1 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a13ded1..8cd4f6a 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1124,6 +1124,23 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
}
#endif

+static void do_drain_zone(struct zone *zone, int cpu)
+{
+ unsigned long flags;
+ struct per_cpu_pageset *pset;
+ struct per_cpu_pages *pcp;
+
+ local_irq_save(flags);
+ pset = per_cpu_ptr(zone->pageset, cpu);
+
+ pcp = &pset->pcp;
+ if (pcp->count) {
+ free_pcppages_bulk(zone, pcp->count, pcp);
+ pcp->count = 0;
+ }
+ local_irq_restore(flags);
+}
+
/*
* Drain pages of the indicated processor.
*
@@ -1133,22 +1150,33 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
*/
static void drain_pages(unsigned int cpu)
{
- unsigned long flags;
struct zone *zone;

- for_each_populated_zone(zone) {
- struct per_cpu_pageset *pset;
- struct per_cpu_pages *pcp;
-
- local_irq_save(flags);
- pset = per_cpu_ptr(zone->pageset, cpu);
-
- pcp = &pset->pcp;
- if (pcp->count) {
- free_pcppages_bulk(zone, pcp->count, pcp);
- pcp->count = 0;
- }
- local_irq_restore(flags);
+ /*
+ * Let half of the CPUs go through the zones forwards
+ * and the other half backwards. This reduces lock contention.
+ */
+ if ((cpu % 2) == 0) {
+ for_each_populated_zone(zone)
+ do_drain_zone(zone, cpu);
+ } else {
+ int i, j, k = 0;
+
+ /*
+ * Backwards zone walk. Opencoded because its quite obscure.
+ */
+ for (i = MAX_NUMNODES - 1; i >= 0; i--) {
+ if (!node_states[N_ONLINE].bits[i / BITS_PER_LONG]) {
+ i -= i % BITS_PER_LONG;
+ continue;
+ }
+ if (!node_isset(i, node_states[N_ONLINE]))
+ continue;
+ k++;
+ for (j = MAX_NR_ZONES - 1; j >= 0; j--)
+ do_drain_zone(&NODE_DATA(i)->node_zones[j], cpu);
+ }
+ WARN_ON(k != num_online_nodes());
}
}

--
1.7.7.6


\
 
 \ /
  Last update: 2012-03-28 02:43    [W:0.026 / U:7.240 seconds]
©2003-2018 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site