Kyle McMartin 76a7eb
From 235d4a7af803e65d8fabdf9cd6d1445c11e2fd45 Mon Sep 17 00:00:00 2001
Kyle McMartin 76a7eb
From: Mel Gorman <mel@csn.ul.ie>
Kyle McMartin 76a7eb
Date: Wed, 24 Nov 2010 22:24:24 -0500
Kyle McMartin 76a7eb
Subject: [PATCH 2/2] mm: vmstat: Use a single setter function and callback for adjusting percpu thresholds
Kyle McMartin 76a7eb
Kyle McMartin 76a7eb
reduce_pgdat_percpu_threshold() and restore_pgdat_percpu_threshold() exist
Kyle McMartin 76a7eb
to adjust the per-cpu vmstat thresholds while kswapd is awake to avoid
Kyle McMartin 76a7eb
errors due to counter drift.  The functions duplicate some code so this
Kyle McMartin 76a7eb
patch replaces them with a single set_pgdat_percpu_threshold() that takes
Kyle McMartin 76a7eb
a callback function to calculate the desired threshold as a parameter.
Kyle McMartin 76a7eb
Kyle McMartin 76a7eb
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Kyle McMartin 76a7eb
Reviewed-by: Christoph Lameter <cl@linux.com>
Kyle McMartin 76a7eb
Reviewed-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Kyle McMartin 76a7eb
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Kyle McMartin 76a7eb
[the various mmotm patches updating this were rolled up. --kyle]
Kyle McMartin 76a7eb
---
Kyle McMartin 76a7eb
 include/linux/vmstat.h |   10 ++++++----
Kyle McMartin 76a7eb
 mm/vmscan.c            |   18 ++++++++++++++++--
Kyle McMartin 76a7eb
 mm/vmstat.c            |   36 +++++++-----------------------------
Kyle McMartin 76a7eb
 3 files changed, 29 insertions(+), 35 deletions(-)
Kyle McMartin 76a7eb
Kyle McMartin 76a7eb
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
Kyle McMartin 76a7eb
index e4cc21c..833e676 100644
Kyle McMartin 76a7eb
--- a/include/linux/vmstat.h
Kyle McMartin 76a7eb
+++ b/include/linux/vmstat.h
Kyle McMartin 76a7eb
@@ -254,8 +254,11 @@ extern void dec_zone_state(struct zone *, enum zone_stat_item);
Kyle McMartin 76a7eb
 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
 void refresh_cpu_vm_stats(int);
Kyle McMartin 76a7eb
-void reduce_pgdat_percpu_threshold(pg_data_t *pgdat);
Kyle McMartin 76a7eb
-void restore_pgdat_percpu_threshold(pg_data_t *pgdat);
Kyle McMartin 76a7eb
+
Kyle McMartin 76a7eb
+int calculate_pressure_threshold(struct zone *zone);
Kyle McMartin 76a7eb
+int calculate_normal_threshold(struct zone *zone);
Kyle McMartin 76a7eb
+void set_pgdat_percpu_threshold(pg_data_t *pgdat,
Kyle McMartin 76a7eb
+				int (*calculate_pressure)(struct zone *));
Kyle McMartin 76a7eb
 #else /* CONFIG_SMP */
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
 /*
Kyle McMartin 76a7eb
@@ -300,8 +303,7 @@ static inline void __dec_zone_page_state(struct page *page,
Kyle McMartin 76a7eb
 #define dec_zone_page_state __dec_zone_page_state
Kyle McMartin 76a7eb
 #define mod_zone_page_state __mod_zone_page_state
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-static inline void reduce_pgdat_percpu_threshold(pg_data_t *pgdat) { }
Kyle McMartin 76a7eb
-static inline void restore_pgdat_percpu_threshold(pg_data_t *pgdat) { }
Kyle McMartin 76a7eb
+#define set_pgdat_percpu_threshold(pgdat, callback) { }
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
 static inline void refresh_cpu_vm_stats(int cpu) { }
Kyle McMartin 76a7eb
 #endif
Kyle McMartin 76a7eb
diff --git a/mm/vmscan.c b/mm/vmscan.c
Kyle McMartin 76a7eb
index 18f4038..e8df269 100644
Kyle McMartin 76a7eb
--- a/mm/vmscan.c
Kyle McMartin 76a7eb
+++ b/mm/vmscan.c
Kyle McMartin 76a7eb
@@ -2327,9 +2327,23 @@ static int kswapd(void *p)
Kyle McMartin 76a7eb
 				 * to sleep until explicitly woken up
Kyle McMartin 76a7eb
 				 */
Kyle McMartin 76a7eb
 				if (!sleeping_prematurely(pgdat, order, remaining)) {
Kyle McMartin 76a7eb
-					restore_pgdat_percpu_threshold(pgdat);
Kyle McMartin 76a7eb
+					/*
Kyle McMartin 76a7eb
+					 * vmstat counters are not perfectly
Kyle McMartin 76a7eb
+					 * accurate and the estimated value
Kyle McMartin 76a7eb
+					 * for counters such as NR_FREE_PAGES
Kyle McMartin 76a7eb
+					 * can deviate from the true value by
Kyle McMartin 76a7eb
+					 * nr_online_cpus * threshold. To
Kyle McMartin 76a7eb
+					 * avoid the zone watermarks being
Kyle McMartin 76a7eb
+					 * breached while under pressure, we
Kyle McMartin 76a7eb
+					 * reduce the per-cpu vmstat threshold
Kyle McMartin 76a7eb
+					 * while kswapd is awake and restore
Kyle McMartin 76a7eb
+					 * them before going back to sleep.
Kyle McMartin 76a7eb
+					 */
Kyle McMartin 76a7eb
+					set_pgdat_percpu_threshold(pgdat,
Kyle McMartin 76a7eb
+						calculate_normal_threshold);
Kyle McMartin 76a7eb
 					schedule();
Kyle McMartin 76a7eb
-					reduce_pgdat_percpu_threshold(pgdat);
Kyle McMartin 76a7eb
+					set_pgdat_percpu_threshold(pgdat,
Kyle McMartin 76a7eb
+						calculate_pressure_threshold);
Kyle McMartin 76a7eb
 				} else {
Kyle McMartin 76a7eb
 					if (remaining)
Kyle McMartin 76a7eb
 						count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
Kyle McMartin 76a7eb
diff --git a/mm/vmstat.c b/mm/vmstat.c
Kyle McMartin 76a7eb
index 41dc8cd..5b44b82 100644
Kyle McMartin 76a7eb
--- a/mm/vmstat.c
Kyle McMartin 76a7eb
+++ b/mm/vmstat.c
Kyle McMartin 76a7eb
@@ -81,7 +81,7 @@ EXPORT_SYMBOL(vm_stat);
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
 #ifdef CONFIG_SMP
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-static int calculate_pressure_threshold(struct zone *zone)
Kyle McMartin 76a7eb
+int calculate_pressure_threshold(struct zone *zone)
Kyle McMartin 76a7eb
 {
Kyle McMartin 76a7eb
 	int threshold;
Kyle McMartin 76a7eb
 	int watermark_distance;
Kyle McMartin 76a7eb
@@ -105,7 +105,7 @@ static int calculate_pressure_threshold(struct zone *zone)
Kyle McMartin 76a7eb
 	return threshold;
Kyle McMartin 76a7eb
 }
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-static int calculate_threshold(struct zone *zone)
Kyle McMartin 76a7eb
+int calculate_normal_threshold(struct zone *zone)
Kyle McMartin 76a7eb
 {
Kyle McMartin 76a7eb
 	int threshold;
Kyle McMartin 76a7eb
 	int mem;	/* memory in 128 MB units */
Kyle McMartin 76a7eb
@@ -164,7 +164,7 @@ static void refresh_zone_stat_thresholds(void)
Kyle McMartin 76a7eb
 	for_each_populated_zone(zone) {
Kyle McMartin 76a7eb
 		unsigned long max_drift, tolerate_drift;
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-		threshold = calculate_threshold(zone);
Kyle McMartin 76a7eb
+		threshold = calculate_normal_threshold(zone);
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
 		for_each_online_cpu(cpu)
Kyle McMartin 76a7eb
 			per_cpu_ptr(zone->pageset, cpu)->stat_threshold
Kyle McMartin 76a7eb
@@ -183,46 +183,24 @@ static void refresh_zone_stat_thresholds(void)
Kyle McMartin 76a7eb
 	}
Kyle McMartin 76a7eb
 }
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-void reduce_pgdat_percpu_threshold(pg_data_t *pgdat)
Kyle McMartin 76a7eb
+void set_pgdat_percpu_threshold(pg_data_t *pgdat,
Kyle McMartin 76a7eb
+				int (*calculate_pressure)(struct zone *))
Kyle McMartin 76a7eb
 {
Kyle McMartin 76a7eb
 	struct zone *zone;
Kyle McMartin 76a7eb
 	int cpu;
Kyle McMartin 76a7eb
 	int threshold;
Kyle McMartin 76a7eb
 	int i;
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-	get_online_cpus();
Kyle McMartin 76a7eb
-	for (i = 0; i < pgdat->nr_zones; i++) {
Kyle McMartin 76a7eb
-		zone = &pgdat->node_zones[i];
Kyle McMartin 76a7eb
-		if (!zone->percpu_drift_mark)
Kyle McMartin 76a7eb
-			continue;
Kyle McMartin 76a7eb
-
Kyle McMartin 76a7eb
-		threshold = calculate_pressure_threshold(zone);
Kyle McMartin 76a7eb
-		for_each_online_cpu(cpu)
Kyle McMartin 76a7eb
-			per_cpu_ptr(zone->pageset, cpu)->stat_threshold
Kyle McMartin 76a7eb
-							= threshold;
Kyle McMartin 76a7eb
-	}
Kyle McMartin 76a7eb
-	put_online_cpus();
Kyle McMartin 76a7eb
-}
Kyle McMartin 76a7eb
-
Kyle McMartin 76a7eb
-void restore_pgdat_percpu_threshold(pg_data_t *pgdat)
Kyle McMartin 76a7eb
-{
Kyle McMartin 76a7eb
-	struct zone *zone;
Kyle McMartin 76a7eb
-	int cpu;
Kyle McMartin 76a7eb
-	int threshold;
Kyle McMartin 76a7eb
-	int i;
Kyle McMartin 76a7eb
-
Kyle McMartin 76a7eb
-	get_online_cpus();
Kyle McMartin 76a7eb
 	for (i = 0; i < pgdat->nr_zones; i++) {
Kyle McMartin 76a7eb
 		zone = &pgdat->node_zones[i];
Kyle McMartin 76a7eb
 		if (!zone->percpu_drift_mark)
Kyle McMartin 76a7eb
 			continue;
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
-		threshold = calculate_threshold(zone);
Kyle McMartin 76a7eb
-		for_each_online_cpu(cpu)
Kyle McMartin 76a7eb
+		threshold = (*calculate_pressure)(zone);
Kyle McMartin 76a7eb
+		for_each_possible_cpu(cpu)
Kyle McMartin 76a7eb
 			per_cpu_ptr(zone->pageset, cpu)->stat_threshold
Kyle McMartin 76a7eb
 							= threshold;
Kyle McMartin 76a7eb
 	}
Kyle McMartin 76a7eb
-	put_online_cpus();
Kyle McMartin 76a7eb
 }
Kyle McMartin 76a7eb
 
Kyle McMartin 76a7eb
 /*
Kyle McMartin 76a7eb
-- 
Kyle McMartin 76a7eb
1.7.3.2
Kyle McMartin 76a7eb