Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1376938

[PATCH tip/core/rcu 16/16] rcu: Overlap wakeups with next expedited grace period

Show all headers | View raw

The current expedited grace-period implementation makes subsequent grace
periods wait on wakeups for the prior grace period.  This does not fit
the dictionary definition of "expedited", so this commit allows these two
phases to overlap.  Doing this requires four waitqueues rather than two
because tasks can now be waiting on the previous, current, and next grace
periods.  The fourth waitqueue makes the bit masking work out nicely.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 17 ++++++++++++++---
 kernel/rcu/tree.h |  3 ++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e8fff14e417b..1df100cb7a62 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -103,6 +103,7 @@ struct rcu_state sname##_state = { \
 	.name = RCU_STATE_NAME(sname), \
 	.abbr = sabbr, \
 	.exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \
+	.exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \
 }
 
 RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
@@ -3637,7 +3638,7 @@ static bool exp_funnel_lock(struct rcu_state *rsp, unsigned long s)
 			trace_rcu_exp_funnel_lock(rsp->name, rnp->level,
 						  rnp->grplo, rnp->grphi,
 						  TPS("wait"));
-			wait_event(rnp->exp_wq[(s >> 1) & 0x1],
+			wait_event(rnp->exp_wq[(s >> 1) & 0x3],
 				   sync_exp_work_done(rsp,
 						      &rdp->exp_workdone2, s));
 			return true;
@@ -3857,6 +3858,14 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)
 	synchronize_sched_expedited_wait(rsp);
 	rcu_exp_gp_seq_end(rsp);
 	trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));
+
+	/*
+	 * Switch over to wakeup mode, allowing the next GP, but -only- the
+	 * next GP, to proceed.
+	 */
+	mutex_lock(&rsp->exp_wake_mutex);
+	mutex_unlock(&rsp->exp_mutex);
+
 	rcu_for_each_node_breadth_first(rsp, rnp) {
 		if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {
 			spin_lock(&rnp->exp_lock);
@@ -3865,10 +3874,10 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)
 				rnp->exp_seq_rq = s;
 			spin_unlock(&rnp->exp_lock);
 		}
-		wake_up_all(&rnp->exp_wq[(rsp->expedited_sequence >> 1) & 0x1]);
+		wake_up_all(&rnp->exp_wq[(rsp->expedited_sequence >> 1) & 0x3]);
 	}
 	trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));
-	mutex_unlock(&rsp->exp_mutex);
+	mutex_unlock(&rsp->exp_wake_mutex);
 }
 
 /**
@@ -4530,6 +4539,8 @@ static void __init rcu_init_one(struct rcu_state *rsp)
 			rcu_init_one_nocb(rnp);
 			init_waitqueue_head(&rnp->exp_wq[0]);
 			init_waitqueue_head(&rnp->exp_wq[1]);
+			init_waitqueue_head(&rnp->exp_wq[2]);
+			init_waitqueue_head(&rnp->exp_wq[3]);
 			spin_lock_init(&rnp->exp_lock);
 		}
 	}
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index f9d4fbb1e014..1194ab0da56a 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -250,7 +250,7 @@ struct rcu_node {
 
 	spinlock_t exp_lock ____cacheline_internodealigned_in_smp;
 	unsigned long exp_seq_rq;
-	wait_queue_head_t exp_wq[2];
+	wait_queue_head_t exp_wq[4];
 } ____cacheline_internodealigned_in_smp;
 
 /*
@@ -502,6 +502,7 @@ struct rcu_state {
 	/* End of fields guarded by barrier_mutex. */
 
 	struct mutex exp_mutex;			/* Serialize expedited GP. */
+	struct mutex exp_wake_mutex;		/* Serialize wakeup. */
 	unsigned long expedited_sequence;	/* Take a ticket. */
 	atomic_long_t expedited_normal;		/* # fallbacks to normal. */
 	atomic_t expedited_need_qs;		/* # CPUs left to check in. */
-- 
2.5.2

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[PATCH tip/core/rcu 0/16] Expedited grace-period changes for 4.7 "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:30 +0200
  [PATCH tip/core/rcu 07/16] rcu: Add expedited-grace-period event tracing "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:30 +0200
  [PATCH tip/core/rcu 02/16] rcu: Make expedited RCU-preempt stall warnings count accurately "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:30 +0200
  [PATCH tip/core/rcu 10/16] rcu: Shorten expedited_workdone* to exp_workdone* "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:30 +0200
  [PATCH tip/core/rcu 09/16] rcu: Force boolean subscript for expedited stall warnings "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:30 +0200
  [PATCH tip/core/rcu 08/16] rcu: Remove expedited GP funnel-lock bypass "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 05/16] rcu: Add event tracing definitions for expedited grace periods "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 03/16] rcu: Make cond_resched_rcu_qs() supply RCU-sched expedited QS "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 06/16] rcu: Add funnel-locking tracing for expedited grace periods "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 14/16] rcu: Consolidate expedited GP tracing into rcu_exp_gp_seq_snap() "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 01/16] rcu: Make expedited RCU-sched grace period immediately detect idle "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 15/16] rcu: Consolidate expedited GP code into exp_funnel_lock() "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 16/16] rcu: Overlap wakeups with next expedited grace period "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 12/16] rcu: Add exp_funnel_lock() fastpath "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 11/16] rcu: Enforce expedited-GP fairness via funnel wait queue "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 04/16] rcu: Fix synchronize_rcu_expedited() header comment "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200
  [PATCH tip/core/rcu 13/16] rcu: Consolidate expedited GP code into rcu_exp_wait_wake() "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2016-04-12 17:40 +0200

csiph-web