Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1206039

[tip:locking/core] locking/qrwlock: Make use of _{acquire|release|relaxed}() atomics

From tip-bot for Will Deacon <tipbot@zytor.com>
Newsgroups linux.kernel
Subject [tip:locking/core] locking/qrwlock: Make use of _{acquire|release|relaxed}() atomics
Date 2015-08-12 14:40 +0200
Message-ID <pWDu4-4ta-63@gated-at.bofh.it> (permalink)
References <pUwGm-6w0-15@gated-at.bofh.it>
Organization linux.* mail to news gateway

Show all headers | View raw

Commit-ID:  77e430e3e45662b696dc49aa53ea0f7ac63f2574
Gitweb:     http://git.kernel.org/tip/77e430e3e45662b696dc49aa53ea0f7ac63f2574
Author:     Will Deacon <will.deacon@arm.com>
AuthorDate: Thu, 6 Aug 2015 17:54:42 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 12 Aug 2015 11:59:06 +0200

locking/qrwlock: Make use of _{acquire|release|relaxed}() atomics

The qrwlock implementation is slightly heavy in its use of memory
barriers, mainly through the use of _cmpxchg() and _return() atomics, which
imply full barrier semantics.

This patch modifies the qrwlock code to use the more relaxed atomic
routines so that we can reduce the unnecessary barrier overhead on
weakly-ordered architectures.

Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Waiman.Long@hp.com
Cc: paulmck@linux.vnet.ibm.com
Link: http://lkml.kernel.org/r/1438880084-18856-7-git-send-email-will.deacon@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 include/asm-generic/qrwlock.h | 13 ++++++-------
 kernel/locking/qrwlock.c      | 24 ++++++++++++------------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h
index eb673dd..54a8e65 100644
--- a/include/asm-generic/qrwlock.h
+++ b/include/asm-generic/qrwlock.h
@@ -68,7 +68,7 @@ static inline int queued_read_trylock(struct qrwlock *lock)
 
 	cnts = atomic_read(&lock->cnts);
 	if (likely(!(cnts & _QW_WMASK))) {
-		cnts = (u32)atomic_add_return(_QR_BIAS, &lock->cnts);
+		cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
 		if (likely(!(cnts & _QW_WMASK)))
 			return 1;
 		atomic_sub(_QR_BIAS, &lock->cnts);
@@ -89,8 +89,8 @@ static inline int queued_write_trylock(struct qrwlock *lock)
 	if (unlikely(cnts))
 		return 0;
 
-	return likely(atomic_cmpxchg(&lock->cnts,
-				     cnts, cnts | _QW_LOCKED) == cnts);
+	return likely(atomic_cmpxchg_acquire(&lock->cnts,
+					     cnts, cnts | _QW_LOCKED) == cnts);
 }
 /**
  * queued_read_lock - acquire read lock of a queue rwlock
@@ -100,7 +100,7 @@ static inline void queued_read_lock(struct qrwlock *lock)
 {
 	u32 cnts;
 
-	cnts = atomic_add_return(_QR_BIAS, &lock->cnts);
+	cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
 	if (likely(!(cnts & _QW_WMASK)))
 		return;
 
@@ -115,7 +115,7 @@ static inline void queued_read_lock(struct qrwlock *lock)
 static inline void queued_write_lock(struct qrwlock *lock)
 {
 	/* Optimize for the unfair lock case where the fair flag is 0. */
-	if (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0)
+	if (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)
 		return;
 
 	queued_write_lock_slowpath(lock);
@@ -130,8 +130,7 @@ static inline void queued_read_unlock(struct qrwlock *lock)
 	/*
 	 * Atomically decrement the reader count
 	 */
-	smp_mb__before_atomic();
-	atomic_sub(_QR_BIAS, &lock->cnts);
+	(void)atomic_sub_return_release(_QR_BIAS, &lock->cnts);
 }
 
 /**
diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index 6a7a3b8..f17a3e3 100644
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -55,7 +55,7 @@ rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
 {
 	while ((cnts & _QW_WMASK) == _QW_LOCKED) {
 		cpu_relax_lowlatency();
-		cnts = smp_load_acquire((u32 *)&lock->cnts);
+		cnts = atomic_read_acquire(&lock->cnts);
 	}
 }
 
@@ -74,8 +74,9 @@ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts)
 		 * Readers in interrupt context will get the lock immediately
 		 * if the writer is just waiting (not holding the lock yet).
 		 * The rspin_until_writer_unlock() function returns immediately
-		 * in this case. Otherwise, they will spin until the lock
-		 * is available without waiting in the queue.
+		 * in this case. Otherwise, they will spin (with ACQUIRE
+		 * semantics) until the lock is available without waiting in
+		 * the queue.
 		 */
 		rspin_until_writer_unlock(lock, cnts);
 		return;
@@ -88,12 +89,11 @@ void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts)
 	arch_spin_lock(&lock->lock);
 
 	/*
-	 * At the head of the wait queue now, increment the reader count
-	 * and wait until the writer, if it has the lock, has gone away.
-	 * At ths stage, it is not possible for a writer to remain in the
-	 * waiting state (_QW_WAITING). So there won't be any deadlock.
+	 * The ACQUIRE semantics of the following spinning code ensure
+	 * that accesses can't leak upwards out of our subsequent critical
+	 * section in the case that the lock is currently held for write.
 	 */
-	cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS;
+	cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts) - _QR_BIAS;
 	rspin_until_writer_unlock(lock, cnts);
 
 	/*
@@ -116,7 +116,7 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
 
 	/* Try to acquire the lock directly if no reader is present */
 	if (!atomic_read(&lock->cnts) &&
-	    (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0))
+	    (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
 		goto unlock;
 
 	/*
@@ -127,7 +127,7 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
 		struct __qrwlock *l = (struct __qrwlock *)lock;
 
 		if (!READ_ONCE(l->wmode) &&
-		   (cmpxchg(&l->wmode, 0, _QW_WAITING) == 0))
+		   (cmpxchg_relaxed(&l->wmode, 0, _QW_WAITING) == 0))
 			break;
 
 		cpu_relax_lowlatency();
@@ -137,8 +137,8 @@ void queued_write_lock_slowpath(struct qrwlock *lock)
 	for (;;) {
 		cnts = atomic_read(&lock->cnts);
 		if ((cnts == _QW_WAITING) &&
-		    (atomic_cmpxchg(&lock->cnts, _QW_WAITING,
-				    _QW_LOCKED) == _QW_WAITING))
+		    (atomic_cmpxchg_acquire(&lock->cnts, _QW_WAITING,
+					    _QW_LOCKED) == _QW_WAITING))
 			break;
 
 		cpu_relax_lowlatency();
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[PATCH v5 0/8] Add generic support for relaxed atomics Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
  [PATCH v5 2/8] asm-generic: rework atomic-long.h to avoid bulk code duplication Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking, asm-generic:   Rework atomic-long.h to avoid bulk code duplication tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  [PATCH v5 8/8] ARM: atomics: define our SMP atomics in terms of _relaxed operations Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking, ARM, atomics:   Define our SMP atomics in terms of _relaxed() operations tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  [PATCH v5 5/8] locking/qrwlock: implement queue_write_unlock using smp_store_release Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking/qrwlock: Implement queue_write_unlock(  ) using smp_store_release() tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  [PATCH v5 3/8] asm-generic: add relaxed/acquire/release variants for atomic_long_t Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking, asm-generic:   Add _{relaxed|acquire|release}() variants for 'atomic_long_t' tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  [PATCH v5 1/8] atomics: add acquire/release/relaxed variants of some atomic operations Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking/atomics:   Add _{acquire|release|relaxed}() variants of some atomic operations tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  [PATCH v5 6/8] locking/qrwlock: make use of acquire/release/relaxed atomics Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking/qrwlock:   Make use of _{acquire|release|relaxed}() atomics tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  [PATCH v5 7/8] include/llist: use linux/atomic.h instead of asm/cmpxchg.h Will Deacon <will.deacon@arm.com> - 2015-08-06 19:00 +0200
    [tip:locking/core] locking, include/llist: Use linux/  atomic.h instead of asm/cmpxchg.h tip-bot for Will Deacon <tipbot@zytor.com> - 2015-08-12 14:40 +0200
  Re: [PATCH v5 0/8] Add generic support for relaxed atomics Peter Zijlstra <peterz@infradead.org> - 2015-08-07 17:20 +0200
    Re: [PATCH v5 0/8] Add generic support for relaxed atomics Will Deacon <will.deacon@arm.com> - 2015-08-07 18:30 +0200

csiph-web