Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1259849
| From | Waiman Long <Waiman.Long@hpe.com> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [PATCH tip/locking/core v9 1/6] locking/qspinlock: Use _acquire/_release versions of cmpxchg & xchg |
| Date | 2015-10-31 00:30 +0100 |
| Message-ID | <qprhn-3F0-3@gated-at.bofh.it> (permalink) |
| References | <qprhn-3F0-1@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
This patch replaces the cmpxchg() and xchg() calls in the native
qspinlock code with the more relaxed _acquire or _release versions of
those calls to enable other architectures to adopt queued spinlocks
with less memory barrier performance overhead.
Signed-off-by: Waiman Long <Waiman.Long@hpe.com>
---
include/asm-generic/qspinlock.h | 9 +++++----
kernel/locking/qspinlock.c | 29 ++++++++++++++++++++++++-----
2 files changed, 29 insertions(+), 9 deletions(-)
diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h
index e2aadbc..39e1cb2 100644
--- a/include/asm-generic/qspinlock.h
+++ b/include/asm-generic/qspinlock.h
@@ -12,8 +12,9 @@
* GNU General Public License for more details.
*
* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
+ * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
*
- * Authors: Waiman Long <waiman.long@hp.com>
+ * Authors: Waiman Long <waiman.long@hpe.com>
*/
#ifndef __ASM_GENERIC_QSPINLOCK_H
#define __ASM_GENERIC_QSPINLOCK_H
@@ -62,7 +63,7 @@ static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
{
if (!atomic_read(&lock->val) &&
- (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) == 0))
+ (atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL) == 0))
return 1;
return 0;
}
@@ -77,7 +78,7 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock)
{
u32 val;
- val = atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL);
+ val = atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL);
if (likely(val == 0))
return;
queued_spin_lock_slowpath(lock, val);
@@ -93,7 +94,7 @@ static __always_inline void queued_spin_unlock(struct qspinlock *lock)
/*
* smp_mb__before_atomic() in order to guarantee release semantics
*/
- smp_mb__before_atomic_dec();
+ smp_mb__before_atomic();
atomic_sub(_Q_LOCKED_VAL, &lock->val);
}
#endif
diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c
index 87e9ce6..7868418 100644
--- a/kernel/locking/qspinlock.c
+++ b/kernel/locking/qspinlock.c
@@ -14,8 +14,9 @@
* (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
* (C) Copyright 2013-2014 Red Hat, Inc.
* (C) Copyright 2015 Intel Corp.
+ * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
*
- * Authors: Waiman Long <waiman.long@hp.com>
+ * Authors: Waiman Long <waiman.long@hpe.com>
* Peter Zijlstra <peterz@infradead.org>
*/
@@ -176,7 +177,12 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
{
struct __qspinlock *l = (void *)lock;
- return (u32)xchg(&l->tail, tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
+ /*
+ * Use release semantics to make sure that the MCS node is properly
+ * initialized before changing the tail code.
+ */
+ return (u32)xchg_release(&l->tail,
+ tail >> _Q_TAIL_OFFSET) << _Q_TAIL_OFFSET;
}
#else /* _Q_PENDING_BITS == 8 */
@@ -208,7 +214,11 @@ static __always_inline u32 xchg_tail(struct qspinlock *lock, u32 tail)
for (;;) {
new = (val & _Q_LOCKED_PENDING_MASK) | tail;
- old = atomic_cmpxchg(&lock->val, val, new);
+ /*
+ * Use release semantics to make sure that the MCS node is
+ * properly initialized before changing the tail code.
+ */
+ old = atomic_cmpxchg_release(&lock->val, val, new);
if (old == val)
break;
@@ -319,7 +329,11 @@ void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
if (val == new)
new |= _Q_PENDING_VAL;
- old = atomic_cmpxchg(&lock->val, val, new);
+ /*
+ * Acquire semantic is required here as the function may
+ * return immediately if the lock was free.
+ */
+ old = atomic_cmpxchg_acquire(&lock->val, val, new);
if (old == val)
break;
@@ -426,7 +440,12 @@ queue:
set_locked(lock);
break;
}
- old = atomic_cmpxchg(&lock->val, val, _Q_LOCKED_VAL);
+ /*
+ * The smp_load_acquire() call above has provided the necessary
+ * acquire semantics required for locking. At most two
+ * iterations of this loop may be ran.
+ */
+ old = atomic_cmpxchg_relaxed(&lock->val, val, _Q_LOCKED_VAL);
if (old == val)
goto release; /* No contention */
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[PATCH tip/locking/core v9 0/6] locking/qspinlock: Enhance pvqspinlock Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
[PATCH tip/locking/core v9 1/6] locking/qspinlock: Use _acquire/_release versions of cmpxchg & xchg Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
[PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Peter Zijlstra <peterz@infradead.org> - 2015-11-02 17:40 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Peter Zijlstra <peterz@infradead.org> - 2015-11-03 00:00 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Waiman Long <waiman.long@hpe.com> - 2015-11-05 17:50 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Peter Zijlstra <peterz@infradead.org> - 2015-11-05 18:00 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Waiman Long <waiman.long@hpe.com> - 2015-11-05 17:10 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Peter Zijlstra <peterz@infradead.org> - 2015-11-05 17:40 +0100
Re: [PATCH tip/locking/core v9 2/6] locking/qspinlock: prefetch next node cacheline Waiman Long <waiman.long@hpe.com> - 2015-11-05 18:00 +0100
[PATCH tip/locking/core v9 5/6] locking/pvqspinlock: Allow 1 lock stealing attempt Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
Re: [PATCH tip/locking/core v9 5/6] locking/pvqspinlock: Allow 1 lock stealing attempt Peter Zijlstra <peterz@infradead.org> - 2015-11-06 16:00 +0100
Re: [PATCH tip/locking/core v9 5/6] locking/pvqspinlock: Allow 1 lock stealing attempt Waiman Long <waiman.long@hpe.com> - 2015-11-06 18:50 +0100
Re: [PATCH tip/locking/core v9 5/6] locking/pvqspinlock: Allow 1 lock stealing attempt Peter Zijlstra <peterz@infradead.org> - 2015-11-09 18:40 +0100
Re: [PATCH tip/locking/core v9 5/6] locking/pvqspinlock: Allow 1 lock stealing attempt Waiman Long <waiman.long@hpe.com> - 2015-11-09 21:00 +0100
[PATCH tip/locking/core v9 6/6] locking/pvqspinlock: Queue node adaptive spinning Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
Re: [PATCH tip/locking/core v9 6/6] locking/pvqspinlock: Queue node adaptive spinning Peter Zijlstra <peterz@infradead.org> - 2015-11-06 16:10 +0100
Re: [PATCH tip/locking/core v9 6/6] locking/pvqspinlock: Queue node adaptive spinning Waiman Long <waiman.long@hpe.com> - 2015-11-06 19:00 +0100
Re: [PATCH tip/locking/core v9 6/6] locking/pvqspinlock: Queue node adaptive spinning Peter Zijlstra <peterz@infradead.org> - 2015-11-06 21:40 +0100
Re: [PATCH tip/locking/core v9 6/6] locking/pvqspinlock: Queue node adaptive spinning Waiman Long <waiman.long@hpe.com> - 2015-11-09 18:00 +0100
Re: [PATCH tip/locking/core v9 6/6] locking/pvqspinlock: Queue node adaptive spinning Peter Zijlstra <peterz@infradead.org> - 2015-11-09 18:40 +0100
[PATCH tip/locking/core v9 3/6] locking/pvqspinlock, x86: Optimize PV unlock code path Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
[PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Waiman Long <Waiman.Long@hpe.com> - 2015-10-31 00:30 +0100
Re: [PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Peter Zijlstra <peterz@infradead.org> - 2015-11-02 17:50 +0100
Re: [PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Waiman Long <waiman.long@hpe.com> - 2015-11-05 17:30 +0100
Re: [PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Peter Zijlstra <peterz@infradead.org> - 2015-11-05 17:50 +0100
Re: [PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Waiman Long <waiman.long@hpe.com> - 2015-11-05 18:00 +0100
Re: [PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Peter Zijlstra <peterz@infradead.org> - 2015-11-05 18:10 +0100
Re: [PATCH tip/locking/core v9 4/6] locking/pvqspinlock: Collect slowpath lock statistics Waiman Long <waiman.long@hpe.com> - 2015-11-05 18:40 +0100
csiph-web