Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1470335 > unrolled thread
| Started by | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| First post | 2016-08-25 20:50 +0200 |
| Last post | 2016-08-30 14:00 +0200 |
| Articles | 7 — 1 participant |
Back to article view | Back to linux.kernel
[RFC][PATCH -v2 0/4] locking/mutex: Rewrite basic mutex Peter Zijlstra <peterz@infradead.org> - 2016-08-25 20:50 +0200
[RFC][PATCH -v2 3/4] locking/mutex: Allow MUTEX_SPIN_ON_OWNER when DEBUG_MUTEXES Peter Zijlstra <peterz@infradead.org> - 2016-08-25 20:50 +0200
[RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation Peter Zijlstra <peterz@infradead.org> - 2016-08-25 20:50 +0200
Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation Peter Zijlstra <peterz@infradead.org> - 2016-08-26 00:30 +0200
Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation Peter Zijlstra <peterz@infradead.org> - 2016-08-26 17:20 +0200
Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation Peter Zijlstra <peterz@infradead.org> - 2016-08-29 17:50 +0200
Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation Peter Zijlstra <peterz@infradead.org> - 2016-08-30 14:00 +0200
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-25 20:50 +0200 |
| Subject | [RFC][PATCH -v2 0/4] locking/mutex: Rewrite basic mutex |
| Message-ID | <sa7SV-4G3-9@gated-at.bofh.it> |
... might still eat your pets and set your house on fire ... Passes reaim high_systime on 4 socket haswell.. 4.8.0-rc3-00185-g9f55477 +2 +all Forks Jobs/min Jobs/min Jobs/min 10 1040.00 1039.83 1039.83 20 1600.00 1599.79 1600.00 30 2127.27 2127.27 2127.03 40 2599.73 2599.46 2599.19 50 3088.50 3088.50 3088.50 60 3531.08 3531.08 3531.41 70 3969.83 3969.47 3969.83 80 4377.80 4377.80 4377.80 90 4797.95 4798.77 4798.77 100 5198.27 5198.27 5198.70 200 9038.89 9038.89 9040.20 300 12642.67 12643.52 12643.52 400 15970.31 15993.85 15994.87 500 19365.65 19339.24 19368.06 600 22537.92 22542.00 22513.53 700 25536.39 25653.37 25552.83 800 28673.18 28664.94 28671.53 900 31596.71 31685.85 31700.16 1000 34651.27 34643.57 34649.34 1100 37679.09 37693.57 37693.57 1200 40675.76 40675.76 40175.98 1300 43573.08 43575.42 42993.43 1400 46398.98 46406.37 46440.91 1500 48977.03 49203.60 48432.16 1600 51940.48 51970.23 51037.73 1700 54929.58 54646.61 54923.89 1800 57432.12 57538.04 57529.19 1900 60437.38 60434.30 60446.62 2000 62950.82 62973.05 61760.78 Still need to add spinning to the top waiter etc... lockdep is still broken and there might still be races. --- arch/alpha/include/asm/mutex.h | 9 - arch/arc/include/asm/mutex.h | 18 -- arch/arm/include/asm/mutex.h | 21 -- arch/arm64/include/asm/Kbuild | 1 - arch/avr32/include/asm/mutex.h | 9 - arch/blackfin/include/asm/Kbuild | 1 - arch/c6x/include/asm/mutex.h | 6 - arch/cris/include/asm/mutex.h | 9 - arch/frv/include/asm/mutex.h | 9 - arch/h8300/include/asm/mutex.h | 9 - arch/hexagon/include/asm/mutex.h | 8 - arch/ia64/include/asm/mutex.h | 90 -------- arch/m32r/include/asm/mutex.h | 9 - arch/m68k/include/asm/Kbuild | 1 - arch/metag/include/asm/Kbuild | 1 - arch/microblaze/include/asm/mutex.h | 1 - arch/mips/include/asm/Kbuild | 1 - arch/mn10300/include/asm/mutex.h | 16 -- arch/nios2/include/asm/mutex.h | 1 - arch/openrisc/include/asm/mutex.h | 27 --- arch/parisc/include/asm/Kbuild | 1 - arch/powerpc/include/asm/mutex.h | 132 ------------ arch/s390/include/asm/mutex.h | 9 - arch/score/include/asm/mutex.h | 6 - arch/sh/include/asm/mutex-llsc.h | 109 ---------- arch/sh/include/asm/mutex.h | 12 -- arch/sparc/include/asm/Kbuild | 1 - arch/tile/include/asm/Kbuild | 1 - arch/um/include/asm/Kbuild | 1 - arch/unicore32/include/asm/mutex.h | 20 -- arch/x86/include/asm/mutex.h | 5 - arch/x86/include/asm/mutex_32.h | 110 ---------- arch/x86/include/asm/mutex_64.h | 127 ----------- arch/xtensa/include/asm/mutex.h | 9 - drivers/gpu/drm/i915/i915_gem_shrinker.c | 26 +-- include/asm-generic/mutex-dec.h | 88 -------- include/asm-generic/mutex-null.h | 19 -- include/asm-generic/mutex-xchg.h | 120 ----------- include/asm-generic/mutex.h | 9 - include/linux/mutex-debug.h | 24 --- include/linux/mutex.h | 46 ++-- kernel/Kconfig.locks | 2 +- kernel/locking/mutex-debug.c | 13 -- kernel/locking/mutex-debug.h | 10 - kernel/locking/mutex.c | 347 +++++++++++++++---------------- kernel/locking/mutex.h | 26 --- kernel/sched/core.c | 2 +- 47 files changed, 199 insertions(+), 1323 deletions(-)
[toc] | [next] | [standalone]
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-25 20:50 +0200 |
| Subject | [RFC][PATCH -v2 3/4] locking/mutex: Allow MUTEX_SPIN_ON_OWNER when DEBUG_MUTEXES |
| Message-ID | <sa7SV-4G3-19@gated-at.bofh.it> |
| In reply to | #1470335 |
Now that mutex::count and mutex::owner are the same field, we can
allow SPIN_ON_OWNER while DEBUG_MUTEX.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
kernel/Kconfig.locks | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--- a/kernel/Kconfig.locks
+++ b/kernel/Kconfig.locks
@@ -225,7 +225,7 @@ config ARCH_SUPPORTS_ATOMIC_RMW
config MUTEX_SPIN_ON_OWNER
def_bool y
- depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW
+ depends on SMP && ARCH_SUPPORTS_ATOMIC_RMW
config RWSEM_SPIN_ON_OWNER
def_bool y
[toc] | [prev] | [next] | [standalone]
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-25 20:50 +0200 |
| Subject | [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation |
| Message-ID | <sa7SW-4G3-27@gated-at.bofh.it> |
| In reply to | #1470335 |
Now that we have an atomic owner field, we can do explicit lock
handoff. Use this to avoid starvation.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
kernel/locking/mutex.c | 52 +++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 46 insertions(+), 6 deletions(-)
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -52,6 +52,7 @@ __mutex_init(struct mutex *lock, const c
EXPORT_SYMBOL(__mutex_init);
#define MUTEX_FLAG_WAITERS 0x01
+#define MUTEX_FLAG_HANDOFF 0x02
#define MUTEX_FLAGS 0x03
@@ -117,6 +118,33 @@ static inline void __mutex_clear_flag(st
atomic_long_andnot(flag, &lock->owner);
}
+static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
+{
+ return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
+}
+
+static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
+{
+ unsigned long owner = atomic_long_read(&lock->owner);
+
+ for (;;) {
+ unsigned long old, new;
+
+#ifdef CONFIG_DEBUG_MUTEXES
+ DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
+#endif
+
+ new = (owner & MUTEX_FLAG_WAITERS);
+ new |= (unsigned long)task;
+
+ old = atomic_long_cmpxchg(&lock->owner, owner, new);
+ if (old == owner)
+ break;
+
+ owner = old;
+ }
+}
+
#ifndef CONFIG_DEBUG_LOCK_ALLOC
/*
* We split the mutex lock/unlock logic into separate fastpath and
@@ -447,7 +475,7 @@ static bool mutex_optimistic_spin(struct
}
#endif
-static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock);
+static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long owner);
/**
* mutex_unlock - release the mutex
@@ -468,9 +496,12 @@ void __sched mutex_unlock(struct mutex *
DEBUG_LOCKS_WARN_ON(__mutex_owner(lock) != current);
#endif
- owner = atomic_long_fetch_and(MUTEX_FLAGS, &lock->owner);
+ owner = atomic_long_read(&lock->owner);
+ if (!(owner & MUTEX_FLAG_HANDOFF))
+ owner = atomic_long_fetch_and(MUTEX_FLAGS, &lock->owner);
+
if (__owner_flags(owner))
- __mutex_unlock_slowpath(lock);
+ __mutex_unlock_slowpath(lock, owner);
}
EXPORT_SYMBOL(mutex_unlock);
@@ -568,7 +599,7 @@ __mutex_lock_common(struct mutex *lock,
list_add_tail(&waiter.list, &lock->wait_list);
waiter.task = task;
- if (list_first_entry(&lock->wait_list, struct mutex_waiter, list) == &waiter) {
+ if (__mutex_waiter_is_first(lock, &waiter)) {
__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
/*
* We must be sure to set WAITERS before attempting the trylock
@@ -605,13 +636,16 @@ __mutex_lock_common(struct mutex *lock,
spin_unlock_mutex(&lock->wait_lock, flags);
schedule_preempt_disabled();
spin_lock_mutex(&lock->wait_lock, flags);
+
+ if (__mutex_waiter_is_first(lock, &waiter))
+ __mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
}
__set_task_state(task, TASK_RUNNING);
mutex_remove_waiter(lock, &waiter, task);
/* set it to 0 if there are no waiters left: */
if (likely(list_empty(&lock->wait_list)))
- __mutex_clear_flag(lock, MUTEX_FLAG_WAITERS);
+ __mutex_clear_flag(lock, MUTEX_FLAGS);
debug_mutex_free_waiter(&waiter);
@@ -737,8 +771,9 @@ EXPORT_SYMBOL_GPL(__ww_mutex_lock_interr
/*
* Release the lock, slowpath:
*/
-static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock)
+static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long owner)
{
+ struct task_struct *next = NULL;
unsigned long flags;
WAKE_Q(wake_q);
@@ -752,10 +787,15 @@ static noinline void __sched __mutex_unl
list_entry(lock->wait_list.next,
struct mutex_waiter, list);
+ next = waiter->task;
+
debug_mutex_wake_waiter(lock, waiter);
wake_q_add(&wake_q, waiter->task);
}
+ if (owner & MUTEX_FLAG_HANDOFF)
+ __mutex_handoff(lock, next);
+
spin_unlock_mutex(&lock->wait_lock, flags);
wake_up_q(&wake_q);
}
[toc] | [prev] | [next] | [standalone]
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-26 00:30 +0200 |
| Subject | Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation |
| Message-ID | <sabjP-72N-3@gated-at.bofh.it> |
| In reply to | #1470337 |
On Thu, Aug 25, 2016 at 06:00:19PM -0400, Waiman Long wrote: > On 08/25/2016 02:37 PM, Peter Zijlstra wrote: > >@@ -468,9 +496,12 @@ void __sched mutex_unlock(struct mutex * > > DEBUG_LOCKS_WARN_ON(__mutex_owner(lock) != current); > > #endif > > > >- owner = atomic_long_fetch_and(MUTEX_FLAGS,&lock->owner); > >+ owner = atomic_long_read(&lock->owner); > >+ if (!(owner& MUTEX_FLAG_HANDOFF)) > >+ owner = atomic_long_fetch_and(MUTEX_FLAGS,&lock->owner); > >+ > > if (__owner_flags(owner)) > >- __mutex_unlock_slowpath(lock); > >+ __mutex_unlock_slowpath(lock, owner); > > } > > EXPORT_SYMBOL(mutex_unlock); > > I don't think the race condition is fixed when we don't make sure that lock > handoff only happens from current=>new. The problem is due to the fact that > the MUTEX_FLAG_HANDOFF check in the unlock fastpath isn't serialized by the > wait_lock. As a result, it is possible that the owner is NULL while the > HANDOFF bit is set. Or an optimistic spinner may have stolen the lock in the > interim. You're right.. I'll try again tomorrow.
[toc] | [prev] | [next] | [standalone]
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-26 17:20 +0200 |
| Subject | Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation |
| Message-ID | <sar5f-nz-1@gated-at.bofh.it> |
| In reply to | #1470468 |
On Fri, Aug 26, 2016 at 10:30:31AM -0400, Waiman Long wrote: > On 08/25/2016 06:23 PM, Peter Zijlstra wrote: > >You're right.. I'll try again tomorrow. > > Actually, it is also OK to handoff from NULL=>new, but nothing else is > appropriate. I pushed out new patches to git, could you have a peek? - killed another of that mutex abuse - fixed ATOMIC_FETCH_OP for SH4 - split: rm arch/*/include/asm/mutex*.h include/asm-generic/mutex*.h into a separate patch to aid in review - fixed lockdep - added an uncontended unlock fast path, to mirror the unconteded lock fast path - reworked the handoff stuff (again) I didn't try and be too clever, since its far too hot to think. Still need to look at adding spinning to the handoff case. Also need to look at writing (much) better changelogs, they stink.
[toc] | [prev] | [next] | [standalone]
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-29 17:50 +0200 |
| Subject | Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation |
| Message-ID | <sbwYV-EU-13@gated-at.bofh.it> |
| In reply to | #1470806 |
On Fri, Aug 26, 2016 at 07:40:34PM -0400, Waiman Long wrote:
> On 08/26/2016 11:18 AM, Peter Zijlstra wrote:
> >Still need to look at adding spinning to the handoff case.
> >Also need to look at writing (much) better changelogs, they stink.
> >
>
> I have looked at the handoff code and I didn't see any problem.
So I found (or rather the buildbot did) a problem with it.
locking-selftest has testcases like:
lock(&A);
if (trylock(&A))
/* fail */
and
ww_lock(&A)
if (ww_lock(&A) != -EDEADLK)
/* fail */
But with the 'trylock' accepting the lock if owner==current, in order to
accept the hand-off, this breaks in interesting ways.
Now, ARCH_MIN_TASKALIGN is at least 8 (mips, s390, parisc) which would
give us one more FLAG bit to play with.
The below seems to make things happy again..
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -53,8 +53,9 @@ EXPORT_SYMBOL(__mutex_init);
#define MUTEX_FLAG_WAITERS 0x01
#define MUTEX_FLAG_HANDOFF 0x02
+#define MUTEX_FLAG_GIFT 0x04
-#define MUTEX_FLAGS 0x03
+#define MUTEX_FLAGS 0x07
static inline struct task_struct *__owner_task(unsigned long owner)
{
@@ -66,33 +67,6 @@ static inline unsigned long __owner_flag
return owner & MUTEX_FLAGS;
}
-/*
- * Actual trylock that will work on any unlocked state.
- */
-static inline bool __mutex_trylock(struct mutex *lock)
-{
- unsigned long owner, curr = (unsigned long)current;
-
- owner = atomic_long_read(&lock->owner);
- for (;;) { /* must loop, can race against a flag */
- unsigned long old;
-
- if (__owner_task(owner)) {
- if ((unsigned long)__owner_task(owner) == curr)
- return true;
-
- return false;
- }
-
- old = atomic_long_cmpxchg_acquire(&lock->owner, owner,
- curr | __owner_flags(owner));
- if (old == owner)
- return true;
-
- owner = old;
- }
-}
-
#ifndef CONFIG_DEBUG_LOCK_ALLOC
/*
* Optimistic trylock that only works in the uncontended case. Make sure to
@@ -134,6 +108,10 @@ static inline bool __mutex_waiter_is_fir
return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
}
+/*
+ * Give up ownership to a specific task, when @task = NULL, this is equivalent
+ * to a regular unlock.
+ */
static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
{
unsigned long owner = atomic_long_read(&lock->owner);
@@ -146,7 +124,15 @@ static void __mutex_handoff(struct mutex
#endif
new = (owner & MUTEX_FLAG_WAITERS);
- new |= (unsigned long)task;
+ if (task) {
+ /*
+ * To distinguish between a recursive lock attempt
+ * and having been given the lock by someone else
+ * we need to set the GIFT bit.
+ */
+ new |= MUTEX_FLAG_GIFT; /* clear HANDOFF, set GIFT */
+ new |= (unsigned long)task;
+ }
old = atomic_long_cmpxchg(&lock->owner, owner, new);
if (old == owner)
@@ -154,6 +140,50 @@ static void __mutex_handoff(struct mutex
owner = old;
}
+}
+
+/*
+ * Someone handed us its lock ownership using __mutex_handoff(), say thank you
+ * and accept this nice gift.
+ */
+static bool __mutex_accept(struct mutex *lock, unsigned long owner)
+{
+ if (!(owner & MUTEX_FLAG_GIFT))
+ return false;
+
+ if (__owner_task(owner) != current)
+ return false;
+
+ __mutex_clear_flag(lock, MUTEX_FLAG_GIFT);
+ smp_mb__after_atomic(); /* ACQUIRE */
+ return true;
+}
+
+/*
+ * Actual trylock that will work on any unlocked state.
+ */
+static inline bool __mutex_trylock(struct mutex *lock)
+{
+ unsigned long owner, curr = (unsigned long)current;
+
+ owner = atomic_long_read(&lock->owner);
+ for (;;) { /* must loop, can race against a flag */
+ unsigned long old;
+
+ if (__owner_task(owner)) {
+ if (unlikely(owner & MUTEX_FLAG_GIFT))
+ return __mutex_accept(lock, owner);
+
+ return false;
+ }
+
+ old = atomic_long_cmpxchg_acquire(&lock->owner, owner,
+ curr | __owner_flags(owner));
+ if (old == owner)
+ return true;
+
+ owner = old;
+ }
}
#ifndef CONFIG_DEBUG_LOCK_ALLOC
[toc] | [prev] | [next] | [standalone]
| From | Peter Zijlstra <peterz@infradead.org> |
|---|---|
| Date | 2016-08-30 14:00 +0200 |
| Subject | Re: [RFC][PATCH -v2 4/4] locking/mutex: Add lock handoff to avoid starvation |
| Message-ID | <sbPRT-4hC-9@gated-at.bofh.it> |
| In reply to | #1471939 |
On Mon, Aug 29, 2016 at 05:41:09PM +0200, Peter Zijlstra wrote:
> On Fri, Aug 26, 2016 at 07:40:34PM -0400, Waiman Long wrote:
> > On 08/26/2016 11:18 AM, Peter Zijlstra wrote:
>
> > >Still need to look at adding spinning to the handoff case.
> > >Also need to look at writing (much) better changelogs, they stink.
> > >
> >
> > I have looked at the handoff code and I didn't see any problem.
>
> So I found (or rather the buildbot did) a problem with it.
>
> locking-selftest has testcases like:
>
>
> lock(&A);
> if (trylock(&A))
> /* fail */
>
> and
>
> ww_lock(&A)
> if (ww_lock(&A) != -EDEADLK)
> /* fail */
>
> But with the 'trylock' accepting the lock if owner==current, in order to
> accept the hand-off, this breaks in interesting ways.
>
> Now, ARCH_MIN_TASKALIGN is at least 8 (mips, s390, parisc) which would
> give us one more FLAG bit to play with.
>
>
> The below seems to make things happy again..
Much simpler solution... only accept handoffs when we're stuck in the
wait loop (which precludes doing recursive locking, since that would've
failed much earlier).
Now, let me look at that spinner patch you sent.
---
--- kernel/locking/mutex.c.mod 2016-08-30 11:08:15.410551744 +0200
+++ kernel/locking/mutex.c 2016-08-30 13:38:30.185550669 +0200
@@ -69,7 +69,7 @@
/*
* Actual trylock that will work on any unlocked state.
*/
-static inline bool __mutex_trylock(struct mutex *lock)
+static inline bool __mutex_trylock(struct mutex *lock, const bool handoff)
{
unsigned long owner, curr = (unsigned long)current;
@@ -78,8 +78,10 @@
unsigned long old;
if (__owner_task(owner)) {
- if ((unsigned long)__owner_task(owner) == curr)
+ if (handoff && unlikely(__owner_task(owner) == current)) {
+ smp_mb(); /* ACQUIRE */
return true;
+ }
return false;
}
@@ -134,6 +136,10 @@
return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
}
+/*
+ * Give up ownership to a specific task, when @task = NULL, this is equivalent
+ * to a regular unlock.
+ */
static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
{
unsigned long owner = atomic_long_read(&lock->owner);
@@ -148,7 +154,7 @@
new = (owner & MUTEX_FLAG_WAITERS);
new |= (unsigned long)task;
- old = atomic_long_cmpxchg(&lock->owner, owner, new);
+ old = atomic_long_cmpxchg_release(&lock->owner, owner, new);
if (old == owner)
break;
@@ -425,7 +431,7 @@
break;
/* Try to acquire the mutex if it is unlocked. */
- if (__mutex_trylock(lock)) {
+ if (__mutex_trylock(lock, false)) {
osq_unlock(&lock->osq);
return true;
}
@@ -570,7 +576,12 @@
preempt_disable();
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
- if (__mutex_trylock(lock) || mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
+ /*
+ * The first __mutex_trylock() must not accept handoffs, otherwise its
+ * possible to allow recursive lock attempts by accident.
+ */
+ if (__mutex_trylock(lock, false) ||
+ mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
/* got the lock, yay! */
lock_acquired(&lock->dep_map, ip);
if (use_ww_ctx) {
@@ -588,7 +599,7 @@
/*
* Once more, try to acquire the lock.
*/
- if (__mutex_trylock(lock))
+ if (__mutex_trylock(lock, false))
goto skip_wait;
debug_mutex_lock_common(lock, &waiter);
@@ -601,7 +612,7 @@
if (__mutex_waiter_is_first(lock, &waiter))
__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
- if (__mutex_trylock(lock))
+ if (__mutex_trylock(lock, false))
goto remove_waiter;
lock_contended(&lock->dep_map, ip);
@@ -629,7 +640,7 @@
schedule_preempt_disabled();
spin_lock_mutex(&lock->wait_lock, flags);
- if (__mutex_trylock(lock))
+ if (__mutex_trylock(lock, true))
break;
if (__mutex_waiter_is_first(lock, &waiter))
@@ -923,7 +934,7 @@
*/
int __sched mutex_trylock(struct mutex *lock)
{
- bool locked = __mutex_trylock(lock);
+ bool locked = __mutex_trylock(lock, false);
if (locked)
mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
[toc] | [prev] | [standalone]
Back to top | Article view | linux.kernel
csiph-web