Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1284379
| From | "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [PATCH tip/core/rcu 10/13] rcu: Don't redundantly disable irqs in rcu_irq_{enter,exit}() |
| Date | 2015-12-05 01:20 +0100 |
| Message-ID | <qC8K0-1nw-39@gated-at.bofh.it> (permalink) |
| References | <qC8qC-10I-21@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
This commit replaces a local_irq_save()/local_irq_restore() pair with
a lockdep assertion that interrupts are already disabled. This should
remove the corresponding overhead from the interrupt entry/exit fastpaths.
This change was inspired by the fact that Iftekhar Ahmed's mutation
testing showed that removing rcu_irq_enter()'s call to local_ird_restore()
had no effect, which might indicate that interrupts were always enabled
anyway.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
include/linux/rcupdate.h | 4 ++--
include/linux/rcutiny.h | 8 ++++++++
include/linux/rcutree.h | 2 ++
include/linux/tracepoint.h | 4 ++--
kernel/rcu/tree.c | 32 ++++++++++++++++++++++++++------
5 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index a0189ba67fde..f2b667df1131 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -379,9 +379,9 @@ static inline void rcu_init_nohz(void)
*/
#define RCU_NONIDLE(a) \
do { \
- rcu_irq_enter(); \
+ rcu_irq_enter_irqson(); \
do { a; } while (0); \
- rcu_irq_exit(); \
+ rcu_irq_exit_irqson(); \
} while (0)
/*
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 4c1aaf9cce7b..64809aea661c 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -181,6 +181,14 @@ static inline void rcu_irq_enter(void)
{
}
+static inline void rcu_irq_exit_irqson(void)
+{
+}
+
+static inline void rcu_irq_enter_irqson(void)
+{
+}
+
static inline void rcu_irq_exit(void)
{
}
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 9d3eda39bcd2..ad1eda9fa4da 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -97,6 +97,8 @@ void rcu_idle_enter(void);
void rcu_idle_exit(void);
void rcu_irq_enter(void);
void rcu_irq_exit(void);
+void rcu_irq_enter_irqson(void);
+void rcu_irq_exit_irqson(void);
void exit_rcu(void);
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 696a339c592c..7834a8a8bf1e 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -171,8 +171,8 @@ extern void syscall_unregfunc(void);
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), \
- rcu_irq_enter(), \
- rcu_irq_exit()); \
+ rcu_irq_enter_irqson(), \
+ rcu_irq_exit_irqson()); \
}
#else
#define __DECLARE_TRACE_RCU(name, proto, args, cond, data_proto, data_args)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index d6863bceeb45..40940b0d0310 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -732,7 +732,7 @@ void rcu_user_enter(void)
*
* Exit from an interrupt handler, which might possibly result in entering
* idle mode, in other words, leaving the mode in which read-side critical
- * sections can occur.
+ * sections can occur. The caller must have disabled interrupts.
*
* This code assumes that the idle loop never does anything that might
* result in unbalanced calls to irq_enter() and irq_exit(). If your
@@ -745,11 +745,10 @@ void rcu_user_enter(void)
*/
void rcu_irq_exit(void)
{
- unsigned long flags;
long long oldval;
struct rcu_dynticks *rdtp;
- local_irq_save(flags);
+ RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_exit() invoked with irqs enabled!!!");
rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting--;
@@ -760,6 +759,17 @@ void rcu_irq_exit(void)
else
rcu_eqs_enter_common(oldval, true);
rcu_sysidle_enter(1);
+}
+
+/*
+ * Wrapper for rcu_irq_exit() where interrupts are enabled.
+ */
+void rcu_irq_exit_irqson(void)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ rcu_irq_exit();
local_irq_restore(flags);
}
@@ -857,7 +867,7 @@ void rcu_user_exit(void)
*
* Enter an interrupt handler, which might possibly result in exiting
* idle mode, in other words, entering the mode in which read-side critical
- * sections can occur.
+ * sections can occur. The caller must have disabled interrupts.
*
* Note that the Linux kernel is fully capable of entering an interrupt
* handler that it never exits, for example when doing upcalls to
@@ -873,11 +883,10 @@ void rcu_user_exit(void)
*/
void rcu_irq_enter(void)
{
- unsigned long flags;
struct rcu_dynticks *rdtp;
long long oldval;
- local_irq_save(flags);
+ RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_enter() invoked with irqs enabled!!!");
rdtp = this_cpu_ptr(&rcu_dynticks);
oldval = rdtp->dynticks_nesting;
rdtp->dynticks_nesting++;
@@ -888,6 +897,17 @@ void rcu_irq_enter(void)
else
rcu_eqs_exit_common(oldval, true);
rcu_sysidle_exit(1);
+}
+
+/*
+ * Wrapper for rcu_irq_enter() where interrupts are enabled.
+ */
+void rcu_irq_enter_irqson(void)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ rcu_irq_enter();
local_irq_restore(flags);
}
--
2.5.2
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[PATCH tip/core/rcu 0/10] Expedited-grace-period changes for 4.5 "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:00 +0100
[PATCH tip/core/rcu 09/13] rcu: Make cpu_needs_another_gp() be bool "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:10 +0100
[PATCH tip/core/rcu 04/13] rcu: Remove lock-acquisition loop from rcu_read_unlock_special() "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:10 +0100
[PATCH tip/core/rcu 01/13] rcu: Move lock_class_key to local scope "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 02/13] kernel: Make rcu/tree_trace.c explicitly non-modular "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 03/13] rcu: Simplify rcu_sched_qs() control flow "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 06/13] rcu: Avoid tick_nohz_active checks on NOCBs CPUs "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 07/13] rcu: Stop disabling interrupts in scheduler fastpaths "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 05/13] rcu: Fix obsolete rcu_bootup_announce_oddness() comment "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 12/13] rcu: Move wakeup out from under rnp->lock "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 11/13] rcu: Fix comment for rcu_dereference_raw_notrace "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 13/13] rcu: Make rcu_gp_init() be bool rather than int "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 08/13] rcu: Eliminate unused rcu_init_one() argument "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
[PATCH tip/core/rcu 10/13] rcu: Don't redundantly disable irqs in rcu_irq_{enter,exit}() "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-05 01:20 +0100
Re: [PATCH tip/core/rcu 0/10] Expedited-grace-period changes for 4.5 Josh Triplett <josh@joshtriplett.org> - 2015-12-05 02:10 +0100
Re: [PATCH tip/core/rcu 0/10] Expedited-grace-period changes for 4.5 "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> - 2015-12-06 03:30 +0100
csiph-web