Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]


Groups > linux.kernel > #1625222 > unrolled thread

[PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp()

Started byPeter Zijlstra <peterz@infradead.org>
First post2017-04-18 12:40 +0200
Last post2017-04-20 13:40 +0200
Articles 11 — 4 participants

Back to article view | Back to linux.kernel

This discussion starts older than the indexed window; earlier articles aren't shown. The article labeled Started by below is the oldest one visible, not the original post.


Contents

  [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Peter Zijlstra <peterz@infradead.org> - 2017-04-18 12:40 +0200
    Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Peter Zijlstra <peterz@infradead.org> - 2017-04-18 15:10 +0200
      Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Steven Rostedt <rostedt@goodmis.org> - 2017-04-18 18:50 +0200
        Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Peter Zijlstra <peterz@infradead.org> - 2017-04-18 22:30 +0200
          Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Thomas Gleixner <tglx@linutronix.de> - 2017-04-18 23:00 +0200
            Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Peter Zijlstra <peterz@infradead.org> - 2017-04-19 08:40 +0200
              Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Thomas Gleixner <tglx@linutronix.de> - 2017-04-19 11:10 +0200
                Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Steven Rostedt <rostedt@goodmis.org> - 2017-04-19 14:50 +0200
                  Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Peter Zijlstra <peterz@infradead.org> - 2017-04-19 16:10 +0200
                    Re: [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp() Steven Rostedt <rostedt@goodmis.org> - 2017-04-19 16:30 +0200
    [tip:smp/hotplug] jump_label: Provide  static_key_slow_inc_cpuslocked() "tip-bot for Peter Zijlstra (Intel)" <tipbot@zytor.com> - 2017-04-20 13:40 +0200

#1625222 — [PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp()

FromPeter Zijlstra <peterz@infradead.org>
Date2017-04-18 12:40 +0200
Subject[PATCH 2/3] jump_label: Provide static_key_slow_inc_nohp()
Message-ID<txyI9-54q-1@gated-at.bofh.it>
Provide static_key_slow_inc_nohp(), a variant that doesn't take
cpu_hotplug_lock().

XXX maybe add an assertion that it is taken.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/jump_label.h |    3 +++
 kernel/jump_label.c        |   21 +++++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -158,6 +158,7 @@ extern void arch_jump_label_transform_st
 					     enum jump_label_type type);
 extern int jump_label_text_reserved(void *start, void *end);
 extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_inc_nohp(struct static_key *key);
 extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 extern int static_key_count(struct static_key *key);
@@ -213,6 +214,8 @@ static inline void static_key_slow_inc(s
 	atomic_inc(&key->enabled);
 }
 
+#define static_key_slow_inc_nohp static_key_slow_inc
+
 static inline void static_key_slow_dec(struct static_key *key)
 {
 	STATIC_KEY_CHECK_USE();
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -100,7 +100,7 @@ void static_key_disable(struct static_ke
 }
 EXPORT_SYMBOL_GPL(static_key_disable);
 
-void static_key_slow_inc(struct static_key *key)
+void __static_key_slow_inc(struct static_key *key)
 {
 	int v, v1;
 
@@ -129,7 +129,6 @@ void static_key_slow_inc(struct static_k
 	 * the all CPUs, for that to be serialized against CPU hot-plug
 	 * we need to avoid CPUs coming online.
 	 */
-	get_online_cpus();
 	jump_label_lock();
 	if (atomic_read(&key->enabled) == 0) {
 		atomic_set(&key->enabled, -1);
@@ -139,10 +138,22 @@ void static_key_slow_inc(struct static_k
 		atomic_inc(&key->enabled);
 	}
 	jump_label_unlock();
+}
+
+void static_key_slow_inc(struct static_key *key)
+{
+	get_online_cpus();
+	__static_key_slow_inc(key);
 	put_online_cpus();
 }
 EXPORT_SYMBOL_GPL(static_key_slow_inc);
 
+void static_key_slow_inc_nohp(struct static_key *key)
+{
+	__static_key_slow_inc(key);
+}
+EXPORT_SYMBOL_GPL(static_key_slow_inc_nohp);
+
 static void __static_key_slow_dec(struct static_key *key,
 		unsigned long rate_limit, struct delayed_work *work)
 {
@@ -153,7 +164,6 @@ static void __static_key_slow_dec(struct
 	 * returns is unbalanced, because all other static_key_slow_inc()
 	 * instances block while the update is in progress.
 	 */
-	get_online_cpus();
 	if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
 		WARN(atomic_read(&key->enabled) < 0,
 		     "jump label: negative count!\n");
@@ -167,20 +177,23 @@ static void __static_key_slow_dec(struct
 		jump_label_update(key);
 	}
 	jump_label_unlock();
-	put_online_cpus();
 }
 
 static void jump_label_update_timeout(struct work_struct *work)
 {
 	struct static_key_deferred *key =
 		container_of(work, struct static_key_deferred, work.work);
+	get_online_cpus();
 	__static_key_slow_dec(&key->key, 0, NULL);
+	put_online_cpus();
 }
 
 void static_key_slow_dec(struct static_key *key)
 {
 	STATIC_KEY_CHECK_USE();
+	get_online_cpus();
 	__static_key_slow_dec(key, 0, NULL);
+	put_online_cpus();
 }
 EXPORT_SYMBOL_GPL(static_key_slow_dec);
 

[toc] | [next] | [standalone]


#1625311

FromPeter Zijlstra <peterz@infradead.org>
Date2017-04-18 15:10 +0200
Message-ID<txB3k-6zp-15@gated-at.bofh.it>
In reply to#1625222
On Tue, Apr 18, 2017 at 12:32:15PM +0200, Peter Zijlstra wrote:
> XXX maybe add an assertion that it is taken.

Compile tested only..

---
Subject: hotplug,lockdep: Verify the hotplut lock assumptions
From: Peter Zijlstra <peterz@infradead.org>
Date: Tue Apr 18 14:49:09 CEST 2017

With the recent work to avoid recursive hotplug read locks, various code
paths grew the assumption that hotplug lock is held.

Make sure to verify these assumptions to avoid future broken code.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
 include/linux/cpu.h   |    2 ++
 kernel/cpu.c          |    7 +++++++
 kernel/jump_label.c   |    2 ++
 kernel/padata.c       |    3 +--
 kernel/stop_machine.c |    2 ++
 5 files changed, 14 insertions(+), 2 deletions(-)

--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -105,6 +105,7 @@ extern void cpu_hotplug_begin(void);
 extern void cpu_hotplug_done(void);
 extern void get_online_cpus(void);
 extern void put_online_cpus(void);
+extern void lockdep_assert_hotplug_held(void);
 extern void cpu_hotplug_disable(void);
 extern void cpu_hotplug_enable(void);
 void clear_tasks_mm_cpumask(int cpu);
@@ -118,6 +119,7 @@ static inline void cpu_hotplug_done(void
 #define put_online_cpus()	do { } while (0)
 #define cpu_hotplug_disable()	do { } while (0)
 #define cpu_hotplug_enable()	do { } while (0)
+static inline void lockdep_assert_hotplug_held(void) {}
 #endif		/* CONFIG_HOTPLUG_CPU */
 
 #ifdef CONFIG_PM_SLEEP_SMP
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -229,6 +229,11 @@ void cpu_hotplug_done(void)
 	percpu_up_write(&cpu_hotplug_lock);
 }
 
+void lockdep_assert_hotplug_held(void)
+{
+	percpu_rwsem_assert_held(&cpu_hotplug_lock);
+}
+
 /*
  * Wait for currently running CPU hotplug operations to complete (if any) and
  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
@@ -1398,6 +1403,8 @@ int __cpuhp_setup_state_locked(enum cpuh
 	int cpu, ret = 0;
 	bool dynstate;
 
+	lockdep_assert_hotplug_held();
+
 	if (cpuhp_cb_check(state) || !name)
 		return -EINVAL;
 
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -130,6 +130,7 @@ void __static_key_slow_inc(struct static
 	 * the all CPUs, for that to be serialized against CPU hot-plug
 	 * we need to avoid CPUs coming online.
 	 */
+	lockdep_assert_hotplug_held();
 	jump_label_lock();
 	if (atomic_read(&key->enabled) == 0) {
 		atomic_set(&key->enabled, -1);
@@ -158,6 +159,7 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc_no
 static void __static_key_slow_dec(struct static_key *key,
 		unsigned long rate_limit, struct delayed_work *work)
 {
+	lockdep_assert_hotplug_held();
 	/*
 	 * The negative count check is valid even when a negative
 	 * key->enabled is in use by static_key_slow_inc(); a
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -1008,11 +1008,10 @@ static struct padata_instance *padata_al
  *                         parallel workers.
  *
  * @wq: workqueue to use for the allocated padata instance
- *
- * Must be called from a get_online_cpus() protected region
  */
 struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq)
 {
+	lockdep_assert_hotplug_held();
 	return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask);
 }
 EXPORT_SYMBOL(padata_alloc_possible);
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -561,6 +561,8 @@ int stop_machine_locked(cpu_stop_fn_t fn
 		.active_cpus = cpus,
 	};
 
+	lockdep_assert_hotplug_held();
+
 	if (!stop_machine_initialized) {
 		/*
 		 * Handle the case where stop_machine() is called

[toc] | [prev] | [next] | [standalone]


#1625453

FromSteven Rostedt <rostedt@goodmis.org>
Date2017-04-18 18:50 +0200
Message-ID<txEuf-8r4-45@gated-at.bofh.it>
In reply to#1625311
On Tue, 18 Apr 2017 15:03:50 +0200
Peter Zijlstra <peterz@infradead.org> wrote:

> +++ b/kernel/padata.c
> @@ -1008,11 +1008,10 @@ static struct padata_instance *padata_al
>   *                         parallel workers.
>   *
>   * @wq: workqueue to use for the allocated padata instance
> - *
> - * Must be called from a get_online_cpus() protected region

Find the comment redundant?

Reviewed-by: Steven Rostedt (VMware) <rostedt@goodmis.org>

-- Steve

>   */
>  struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq)
>  {
> +	lockdep_assert_hotplug_held();
>  	return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask);
>  }
>  EXPORT_SYMBOL(padata_alloc_possible);
> --- a/kernel/stop_machine.c
> +++ b/kernel/stop_machine.c
> @@ -561,6 +561,8 @@ int stop_machine_locked(cpu_stop_fn_t fn
>  		.active_cpus = cpus,
>  	};
>  
> +	lockdep_assert_hotplug_held();
> +
>  	if (!stop_machine_initialized) {
>  		/*
>  		 * Handle the case where stop_machine() is called

[toc] | [prev] | [next] | [standalone]


#1625636

FromPeter Zijlstra <peterz@infradead.org>
Date2017-04-18 22:30 +0200
Message-ID<txHV8-29d-21@gated-at.bofh.it>
In reply to#1625453
On Tue, Apr 18, 2017 at 12:46:29PM -0400, Steven Rostedt wrote:
> On Tue, 18 Apr 2017 15:03:50 +0200
> Peter Zijlstra <peterz@infradead.org> wrote:
> 
> > +++ b/kernel/padata.c
> > @@ -1008,11 +1008,10 @@ static struct padata_instance *padata_al
> >   *                         parallel workers.
> >   *
> >   * @wq: workqueue to use for the allocated padata instance
> > - *
> > - * Must be called from a get_online_cpus() protected region
> 
> Find the comment redundant?

Once there's code that enforces it? Yes. Nobody reads comments
;-)

[toc] | [prev] | [next] | [standalone]


#1625647

FromThomas Gleixner <tglx@linutronix.de>
Date2017-04-18 23:00 +0200
Message-ID<txIo9-2iz-1@gated-at.bofh.it>
In reply to#1625636
On Tue, 18 Apr 2017, Peter Zijlstra wrote:
> On Tue, Apr 18, 2017 at 12:46:29PM -0400, Steven Rostedt wrote:
> > On Tue, 18 Apr 2017 15:03:50 +0200
> > Peter Zijlstra <peterz@infradead.org> wrote:
> > 
> > > +++ b/kernel/padata.c
> > > @@ -1008,11 +1008,10 @@ static struct padata_instance *padata_al
> > >   *                         parallel workers.
> > >   *
> > >   * @wq: workqueue to use for the allocated padata instance
> > > - *
> > > - * Must be called from a get_online_cpus() protected region
> > 
> > Find the comment redundant?
> 
> Once there's code that enforces it? Yes. Nobody reads comments
> ;-)

Nobody enables lockdep either .....

[toc] | [prev] | [next] | [standalone]


#1625854

FromPeter Zijlstra <peterz@infradead.org>
Date2017-04-19 08:40 +0200
Message-ID<txRrs-8mT-23@gated-at.bofh.it>
In reply to#1625647
On Tue, Apr 18, 2017 at 10:50:43PM +0200, Thomas Gleixner wrote:
> On Tue, 18 Apr 2017, Peter Zijlstra wrote:
> > On Tue, Apr 18, 2017 at 12:46:29PM -0400, Steven Rostedt wrote:
> > > On Tue, 18 Apr 2017 15:03:50 +0200
> > > Peter Zijlstra <peterz@infradead.org> wrote:
> > > 
> > > > +++ b/kernel/padata.c
> > > > @@ -1008,11 +1008,10 @@ static struct padata_instance *padata_al
> > > >   *                         parallel workers.
> > > >   *
> > > >   * @wq: workqueue to use for the allocated padata instance
> > > > - *
> > > > - * Must be called from a get_online_cpus() protected region
> > > 
> > > Find the comment redundant?
> > 
> > Once there's code that enforces it? Yes. Nobody reads comments
> > ;-)
> 
> Nobody enables lockdep either .....

In the grand scheme of things, true. But there are more people running
with lockdep enabled than there are people writing code, of which there
are more than people reading relevant comments while writing code.
Therefore having the lockdep annotation is two orders better than a
comment ;-)

Also, I would argue that an "assert" at the start of a function is a
fairly readable 'comment' all by itself.

In any case, I don't care too much. But I typically remove such comments
when I stick a lockdep_assert_held() in.

[toc] | [prev] | [next] | [standalone]


#1625949

FromThomas Gleixner <tglx@linutronix.de>
Date2017-04-19 11:10 +0200
Message-ID<txTMB-1zl-17@gated-at.bofh.it>
In reply to#1625854
On Wed, 19 Apr 2017, Peter Zijlstra wrote:
> On Tue, Apr 18, 2017 at 10:50:43PM +0200, Thomas Gleixner wrote:
> > On Tue, 18 Apr 2017, Peter Zijlstra wrote:
> > > On Tue, Apr 18, 2017 at 12:46:29PM -0400, Steven Rostedt wrote:
> > > > On Tue, 18 Apr 2017 15:03:50 +0200
> > > > Peter Zijlstra <peterz@infradead.org> wrote:
> > > > 
> > > > > +++ b/kernel/padata.c
> > > > > @@ -1008,11 +1008,10 @@ static struct padata_instance *padata_al
> > > > >   *                         parallel workers.
> > > > >   *
> > > > >   * @wq: workqueue to use for the allocated padata instance
> > > > > - *
> > > > > - * Must be called from a get_online_cpus() protected region
> > > > 
> > > > Find the comment redundant?
> > > 
> > > Once there's code that enforces it? Yes. Nobody reads comments
> > > ;-)
> > 
> > Nobody enables lockdep either .....
> 
> In the grand scheme of things, true. But there are more people running
> with lockdep enabled than there are people writing code, of which there
> are more than people reading relevant comments while writing code.
> Therefore having the lockdep annotation is two orders better than a
> comment ;-)
> 
> Also, I would argue that an "assert" at the start of a function is a
> fairly readable 'comment' all by itself.
> 
> In any case, I don't care too much. But I typically remove such comments
> when I stick a lockdep_assert_held() in.

I think that's wrong. We are striving for better documentation and the
kernel-doc comments above a function are part of that. Calling conventions
are definitely something which belongs there.

Thanks,

	tglx

[toc] | [prev] | [next] | [standalone]


#1626068

FromSteven Rostedt <rostedt@goodmis.org>
Date2017-04-19 14:50 +0200
Message-ID<txXdv-3vn-13@gated-at.bofh.it>
In reply to#1625949
On Wed, 19 Apr 2017 11:08:35 +0200 (CEST)
Thomas Gleixner <tglx@linutronix.de> wrote:


> > In the grand scheme of things, true. But there are more people running
> > with lockdep enabled than there are people writing code, of which there
> > are more than people reading relevant comments while writing code.
> > Therefore having the lockdep annotation is two orders better than a
> > comment ;-)
> > 
> > Also, I would argue that an "assert" at the start of a function is a
> > fairly readable 'comment' all by itself.
> > 
> > In any case, I don't care too much. But I typically remove such comments
> > when I stick a lockdep_assert_held() in.  
> 
> I think that's wrong. We are striving for better documentation and the
> kernel-doc comments above a function are part of that. Calling conventions
> are definitely something which belongs there.

I agree with Thomas. Removing the comment because a
"lockdep_assert_held()" exists at the top of the code, assumes someone
that is about to use that function did more that read the kerneldoc and
actually looked at the code.

If there's a kerneldoc to a function, than that header should contain
all the info that a developer needs to use that function.

-- Steve

[toc] | [prev] | [next] | [standalone]


#1626137

FromPeter Zijlstra <peterz@infradead.org>
Date2017-04-19 16:10 +0200
Message-ID<txYsW-4oW-21@gated-at.bofh.it>
In reply to#1626068
On Wed, Apr 19, 2017 at 08:49:11AM -0400, Steven Rostedt wrote:
> I agree with Thomas. Removing the comment because a
> "lockdep_assert_held()" exists at the top of the code, assumes someone
> that is about to use that function did more that read the kerneldoc and
> actually looked at the code.

No, it doesn't assume anything. You can use the function without reading
_any_ comments. It will explode at runtime (preferably on the machine of
the guy who wrote it -- who _SHOULD_ have lockdep enabled, but certainly
on other developer's machines and test-bots, who will then loudly yell
at said developer for not doing his job properly).

> If there's a kerneldoc to a function, than that header should contain
> all the info that a developer needs to use that function.

Yeah, rainbows and unicorns are shiny. Also, I put kerneldoc (if I put
it at all) at the definition site, not the declaration. So headers are
useless.


In any case, I don't mind the extra line of comment. Don't really see
the point of it either. What I am convinced of is that
lockdep_assert_held() lines are far more useful than such comment lines.

[toc] | [prev] | [next] | [standalone]


#1626152

FromSteven Rostedt <rostedt@goodmis.org>
Date2017-04-19 16:30 +0200
Message-ID<txYMi-4vr-7@gated-at.bofh.it>
In reply to#1626137
On Wed, 19 Apr 2017 16:08:04 +0200
Peter Zijlstra <peterz@infradead.org> wrote:


> Yeah, rainbows and unicorns are shiny. Also, I put kerneldoc (if I put
> it at all) at the definition site, not the declaration. So headers are
> useless.

What's wrong with rainbows and unicorns. We all have our ponys, and
some of them are unicorns.

https://www.slideshare.net/brendangregg/velocity-2015-linux-perf-tools/105

> 
> 
> In any case, I don't mind the extra line of comment. Don't really see
> the point of it either. What I am convinced of is that
> lockdep_assert_held() lines are far more useful than such comment lines.

I agree with the lockdep assert held being more useful. But I disagree
with removing comments about required locks when it is added. A comment
may save a developer an embarrassing moment of being yelled at because
they didn't test their code with lockdep enabled. And that is useful.

-- Steve

[toc] | [prev] | [next] | [standalone]


#1627388 — [tip:smp/hotplug] jump_label: Provide static_key_slow_inc_cpuslocked()

From"tip-bot for Peter Zijlstra (Intel)" <tipbot@zytor.com>
Date2017-04-20 13:40 +0200
Subject[tip:smp/hotplug] jump_label: Provide static_key_slow_inc_cpuslocked()
Message-ID<tyiBl-8mU-33@gated-at.bofh.it>
In reply to#1625222
Commit-ID:  f5efc6fad63f5533a6083e95286920d5753e52bf
Gitweb:     http://git.kernel.org/tip/f5efc6fad63f5533a6083e95286920d5753e52bf
Author:     Peter Zijlstra (Intel) <peterz@infradead.org>
AuthorDate: Tue, 18 Apr 2017 19:05:04 +0200
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Thu, 20 Apr 2017 13:08:57 +0200

jump_label: Provide static_key_slow_inc_cpuslocked()

Provide static_key_slow_inc_cpuslocked(), a variant that doesn't take
cpu_hotplug_lock().

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Sebastian Siewior <bigeasy@linutronix.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: jbaron@akamai.com
Link: http://lkml.kernel.org/r/20170418103422.636958338@infradead.org

---
 include/linux/jump_label.h |  3 +++
 kernel/jump_label.c        | 21 +++++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 2afd74b..7d07f0b 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -158,6 +158,7 @@ extern void arch_jump_label_transform_static(struct jump_entry *entry,
 					     enum jump_label_type type);
 extern int jump_label_text_reserved(void *start, void *end);
 extern void static_key_slow_inc(struct static_key *key);
+extern void static_key_slow_inc_cpuslocked(struct static_key *key);
 extern void static_key_slow_dec(struct static_key *key);
 extern void jump_label_apply_nops(struct module *mod);
 extern int static_key_count(struct static_key *key);
@@ -213,6 +214,8 @@ static inline void static_key_slow_inc(struct static_key *key)
 	atomic_inc(&key->enabled);
 }
 
+#define static_key_slow_inc_cpuslocked static_key_slow_inc
+
 static inline void static_key_slow_dec(struct static_key *key)
 {
 	STATIC_KEY_CHECK_USE();
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index f3afe07..308b12e 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -101,7 +101,7 @@ void static_key_disable(struct static_key *key)
 }
 EXPORT_SYMBOL_GPL(static_key_disable);
 
-void static_key_slow_inc(struct static_key *key)
+void __static_key_slow_inc(struct static_key *key)
 {
 	int v, v1;
 
@@ -130,7 +130,6 @@ void static_key_slow_inc(struct static_key *key)
 	 * the all CPUs, for that to be serialized against CPU hot-plug
 	 * we need to avoid CPUs coming online.
 	 */
-	get_online_cpus();
 	jump_label_lock();
 	if (atomic_read(&key->enabled) == 0) {
 		atomic_set(&key->enabled, -1);
@@ -140,10 +139,22 @@ void static_key_slow_inc(struct static_key *key)
 		atomic_inc(&key->enabled);
 	}
 	jump_label_unlock();
+}
+
+void static_key_slow_inc(struct static_key *key)
+{
+	get_online_cpus();
+	__static_key_slow_inc(key);
 	put_online_cpus();
 }
 EXPORT_SYMBOL_GPL(static_key_slow_inc);
 
+void static_key_slow_inc_cpuslocked(struct static_key *key)
+{
+	__static_key_slow_inc(key);
+}
+EXPORT_SYMBOL_GPL(static_key_slow_inc_cpuslocked);
+
 static void __static_key_slow_dec(struct static_key *key,
 		unsigned long rate_limit, struct delayed_work *work)
 {
@@ -154,7 +165,6 @@ static void __static_key_slow_dec(struct static_key *key,
 	 * returns is unbalanced, because all other static_key_slow_inc()
 	 * instances block while the update is in progress.
 	 */
-	get_online_cpus();
 	if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
 		WARN(atomic_read(&key->enabled) < 0,
 		     "jump label: negative count!\n");
@@ -168,20 +178,23 @@ static void __static_key_slow_dec(struct static_key *key,
 		jump_label_update(key);
 	}
 	jump_label_unlock();
-	put_online_cpus();
 }
 
 static void jump_label_update_timeout(struct work_struct *work)
 {
 	struct static_key_deferred *key =
 		container_of(work, struct static_key_deferred, work.work);
+	get_online_cpus();
 	__static_key_slow_dec(&key->key, 0, NULL);
+	put_online_cpus();
 }
 
 void static_key_slow_dec(struct static_key *key)
 {
 	STATIC_KEY_CHECK_USE();
+	get_online_cpus();
 	__static_key_slow_dec(key, 0, NULL);
+	put_online_cpus();
 }
 EXPORT_SYMBOL_GPL(static_key_slow_dec);
 

[toc] | [prev] | [standalone]


Back to top | Article view | linux.kernel


csiph-web