Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1624104

[patch 20/20] cpu/hotplug: Convert hotplug locking to percpu rwsem

From Thomas Gleixner <tglx@linutronix.de>
Newsgroups linux.kernel
Subject [patch 20/20] cpu/hotplug: Convert hotplug locking to percpu rwsem
Date 2017-04-15 19:40 +0200
Message-ID <twzPX-123-7@gated-at.bofh.it> (permalink)
References <twzPX-123-3@gated-at.bofh.it>
Organization linux.* mail to news gateway

Show all headers | View raw

There are no more (known) nested calls to get_online_cpus() so it's
possible to remove the nested call magic and convert the mutex to a
percpu-rwsem, which speeds up get/put_online_cpus() significantly for the
uncontended case.

The contended case (write locked for hotplug operations) is slow anyway, so
the slightly more expensive down_write of the percpu rwsem does not matter.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/cpu.c |  102 ++++-------------------------------------------------------
 1 file changed, 8 insertions(+), 94 deletions(-)

--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -27,6 +27,7 @@
 #include <linux/smpboot.h>
 #include <linux/relay.h>
 #include <linux/slab.h>
+#include <linux/percpu-rwsem.h>
 
 #include <trace/events/power.h>
 #define CREATE_TRACE_POINTS
@@ -196,121 +197,36 @@ void cpu_maps_update_done(void)
 	mutex_unlock(&cpu_add_remove_lock);
 }
 
-/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
+/*
+ * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
  * Should always be manipulated under cpu_add_remove_lock
  */
 static int cpu_hotplug_disabled;
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-static struct {
-	struct task_struct *active_writer;
-	/* wait queue to wake up the active_writer */
-	wait_queue_head_t wq;
-	/* verifies that no writer will get active while readers are active */
-	struct mutex lock;
-	/*
-	 * Also blocks the new readers during
-	 * an ongoing cpu hotplug operation.
-	 */
-	atomic_t refcount;
-
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	struct lockdep_map dep_map;
-#endif
-} cpu_hotplug = {
-	.active_writer = NULL,
-	.wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
-	.lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	.dep_map = STATIC_LOCKDEP_MAP_INIT("cpu_hotplug.dep_map", &cpu_hotplug.dep_map),
-#endif
-};
-
-/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
-#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
-#define cpuhp_lock_acquire_tryread() \
-				  lock_map_acquire_tryread(&cpu_hotplug.dep_map)
-#define cpuhp_lock_acquire()      lock_map_acquire(&cpu_hotplug.dep_map)
-#define cpuhp_lock_release()      lock_map_release(&cpu_hotplug.dep_map)
-
+DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 
 void get_online_cpus(void)
 {
-	might_sleep();
-	if (cpu_hotplug.active_writer == current)
-		return;
-	cpuhp_lock_acquire_read();
-	mutex_lock(&cpu_hotplug.lock);
-	atomic_inc(&cpu_hotplug.refcount);
-	mutex_unlock(&cpu_hotplug.lock);
+	percpu_down_read(&cpu_hotplug_lock);
 }
 EXPORT_SYMBOL_GPL(get_online_cpus);
 
 void put_online_cpus(void)
 {
-	int refcount;
-
-	if (cpu_hotplug.active_writer == current)
-		return;
-
-	refcount = atomic_dec_return(&cpu_hotplug.refcount);
-	if (WARN_ON(refcount < 0)) /* try to fix things up */
-		atomic_inc(&cpu_hotplug.refcount);
-
-	if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
-		wake_up(&cpu_hotplug.wq);
-
-	cpuhp_lock_release();
-
+	percpu_up_read(&cpu_hotplug_lock);
 }
 EXPORT_SYMBOL_GPL(put_online_cpus);
 
-/*
- * This ensures that the hotplug operation can begin only when the
- * refcount goes to zero.
- *
- * Note that during a cpu-hotplug operation, the new readers, if any,
- * will be blocked by the cpu_hotplug.lock
- *
- * Since cpu_hotplug_begin() is always called after invoking
- * cpu_maps_update_begin(), we can be sure that only one writer is active.
- *
- * Note that theoretically, there is a possibility of a livelock:
- * - Refcount goes to zero, last reader wakes up the sleeping
- *   writer.
- * - Last reader unlocks the cpu_hotplug.lock.
- * - A new reader arrives at this moment, bumps up the refcount.
- * - The writer acquires the cpu_hotplug.lock finds the refcount
- *   non zero and goes to sleep again.
- *
- * However, this is very difficult to achieve in practice since
- * get_online_cpus() not an api which is called all that often.
- *
- */
 void cpu_hotplug_begin(void)
 {
-	DEFINE_WAIT(wait);
-
-	cpu_hotplug.active_writer = current;
-	cpuhp_lock_acquire();
-
-	for (;;) {
-		mutex_lock(&cpu_hotplug.lock);
-		prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
-		if (likely(!atomic_read(&cpu_hotplug.refcount)))
-				break;
-		mutex_unlock(&cpu_hotplug.lock);
-		schedule();
-	}
-	finish_wait(&cpu_hotplug.wq, &wait);
+	percpu_down_write(&cpu_hotplug_lock);
 }
 
 void cpu_hotplug_done(void)
 {
-	cpu_hotplug.active_writer = NULL;
-	mutex_unlock(&cpu_hotplug.lock);
-	cpuhp_lock_release();
+	percpu_up_write(&cpu_hotplug_lock);
 }
 
 /*
@@ -344,8 +260,6 @@ void cpu_hotplug_enable(void)
 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 #endif	/* CONFIG_HOTPLUG_CPU */
 
-/* Notifier wrappers for transitioning to state machine */
-
 static int bringup_wait_for_ap(unsigned int cpu)
 {
 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[patch 00/20] cpu/hotplug: Convert get_online_cpus() to a percpu_rwsem Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 02/20] stop_machine: Provide stop_machine_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 20/20] cpu/hotplug: Convert hotplug locking to percpu rwsem Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
    Re: [patch 20/20] cpu/hotplug: Convert hotplug locking to percpu  rwsem Peter Zijlstra <peterz@infradead.org> - 2017-04-17 09:00 +0200
  [patch 17/20] PCI: Use cpu_hotplug_disable() instead of  get_online_cpus() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
    Re: [patch 17/20] PCI: Use cpu_hotplug_disable() instead of  get_online_cpus() Peter Zijlstra <peterz@infradead.org> - 2017-04-17 08:50 +0200
      Re: [patch 17/20] PCI: Use cpu_hotplug_disable() instead of  get_online_cpus() Thomas Gleixner <tglx@linutronix.de> - 2017-04-17 09:50 +0200
    Re: [patch 17/20] PCI: Use cpu_hotplug_disable() instead of  get_online_cpus() Bjorn Helgaas <helgaas@kernel.org> - 2017-04-18 21:50 +0200
      Re: [patch 17/20] PCI: Use cpu_hotplug_disable() instead of  get_online_cpus() Thomas Gleixner <tglx@linutronix.de> - 2017-04-18 22:00 +0200
  [patch 12/20] s390/kernel: Use stop_machine_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 13/20] powerpc/powernv: Use stop_machine_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 03/20] padata: Make padata_alloc() static Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
    Re: [patch 03/20] padata: Make padata_alloc() static "Jason A. Donenfeld" <Jason@zx2c4.com> - 2017-04-16 08:30 +0200
      Re: [patch 03/20] padata: Make padata_alloc() static Thomas Gleixner <tglx@linutronix.de> - 2017-04-17 11:20 +0200
  [patch 01/20] cpu/hotplug: Provide  cpuhp_setup/remove_state[_nocalls]_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 18/20] PCI: Replace the racy recursion prevention Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 04/20] padata: Avoid nested calls to get_online_cpus() in  pcrypt_init_padata() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 14/20] kernel/hotplug: Use stop_machine_locked() in  takedown_cpu() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 05/20] x86/mtrr: Remove get_online_cpus() from  mtrr_save_state() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 07/20] KVM/PPC/Book3S HV: Use  cpuhp_setup_state_nocalls_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 16/20] perf/x86/intel: Drop get_online_cpus() in  intel_snb_check_microcode() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
    Re: [patch 16/20] perf/x86/intel: Drop get_online_cpus() in  intel_snb_check_microcode() Borislav Petkov <bp@alien8.de> - 2017-04-18 13:30 +0200
  [patch 09/20] hwtracing/coresight-etm4x: Use  cpuhp_setup_state_nocalls_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 10/20] perf/x86/intel/cqm: Use cpuhp_setup_state_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 06/20] cpufreq: Use cpuhp_setup_state_nocalls_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
    Re: [patch 06/20] cpufreq: Use cpuhp_setup_state_nocalls_locked() "Rafael J. Wysocki" <rafael@kernel.org> - 2017-04-16 01:00 +0200
    Re: [patch 06/20] cpufreq: Use cpuhp_setup_state_nocalls_locked() Viresh Kumar <viresh.kumar@linaro.org> - 2017-04-17 06:20 +0200
  [patch 08/20] hwtracing/coresight-etm3x: Use the locked version of  cpuhp_setup_state_nocalls() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200
  [patch 11/20] ARM/hw_breakpoint: Use cpuhp_setup_state_locked() Thomas Gleixner <tglx@linutronix.de> - 2017-04-15 19:40 +0200

csiph-web