Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1484684

[PATCH v2 8/8] fork: Cache two thread stacks per cpu if CONFIG_VMAP_STACK is set

Show all headers | View raw

vmalloc is a bit slow, and pounding vmalloc/vfree will eventually
force a global TLB flush.

To reduce pressure on them, if CONFIG_VMAP_STACK, cache two thread
stacks per cpu.  This will let us quickly allocate a hopefully
cache-hot, TLB-hot stack under heavy forking workloads (shell script
style).

On my silly pthread_create benchmark, it saves about 2 µs per
pthread_create+join with CONFIG_VMAP_STACK=y.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
 kernel/fork.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/kernel/fork.c b/kernel/fork.c
index 5dd0a516626d..2d44a9d05218 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -159,15 +159,41 @@ void __weak arch_release_thread_stack(unsigned long *stack)
  * kmemcache based allocator.
  */
 # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
+
+#ifdef CONFIG_VMAP_STACK
+/*
+ * vmalloc is a bit slow, and calling vfree enough times will force a TLB
+ * flush.  Try to minimize the number of calls by caching stacks.
+ */
+#define NR_CACHED_STACKS 2
+static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
+#endif
+
 static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 {
 #ifdef CONFIG_VMAP_STACK
-	void *stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
-					   VMALLOC_START, VMALLOC_END,
-					   THREADINFO_GFP | __GFP_HIGHMEM,
-					   PAGE_KERNEL,
-					   0, node,
-					   __builtin_return_address(0));
+	void *stack;
+	int i;
+
+	local_irq_disable();
+	for (i = 0; i < NR_CACHED_STACKS; i++) {
+		struct vm_struct *s = this_cpu_read(cached_stacks[i]);
+
+		if (!s)
+			continue;
+		this_cpu_write(cached_stacks[i], NULL);
+
+		tsk->stack_vm_area = s;
+		local_irq_enable();
+		return s->addr;
+	}
+	local_irq_enable();
+
+	stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
+				     VMALLOC_START, VMALLOC_END,
+				     THREADINFO_GFP | __GFP_HIGHMEM,
+				     PAGE_KERNEL,
+				     0, node, __builtin_return_address(0));
 
 	/*
 	 * We can't call find_vm_area() in interrupt context, and
@@ -187,10 +213,28 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
 
 static inline void free_thread_stack(struct task_struct *tsk)
 {
-	if (task_stack_vm_area(tsk))
+#ifdef CONFIG_VMAP_STACK
+	if (task_stack_vm_area(tsk)) {
+		unsigned long flags;
+		int i;
+
+		local_irq_save(flags);
+		for (i = 0; i < NR_CACHED_STACKS; i++) {
+			if (this_cpu_read(cached_stacks[i]))
+				continue;
+
+			this_cpu_write(cached_stacks[i], tsk->stack_vm_area);
+			local_irq_restore(flags);
+			return;
+		}
+		local_irq_restore(flags);
+
 		vfree(tsk->stack);
-	else
-		__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
+		return;
+	}
+#endif
+
+	__free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
 }
 # else
 static struct kmem_cache *thread_stack_cache;
-- 
2.7.4

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[PATCH v2 0/8] thread_info cleanups and stack caching Andy Lutomirski <luto@kernel.org> - 2016-09-16 07:50 +0200
  [PATCH v2 1/8] x86/entry/64: Fix a minor comment rebase error Andy Lutomirski <luto@kernel.org> - 2016-09-16 07:50 +0200
    [tip:x86/asm] x86/entry/64: Fix a minor comment rebase error tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-16 11:20 +0200
  [PATCH v2 4/8] x86/dumpstack: Pin the target stack when dumping it Andy Lutomirski <luto@kernel.org> - 2016-09-16 07:50 +0200
    [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-16 11:20 +0200
      Re: [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-16 14:00 +0200
        Re: [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-16 14:30 +0200
          Re: [tip:x86/asm] x86/dumpstack: Pin the target stack when dumping it Ingo Molnar <mingo@kernel.org> - 2016-09-16 15:00 +0200
            [PATCH] x86/dumpstack: remove NULL task pointer convention Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-16 15:10 +0200
  [PATCH v2 6/8] lib/syscall: Pin the task stack in collect_syscall() Andy Lutomirski <luto@kernel.org> - 2016-09-16 07:50 +0200
    [tip:x86/asm] lib/syscall: Pin the task stack in collect_syscall() tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-16 11:20 +0200
  [PATCH v2 8/8] fork: Cache two thread stacks per cpu if CONFIG_VMAP_STACK is set Andy Lutomirski <luto@kernel.org> - 2016-09-16 07:50 +0200
    [tip:x86/asm] fork: Optimize task creation by caching two thread  stacks per CPU if CONFIG_VMAP_STACK=y tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-16 11:30 +0200
  [PATCH v2 3/8] kthread: to_live_kthread() needs try_get_task_stack() Andy Lutomirski <luto@kernel.org> - 2016-09-16 07:50 +0200
    [tip:x86/asm] kthread: Pin the stack via  try_get_task_stack()/put_task_stack() in to_live_kthread() function tip-bot for Oleg Nesterov <tipbot@zytor.com> - 2016-09-16 11:20 +0200

csiph-web