Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1482795
| From | Andy Lutomirski <luto@kernel.org> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [PATCH 11/12] sched: Free the stack early if CONFIG_THREAD_INFO_IN_TASK |
| Date | 2016-09-13 23:40 +0200 |
| Message-ID | <sh3AS-7tK-33@gated-at.bofh.it> (permalink) |
| References | <sh3rc-7o2-3@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
We currently keep every task's stack around until the task_struct
itself is freed. This means that we keep the stack allocation alive
for longer than necessary and that, under load, we free stacks in
big batches whenever RCU drops the last task reference. Neither of
these is good for reuse of cache-hot memory, and freeing in batches
prevents us from usefully caching small numbers of vmalloced stacks.
On architectures that have thread_info on the stack, we can't easily
change this, but on architectures that set THREAD_INFO_IN_TASK, we
can free it as soon as the task is dead.
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
include/linux/init_task.h | 4 +++-
include/linux/sched.h | 14 ++++++++++++++
kernel/fork.c | 35 ++++++++++++++++++++++++++++++++++-
kernel/sched/core.c | 4 ++++
4 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 9c04d44eeb3c..325f649d77ff 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -186,7 +186,9 @@ extern struct task_group root_task_group;
#endif
#ifdef CONFIG_THREAD_INFO_IN_TASK
-# define INIT_TASK_TI(tsk) .thread_info = INIT_THREAD_INFO(tsk),
+# define INIT_TASK_TI(tsk) \
+ .thread_info = INIT_THREAD_INFO(tsk), \
+ .stack_refcount = ATOMIC_INIT(1),
#else
# define INIT_TASK_TI(tsk)
#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a95867267e9f..abb795afc823 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1936,6 +1936,10 @@ struct task_struct {
#ifdef CONFIG_VMAP_STACK
struct vm_struct *stack_vm_area;
#endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+ /* A live task holds one reference. */
+ atomic_t stack_refcount;
+#endif
/* CPU-specific state of this task */
struct thread_struct thread;
/*
@@ -3143,12 +3147,22 @@ static inline unsigned long *end_of_stack(struct task_struct *p)
#endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+static inline void *try_get_task_stack(struct task_struct *tsk)
+{
+ return atomic_inc_not_zero(&tsk->stack_refcount) ?
+ task_stack_page(tsk) : NULL;
+}
+
+extern void put_task_stack(struct task_struct *tsk);
+#else
static inline void *try_get_task_stack(struct task_struct *tsk)
{
return task_stack_page(tsk);
}
static inline void put_task_stack(struct task_struct *tsk) {}
+#endif
#define task_stack_end_corrupted(task) \
(*(end_of_stack(task)) != STACK_END_MAGIC)
diff --git a/kernel/fork.c b/kernel/fork.c
index 9b85f6b2cdcd..e0fd5446df58 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -269,11 +269,40 @@ static void account_kernel_stack(struct task_struct *tsk, int account)
}
}
-void free_task(struct task_struct *tsk)
+static void release_task_stack(struct task_struct *tsk)
{
account_kernel_stack(tsk, -1);
arch_release_thread_stack(tsk->stack);
free_thread_stack(tsk);
+ tsk->stack = NULL;
+#ifdef CONFIG_VMAP_STACK
+ tsk->stack_vm_area = NULL;
+#endif
+}
+
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+void put_task_stack(struct task_struct *tsk)
+{
+ if (atomic_dec_and_test(&tsk->stack_refcount))
+ release_task_stack(tsk);
+}
+#endif
+
+void free_task(struct task_struct *tsk)
+{
+#ifndef CONFIG_THREAD_INFO_IN_TASK
+ /*
+ * The task is finally done with both the stack and thread_info,
+ * so free both.
+ */
+ release_task_stack(tsk);
+#else
+ /*
+ * If the task had a separate stack allocation, it should be gone
+ * by now.
+ */
+ WARN_ON_ONCE(atomic_read(&tsk->stack_refcount) != 0);
+#endif
rt_mutex_debug_task_free(tsk);
ftrace_graph_exit_task(tsk);
put_seccomp_filter(tsk);
@@ -411,6 +440,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#ifdef CONFIG_VMAP_STACK
tsk->stack_vm_area = stack_vm_area;
#endif
+#ifdef CONFIG_THREAD_INFO_IN_TASK
+ atomic_set(&tsk->stack_refcount, 1);
+#endif
if (err)
goto free_stack;
@@ -1750,6 +1782,7 @@ bad_fork_cleanup_count:
atomic_dec(&p->cred->user->processes);
exit_creds(p);
bad_fork_free:
+ put_task_stack(p);
free_task(p);
fork_out:
return ERR_PTR(retval);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3d91b63dd2f6..d61bfae91093 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2750,6 +2750,10 @@ static struct rq *finish_task_switch(struct task_struct *prev)
* task and put them back on the free list.
*/
kprobe_flush_task(prev);
+
+ /* Task is done with its stack. */
+ put_task_stack(prev);
+
put_task_struct(prev);
}
--
2.7.4
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[PATCH 00/12] thread_info cleanups and stack caching Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:30 +0200
[PATCH 03/12] um: Stop conflating task_struct::stack with thread_info Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:30 +0200
Re: [PATCH 03/12] um: Stop conflating task_struct::stack with thread_info Ingo Molnar <mingo@kernel.org> - 2016-09-15 08:30 +0200
[tip:x86/asm] um/Stop conflating task_struct::stack with thread_info tip-bot for Linus Torvalds <tipbot@zytor.com> - 2016-09-15 12:50 +0200
[PATCH 12/12] fork: Cache two thread stacks per cpu if CONFIG_VMAP_STACK is set Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[PATCH 01/12] x86/asm: Move 'status' from struct thread_info to struct thread_struct Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[tip:x86/asm] x86/asm: Move the thread_info::status field to thread_struct tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-15 12:50 +0200
[PATCH 10/12] lib/syscall: Pin the task stack in collect_syscall() Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-14 17:00 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Andy Lutomirski <luto@amacapital.net> - 2016-09-14 20:30 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-14 20:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Andy Lutomirski <luto@amacapital.net> - 2016-09-15 20:10 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-15 20:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Andy Lutomirski <luto@amacapital.net> - 2016-09-15 20:50 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Josh Poimboeuf <jpoimboe@redhat.com> - 2016-09-15 21:20 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Peter Zijlstra <peterz@infradead.org> - 2016-09-16 09:50 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Andy Lutomirski <luto@amacapital.net> - 2016-09-16 17:20 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Andy Lutomirski <luto@kernel.org> - 2016-09-16 17:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Peter Zijlstra <peterz@infradead.org> - 2016-09-16 18:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Peter Zijlstra <peterz@infradead.org> - 2016-09-16 17:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Ingo Molnar <mingo@kernel.org> - 2016-09-15 08:40 +0200
Re: [PATCH 08/12] x86/dumpstack: Pin the target stack in save_stack_trace_tsk() Ingo Molnar <mingo@kernel.org> - 2016-09-15 11:30 +0200
[PATCH 09/12] x86/process: Pin the target stack in get_wchan() Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
Re: [PATCH 09/12] x86/process: Pin the target stack in get_wchan() Jann Horn <jann@thejh.net> - 2016-09-17 04:10 +0200
Re: [PATCH 09/12] x86/process: Pin the target stack in get_wchan() Andy Lutomirski <luto@amacapital.net> - 2016-09-23 00:50 +0200
Re: [PATCH 09/12] x86/process: Pin the target stack in get_wchan() Andy Lutomirski <luto@amacapital.net> - 2016-09-23 01:00 +0200
Re: [PATCH 09/12] x86/process: Pin the target stack in get_wchan() Jann Horn <jann@thejh.net> - 2016-09-23 09:50 +0200
[PATCH 06/12] sched: Add try_get_task_stack() and put_task_stack() Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[PATCH 02/12] x86/entry: Get rid of pt_regs_to_thread_info() Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
Re: [PATCH 02/12] x86/entry: Get rid of pt_regs_to_thread_info() Ingo Molnar <mingo@kernel.org> - 2016-09-15 08:30 +0200
[tip:x86/asm] x86/entry: Get rid of pt_regs_to_thread_info() tip-bot for Linus Torvalds <tipbot@zytor.com> - 2016-09-15 12:50 +0200
[PATCH 05/12] x86: Move thread_info into task_struct Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[tip:x86/asm] x86: Move thread_info into task_struct tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-15 12:50 +0200
[PATCH 04/12] sched: Allow putting thread_info into task_struct Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[tip:x86/asm] sched/core: Allow putting thread_info into task_struct tip-bot for Andy Lutomirski <tipbot@zytor.com> - 2016-09-15 12:50 +0200
[PATCH 11/12] sched: Free the stack early if CONFIG_THREAD_INFO_IN_TASK Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
[PATCH 07/12] kthread: to_live_kthread() needs try_get_task_stack() Andy Lutomirski <luto@kernel.org> - 2016-09-13 23:40 +0200
csiph-web