Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1637068
| From | Andy Lutomirski <luto@kernel.org> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic |
| Date | 2017-05-08 00:10 +0200 |
| Message-ID | <tECxk-3oj-19@gated-at.bofh.it> (permalink) |
| References | <tECxj-3oj-5@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
try_to_unmap_flush() used to open-code a rather x86-centric flush
sequence: local_flush_tlb() + flush_tlb_others(). Rearrange the
code so that the arch (only x86 for now) provides
arch_tlbbatch_add_mm() and arch_tlbbatch_flush() and the core code
calls those functions instead.
I'll want this for x86 because, to enable address space ids, I can't
support the flush_tlb_others() mode used by exising
try_to_unmap_flush() implementation with good performance. I can
support the new API fairly easily, though.
I imagine that other architectures may be in a similar position.
Architectures with strong remote flush primitives (arm64?) may have
even worse performance problems with flush_tlb_others() the way that
try_to_unmap_flush() uses it.
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Nadav Amit <namit@vmware.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Sasha Levin <sasha.levin@oracle.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Andy Lutomirski <luto@kernel.org>
---
arch/x86/include/asm/tlbbatch.h | 16 ++++++++++++++++
arch/x86/include/asm/tlbflush.h | 8 ++++++++
arch/x86/mm/tlb.c | 17 +++++++++++++++++
include/linux/mm_types_task.h | 15 +++++++++++----
mm/rmap.c | 15 +--------------
5 files changed, 53 insertions(+), 18 deletions(-)
create mode 100644 arch/x86/include/asm/tlbbatch.h
diff --git a/arch/x86/include/asm/tlbbatch.h b/arch/x86/include/asm/tlbbatch.h
new file mode 100644
index 000000000000..01a6de16fb96
--- /dev/null
+++ b/arch/x86/include/asm/tlbbatch.h
@@ -0,0 +1,16 @@
+#ifndef _ARCH_X86_TLBBATCH_H
+#define _ARCH_X86_TLBBATCH_H
+
+#include <linux/cpumask.h>
+
+#ifdef CONFIG_SMP
+struct arch_tlbflush_unmap_batch {
+ /*
+ * Each bit set is a CPU that potentially has a TLB entry for one of
+ * the PFNs being flushed..
+ */
+ struct cpumask cpumask;
+};
+#endif
+
+#endif /* _ARCH_X86_TLBBATCH_H */
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 5ed64cdaf536..df71e3f2fe4d 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -329,6 +329,14 @@ static inline void reset_lazy_tlbstate(void)
this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
+static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch,
+ struct mm_struct *mm)
+{
+ cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
+}
+
+extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
+
#endif /* SMP */
#ifndef CONFIG_PARAVIRT
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 4d303864b310..743e4c6b4529 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -395,6 +395,23 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
}
}
+void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
+{
+ int cpu = get_cpu();
+
+ if (cpumask_test_cpu(cpu, &batch->cpumask)) {
+ count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
+ local_flush_tlb();
+ trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
+ }
+
+ if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
+ flush_tlb_others(&batch->cpumask, NULL, 0, TLB_FLUSH_ALL);
+ cpumask_clear(&batch->cpumask);
+
+ put_cpu();
+}
+
static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
size_t count, loff_t *ppos)
{
diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index 136dfdf63ba1..fc412fbd80bd 100644
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -14,6 +14,10 @@
#include <asm/page.h>
+#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
+#include <asm/tlbbatch.h>
+#endif
+
#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
@@ -67,12 +71,15 @@ struct page_frag {
struct tlbflush_unmap_batch {
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
/*
- * Each bit set is a CPU that potentially has a TLB entry for one of
- * the PFNs being flushed. See set_tlb_ubc_flush_pending().
+ * The arch code makes the following promise: generic code can modify a
+ * PTE, then call arch_tlbbatch_add_mm() (which internally provides all
+ * needed barriers), then call arch_tlbbatch_flush(), and the entries
+ * will be flushed on all CPUs by the time that arch_tlbbatch_flush()
+ * returns.
*/
- struct cpumask cpumask;
+ struct arch_tlbflush_unmap_batch arch;
- /* True if any bit in cpumask is set */
+ /* True if a flush is needed. */
bool flush_required;
/*
diff --git a/mm/rmap.c b/mm/rmap.c
index f6838015810f..2e568c82f477 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -579,25 +579,12 @@ void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
void try_to_unmap_flush(void)
{
struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
- int cpu;
if (!tlb_ubc->flush_required)
return;
- cpu = get_cpu();
-
- if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) {
- count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
- local_flush_tlb();
- trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
- }
-
- if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids)
- flush_tlb_others(&tlb_ubc->cpumask, NULL, 0, TLB_FLUSH_ALL);
- cpumask_clear(&tlb_ubc->cpumask);
tlb_ubc->flush_required = false;
tlb_ubc->writable = false;
- put_cpu();
}
/* Flush iff there are potentially writable TLB entries that can race with IO */
@@ -613,7 +600,7 @@ static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
{
struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
- cpumask_or(&tlb_ubc->cpumask, &tlb_ubc->cpumask, mm_cpumask(mm));
+ arch_tlbbatch_add_mm(&tlb_ubc->arch, mm);
tlb_ubc->flush_required = true;
/*
--
2.9.3
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[RFC 00/10] x86 TLB flush cleanups, moving toward PCID support Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
[RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
Re: [RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic Dave Hansen <dave.hansen@intel.com> - 2017-05-08 17:40 +0200
Re: [RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic Andy Lutomirski <luto@kernel.org> - 2017-05-09 15:10 +0200
Re: [RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic Mel Gorman <mgorman@suse.de> - 2017-05-09 16:50 +0200
Re: [RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic Dave Hansen <dave.hansen@intel.com> - 2017-05-09 19:20 +0200
Re: [RFC 03/10] x86/mm: Make the batched unmap TLB flush API more generic Andy Lutomirski <luto@kernel.org> - 2017-05-10 01:00 +0200
[RFC 07/10] x86/mm: Use new merged flush logic in arch_tlbbatch_flush() Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
[RFC 04/10] x86/mm: Pass flush_tlb_info to flush_tlb_others() etc Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
Re: [RFC 04/10] x86/mm: Pass flush_tlb_info to flush_tlb_others() etc Nadav Amit <nadav.amit@gmail.com> - 2017-05-11 22:10 +0200
[RFC 05/10] x86/mm: Change the leave_mm() condition for local TLB flushes Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
[RFC 02/10] x86/mm: Reduce indentation in flush_tlb_func() Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
[RFC 08/10] x86/mm: Remove the UP tlbflush code; always use the formerly SMP code Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
[RFC 01/10] x86/mm: Reimplement flush_tlb_page() using flush_tlb_mm_range() Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:10 +0200
Re: [RFC 01/10] x86/mm: Reimplement flush_tlb_page() using flush_tlb_mm_range() Borislav Petkov <bp@suse.de> - 2017-05-11 19:50 +0200
[RFC 10/10] x86,kvm: Teach KVM's VMX code that CR3 isn't a constant Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:20 +0200
[RFC 06/10] x86/mm: Refactor flush_tlb_mm_range() to merge local and remote cases Andy Lutomirski <luto@kernel.org> - 2017-05-08 00:20 +0200
Re: [RFC 00/10] x86 TLB flush cleanups, moving toward PCID support Linus Torvalds <torvalds@linux-foundation.org> - 2017-05-08 01:00 +0200
Re: [RFC 00/10] x86 TLB flush cleanups, moving toward PCID support Ingo Molnar <mingo@kernel.org> - 2017-05-08 01:50 +0200
Re: [RFC 00/10] x86 TLB flush cleanups, moving toward PCID support Nadav Amit <nadav.amit@gmail.com> - 2017-05-08 18:40 +0200
Re: [RFC 00/10] x86 TLB flush cleanups, moving toward PCID support Andy Lutomirski <luto@kernel.org> - 2017-05-09 14:50 +0200
csiph-web