Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1308101 > unrolled thread

[PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX

Started byAndrew Pinski <apinski@cavium.com>
First post2016-01-13 08:10 +0100
Last post2016-01-13 08:20 +0100
Articles 5 — 1 participant

Back to article view | Back to linux.kernel

Contents

  [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX Andrew Pinski <apinski@cavium.com> - 2016-01-13 08:10 +0100
    [PATCH 1/5] ARM64: Support midr detected cpufeature Andrew Pinski <apinski@cavium.com> - 2016-01-13 08:10 +0100
    [PATCH 2/5] ARM64 Improve copy_page for 128 byte cache line Andrew Pinski <apinski@cavium.com> - 2016-01-13 08:20 +0100
    [PATCH 5/5] ARM64: Patch in prefetching in copy_template Andrew Pinski <apinski@cavium.com> - 2016-01-13 08:20 +0100
    [PATCH 3/5] ARM64: Add Needs software prefetching cap Andrew Pinski <apinski@cavium.com> - 2016-01-13 08:20 +0100

#1308101 — [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX

FromAndrew Pinski <apinski@cavium.com>
Date2016-01-13 08:10 +0100
Subject[PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX
Message-ID<qQnJ7-4Cb-3@gated-at.bofh.it>
This is version 2.  I expanded it to include copy_template as it was a simple
extension to the new version of the patch.
Changes since v1:
1) Add infastructure for checking MIDR for features
2) Use the copy_page version from Will Deacon 
3) Add prefetching feature
4) Add prefetching to copy_page
5) Add prefetching to copy_template


Andrew Pinski (5):
  ARM64: Support midr detected cpufeature
  ARM64 Improve copy_page for 128 byte cache line
  ARM64: Add Needs software prefetching cap
  ARM64: Patch in prefetching for copy_page is requested
  ARM64: Patch in prefetching in copy_template

 arch/arm64/include/asm/cpufeature.h |   29 ++++++++++++++--
 arch/arm64/include/asm/cputype.h    |    7 ++++
 arch/arm64/kernel/cpu_errata.c      |   26 --------------
 arch/arm64/kernel/cpufeature.c      |   27 ++++++++++++++-
 arch/arm64/lib/copy_page.S          |   64 ++++++++++++++++++++++++++++++----
 arch/arm64/lib/copy_template.S      |   12 ++++++
 arch/arm64/lib/memcpy.S             |    2 +
 7 files changed, 129 insertions(+), 38 deletions(-)

-- 
1.7.2.5

[toc] | [next] | [standalone]

#1308106 — [PATCH 1/5] ARM64: Support midr detected cpufeature

FromAndrew Pinski <apinski@cavium.com>
Date2016-01-13 08:10 +0100
Subject[PATCH 1/5] ARM64: Support midr detected cpufeature
Message-ID<qQnJ8-4Cb-25@gated-at.bofh.it>
In reply to#1308101
A soon to be added feature is marking the need for
software prefetching of 128 byte cache line.
This feature is not detected via bits of a system register
but rather matching of the Machine ID register.
This adds support for detecting a cpufeature by the MIDR.

Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
 arch/arm64/include/asm/cpufeature.h |   26 ++++++++++++++++++++++++--
 arch/arm64/include/asm/cputype.h    |    7 +++++++
 arch/arm64/kernel/cpu_errata.c      |   26 --------------------------
 arch/arm64/kernel/cpufeature.c      |   21 ++++++++++++++++++++-
 4 files changed, 51 insertions(+), 29 deletions(-)

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 8136afc..92aaac6 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -11,6 +11,7 @@
 
 #include <asm/hwcap.h>
 #include <asm/sysreg.h>
+#include <asm/cputype.h>
 
 /*
  * In the arm64 world (as in the ARM world), elf_hwcap is used both internally
@@ -78,14 +79,14 @@ struct arm64_cpu_capabilities {
 	u16 capability;
 	bool (*matches)(const struct arm64_cpu_capabilities *);
 	void (*enable)(void *);		/* Called on all active CPUs */
+	u32 sys_reg;
 	union {
-		struct {	/* To be used for erratum handling only */
+		struct {	/* MIDR matching used if sys_reg is SYS_MIDR_EL1. */
 			u32 midr_model;
 			u32 midr_range_min, midr_range_max;
 		};
 
 		struct {	/* Feature register checking */
-			u32 sys_reg;
 			int field_pos;
 			int min_field_value;
 			int hwcap_type;
@@ -94,6 +95,27 @@ struct arm64_cpu_capabilities {
 	};
 };
 
+static bool __maybe_unused
+is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
+{
+		u32 midr = read_cpuid_id();
+
+		if ((midr & CPU_MODEL_MASK) != entry->midr_model)
+				return false;
+
+		midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
+
+		return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
+}
+
+#define MIDR_RANGE(model, min, max) \
+		.matches = is_affected_midr_range, \
+		.sys_reg = SYS_MIDR_EL1, \
+		.midr_model = model, \
+		.midr_range_min = min, \
+		.midr_range_max = max
+
+
 extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
 
 static inline bool cpu_have_feature(unsigned int num)
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index 1a59493..cd99d28a 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -75,6 +75,13 @@
 
 #define CAVIUM_CPU_PART_THUNDERX	0x0A1
 
+#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
+#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_THUNDERX   MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+
+#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
+                                                MIDR_ARCHITECTURE_MASK)
+
 #ifndef __ASSEMBLY__
 
 /*
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 01248e8..cec63a9 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -21,32 +21,6 @@
 #include <asm/cputype.h>
 #include <asm/cpufeature.h>
 
-#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
-#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
-#define MIDR_THUNDERX	MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
-
-#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
-			MIDR_ARCHITECTURE_MASK)
-
-static bool __maybe_unused
-is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
-{
-	u32 midr = read_cpuid_id();
-
-	if ((midr & CPU_MODEL_MASK) != entry->midr_model)
-		return false;
-
-	midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
-
-	return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
-}
-
-#define MIDR_RANGE(model, min, max) \
-	.matches = is_affected_midr_range, \
-	.midr_model = model, \
-	.midr_range_min = min, \
-	.midr_range_max = max
-
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #if	defined(CONFIG_ARM64_ERRATUM_826319) || \
 	defined(CONFIG_ARM64_ERRATUM_827319) || \
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 0669c63..b0ee60e 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -869,8 +869,27 @@ void verify_local_cpu_capabilities(void)
 
 	caps = arm64_features;
 	for (i = 0; caps[i].desc; i++) {
-		if (!cpus_have_cap(caps[i].capability) || !caps[i].sys_reg)
+		/*
+		 * If the feature is not enable already, then don't try to
+		 * enable.
+		 */
+		if (!cpus_have_cap(caps[i].capability))
+			continue;
+		if (!caps[i].sys_reg)
+			continue;
+		/* Handle MIDR matching seperately. */
+		if (caps[i].sys_reg == SYS_MIDR_EL1) {
+			/*
+			 * If the new CPU is a different MIDR it means the
+			 * feature is missing, we cannot proceed further,
+			 * park the cpu.
+			 */
+			if (!is_affected_midr_range (&caps[i]))
+				fail_incapable_cpu("arm64_features", &caps[i]);
+			if (caps[i].enable)
+				caps[i].enable(NULL);
 			continue;
+		}
 		/*
 		 * If the new CPU misses an advertised feature, we cannot proceed
 		 * further, park the cpu.
-- 
1.7.2.5

[toc] | [prev] | [next] | [standalone]

#1308110 — [PATCH 2/5] ARM64 Improve copy_page for 128 byte cache line

FromAndrew Pinski <apinski@cavium.com>
Date2016-01-13 08:20 +0100
Subject[PATCH 2/5] ARM64 Improve copy_page for 128 byte cache line
Message-ID<qQnSO-4FD-1@gated-at.bofh.it>
In reply to#1308101
For 128 byte cache line, doing 128 bytes unrolled in
the loop is better.
This is adapted from:
https://lkml.org/lkml/2016/1/6/497

Note this removes prefetching as it is harmful for
processors that includes hardware prefetching.
Note the next patch includes patching in software
prefetching for one target.

Signed-off-by: Andrew Pinski <apinski@cavium.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 arch/arm64/lib/copy_page.S |   47 ++++++++++++++++++++++++++++++++++++-------
 1 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S
index 512b9a7..dfb0316 100644
--- a/arch/arm64/lib/copy_page.S
+++ b/arch/arm64/lib/copy_page.S
@@ -19,6 +19,7 @@
 #include <asm/assembler.h>
 #include <asm/page.h>
 
+
 /*
  * Copy a page from src to dest (both are page aligned)
  *
@@ -27,20 +28,50 @@
  *	x1 - src
  */
 ENTRY(copy_page)
-	/* Assume cache line size is 64 bytes. */
-	prfm	pldl1strm, [x1, #64]
-1:	ldp	x2, x3, [x1]
+	ldp	x2, x3, [x1]
+	ldp	x4, x5, [x1, #16]
+	ldp	x6, x7, [x1, #32]
+	ldp	x8, x9, [x1, #48]
+	ldp	x10, x11, [x1, #64]
+	ldp	x12, x13, [x1, #80]
+	ldp	x14, x15, [x1, #96]
+	ldp	x16, x17, [x1, #112]
+
+	mov	x18, #(PAGE_SIZE - 128)
+	add	x1, x1, #128
+1:
+	subs	x18, x18, #128
+
+	stnp	x2, x3, [x0]
+	ldp	x2, x3, [x1]
+	stnp	x4, x5, [x0, #16]
 	ldp	x4, x5, [x1, #16]
+	stnp	x6, x7, [x0, #32]
 	ldp	x6, x7, [x1, #32]
+	stnp	x8, x9, [x0, #48]
 	ldp	x8, x9, [x1, #48]
-	add	x1, x1, #64
-	prfm	pldl1strm, [x1, #64]
+	stnp	x10, x11, [x0, #64]
+	ldp	x10, x11, [x1, #64]
+	stnp	x12, x13, [x0, #80]
+	ldp	x12, x13, [x1, #80]
+	stnp	x14, x15, [x0, #96]
+	ldp	x14, x15, [x1, #96]
+	stnp	x16, x17, [x0, #112]
+	ldp	x16, x17, [x1, #112]
+
+	add	x0, x0, #128
+	add	x1, x1, #128
+
+	b.gt	1b
+
 	stnp	x2, x3, [x0]
 	stnp	x4, x5, [x0, #16]
 	stnp	x6, x7, [x0, #32]
 	stnp	x8, x9, [x0, #48]
-	add	x0, x0, #64
-	tst	x1, #(PAGE_SIZE - 1)
-	b.ne	1b
+	stnp	x10, x11, [x0, #64]
+	stnp	x12, x13, [x0, #80]
+	stnp	x14, x15, [x0, #96]
+	stnp	x16, x17, [x0, #112]
+
 	ret
 ENDPROC(copy_page)
-- 
1.7.2.5

[toc] | [prev] | [next] | [standalone]

#1308111 — [PATCH 5/5] ARM64: Patch in prefetching in copy_template

FromAndrew Pinski <apinski@cavium.com>
Date2016-01-13 08:20 +0100
Subject[PATCH 5/5] ARM64: Patch in prefetching in copy_template
Message-ID<qQnSO-4FD-7@gated-at.bofh.it>
In reply to#1308101
For ThunderX T88 pass 1.x and 2.x where there is
no hardware prefetcher, we want to patch in
software prefetching instructions in the copy_template.
This speeds up copy_to_user and copy_from_user for large
size.  The main use of large sizes is I/O read/writes.

Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
 arch/arm64/lib/copy_template.S |   12 ++++++++++++
 arch/arm64/lib/memcpy.S        |    2 ++
 2 files changed, 14 insertions(+), 0 deletions(-)

diff --git a/arch/arm64/lib/copy_template.S b/arch/arm64/lib/copy_template.S
index 410fbdb..3f3f0a4 100644
--- a/arch/arm64/lib/copy_template.S
+++ b/arch/arm64/lib/copy_template.S
@@ -163,12 +163,24 @@ D_h	.req	x14
 	*/
 	.p2align	L1_CACHE_SHIFT
 .Lcpy_body_large:
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+	nop
+	nop
+alternative_else
+	prfm	pldl1strm, [src, #128]
+	prfm	pldl1strm, [src, #256]
+alternative_endif
 	/* pre-get 64 bytes data. */
 	ldp1	A_l, A_h, src, #16
 	ldp1	B_l, B_h, src, #16
 	ldp1	C_l, C_h, src, #16
 	ldp1	D_l, D_h, src, #16
 1:
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+	nop
+alternative_else
+	prfm	pldl1strm, [src, #384]
+alternative_endif
 	/*
 	* interlace the load of next 64 bytes data block with store of the last
 	* loaded 64 bytes data.
diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S
index 6761393..3a50cf8b 100644
--- a/arch/arm64/lib/memcpy.S
+++ b/arch/arm64/lib/memcpy.S
@@ -25,6 +25,8 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/cache.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
 
 /*
  * Copy a buffer from src to dest (alignment handled by the hardware)
-- 
1.7.2.5

[toc] | [prev] | [next] | [standalone]

#1308116 — [PATCH 3/5] ARM64: Add Needs software prefetching cap

FromAndrew Pinski <apinski@cavium.com>
Date2016-01-13 08:20 +0100
Subject[PATCH 3/5] ARM64: Add Needs software prefetching cap
Message-ID<qQnSO-4FD-19@gated-at.bofh.it>
In reply to#1308101
On ThunderX T88 pass 1 and pass 2 does not include
a hardware prefetching so the memory copying
functions are slower than necessary.  This adds
a cpu feature for this.
The next patch will add the use in both copy_page
and copy_from_user/copy_to_user.

Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
 arch/arm64/include/asm/cpufeature.h |    3 ++-
 arch/arm64/kernel/cpufeature.c      |    6 ++++++
 2 files changed, 8 insertions(+), 1 deletions(-)

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 92aaac6..784bc63 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -32,8 +32,9 @@
 #define ARM64_WORKAROUND_CAVIUM_23154		6
 #define ARM64_WORKAROUND_834220			7
 #define ARM64_WORKAROUND_CAVIUM_27456		8
+#define ARM64_NEEDS_PREFETCH_128		9
 
-#define ARM64_NCAPS				9
+#define ARM64_NCAPS				10
 
 #ifndef __ASSEMBLY__
 
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index b0ee60e..13cda09 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -651,6 +651,12 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.min_field_value = 2,
 	},
 #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
+	{
+		.desc = "Needs prefetching (128 byte cache line) for copying: ThunderX T88 pass 1.x and 2.x",
+		.capability = ARM64_NEEDS_PREFETCH_128,
+		MIDR_RANGE(MIDR_THUNDERX, 0,
+			   (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK),
+	},
 	{},
 };
 
-- 
1.7.2.5

[toc] | [prev] | [standalone]

Back to top | Article view | linux.kernel

csiph-web