Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1265944

[PATCH 3/3] x86, ras: Add mcsafe_memcpy() function to recover from machine checks

From Tony Luck <tony.luck@intel.com>
Newsgroups linux.kernel
Subject [PATCH 3/3] x86, ras: Add mcsafe_memcpy() function to recover from machine checks
Date 2015-11-09 19:50 +0100
Message-ID <qsZFU-1HI-29@gated-at.bofh.it> (permalink)
References <qsZFT-1HI-9@gated-at.bofh.it>
Organization linux.* mail to news gateway

Show all headers | View raw

Using __copy_user_nocache() as inspiration create a memory copy
routine for use by kernel code with annotations to allow for
recovery from machine checks.

Notes:
1) Unlike the original we make no attempt to copy all the bytes
   up to the faulting address. The original achieves that by
   re-executing the failing part as a byte-by-byte copy,
   which will take another page fault. We don't want to have
   a second machine check!
2) Likewise the return value for the original indicates exactly
   how many bytes were not copied. Instead we provide the physical
   address of the fault (thanks to help from do_machine_check()

Signed-off-by: Tony Luck <tony.luck@intel.com>
---
 arch/x86/include/asm/uaccess_64.h |  3 ++
 arch/x86/kernel/x8664_ksyms_64.c  |  2 +
 arch/x86/lib/copy_user_64.S       | 91 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+)

diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index f2f9b39b274a..79517954e652 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -216,6 +216,9 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
 extern long __copy_user_nocache(void *dst, const void __user *src,
 				unsigned size, int zerorest);
 
+extern phys_addr_t mcsafe_memcpy(void *dst, const void __user *src,
+				unsigned size);
+
 static inline int
 __copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
 {
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index a0695be19864..ec988c92c055 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -37,6 +37,8 @@ EXPORT_SYMBOL(__copy_user_nocache);
 EXPORT_SYMBOL(_copy_from_user);
 EXPORT_SYMBOL(_copy_to_user);
 
+EXPORT_SYMBOL(mcsafe_memcpy);
+
 EXPORT_SYMBOL(copy_page);
 EXPORT_SYMBOL(clear_page);
 
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index 982ce34f4a9b..ffce93cbc9a5 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -319,3 +319,94 @@ ENTRY(__copy_user_nocache)
 	_ASM_EXTABLE(21b,50b)
 	_ASM_EXTABLE(22b,50b)
 ENDPROC(__copy_user_nocache)
+
+/*
+ * mcsafe_memcpy - Uncached memory copy with machine check exception handling
+ * Note that we only catch machine checks when reading the source addresses.
+ * Writes to target are posted and don't generate machine checks.
+ * This will force destination/source out of cache for more performance.
+ */
+ENTRY(mcsafe_memcpy)
+	cmpl $8,%edx
+	jb 20f		/* less then 8 bytes, go to byte copy loop */
+
+	/* check for bad alignment of destination */
+	movl %edi,%ecx
+	andl $7,%ecx
+	jz 102f				/* already aligned */
+	subl $8,%ecx
+	negl %ecx
+	subl %ecx,%edx
+0:	movb (%rsi),%al
+	movb %al,(%rdi)
+	incq %rsi
+	incq %rdi
+	decl %ecx
+	jnz 100b
+102:
+	movl %edx,%ecx
+	andl $63,%edx
+	shrl $6,%ecx
+	jz 17f
+1:	movq (%rsi),%r8
+2:	movq 1*8(%rsi),%r9
+3:	movq 2*8(%rsi),%r10
+4:	movq 3*8(%rsi),%r11
+	movnti %r8,(%rdi)
+	movnti %r9,1*8(%rdi)
+	movnti %r10,2*8(%rdi)
+	movnti %r11,3*8(%rdi)
+9:	movq 4*8(%rsi),%r8
+10:	movq 5*8(%rsi),%r9
+11:	movq 6*8(%rsi),%r10
+12:	movq 7*8(%rsi),%r11
+	movnti %r8,4*8(%rdi)
+	movnti %r9,5*8(%rdi)
+	movnti %r10,6*8(%rdi)
+	movnti %r11,7*8(%rdi)
+	leaq 64(%rsi),%rsi
+	leaq 64(%rdi),%rdi
+	decl %ecx
+	jnz 1b
+17:	movl %edx,%ecx
+	andl $7,%edx
+	shrl $3,%ecx
+	jz 20f
+18:	movq (%rsi),%r8
+	movnti %r8,(%rdi)
+	leaq 8(%rsi),%rsi
+	leaq 8(%rdi),%rdi
+	decl %ecx
+	jnz 18b
+20:	andl %edx,%edx
+	jz 23f
+	movl %edx,%ecx
+21:	movb (%rsi),%al
+	movb %al,(%rdi)
+	incq %rsi
+	incq %rdi
+	decl %ecx
+	jnz 21b
+23:	xorl %eax,%eax
+	sfence
+	ret
+
+	.section .fixup,"ax"
+30:
+	sfence
+	/* do_machine_check() sets %eax return value */
+	ret
+	.previous
+
+	_ASM_MCEXTABLE(0b,30b)
+	_ASM_MCEXTABLE(1b,30b)
+	_ASM_MCEXTABLE(2b,30b)
+	_ASM_MCEXTABLE(3b,30b)
+	_ASM_MCEXTABLE(4b,30b)
+	_ASM_MCEXTABLE(9b,30b)
+	_ASM_MCEXTABLE(10b,30b)
+	_ASM_MCEXTABLE(11b,30b)
+	_ASM_MCEXTABLE(12b,30b)
+	_ASM_MCEXTABLE(18b,30b)
+	_ASM_MCEXTABLE(21b,30b)
+ENDPROC(mcsafe_memcpy)
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[RFC PATCH 0/3] Machine check recovery when kernel accesses poison Tony Luck <tony.luck@intel.com> - 2015-11-09 19:50 +0100
  [PATCH 2/3] x86, ras: Extend machine check recovery code to annotated  ring0 areas Tony Luck <tony.luck@intel.com> - 2015-11-09 19:50 +0100
    Re: [PATCH 2/3] x86, ras: Extend machine check recovery code to  annotated ring0 areas Borislav Petkov <bp@alien8.de> - 2015-11-10 12:30 +0100
      Re: [PATCH 2/3] x86, ras: Extend machine check recovery code to  annotated ring0 areas "Luck, Tony" <tony.luck@intel.com> - 2015-11-10 23:20 +0100
        Re: [PATCH 2/3] x86, ras: Extend machine check recovery code to  annotated ring0 areas Borislav Petkov <bp@alien8.de> - 2015-11-11 12:10 +0100
    Re: [PATCH 2/3] x86, ras: Extend machine check recovery code to  annotated ring0 areas Andy Lutomirski <luto@kernel.org> - 2015-11-12 05:30 +0100
      Re: [PATCH 2/3] x86, ras: Extend machine check recovery code to  annotated ring0 areas "Luck, Tony" <tony.luck@intel.com> - 2015-11-12 21:00 +0100
  [PATCH 3/3] x86, ras: Add mcsafe_memcpy() function to recover from  machine checks Tony Luck <tony.luck@intel.com> - 2015-11-09 19:50 +0100
    Re: [PATCH 3/3] x86, ras: Add mcsafe_memcpy() function to recover  from machine checks Ingo Molnar <mingo@kernel.org> - 2015-11-12 09:00 +0100
      Re: [PATCH 3/3] x86, ras: Add mcsafe_memcpy() function to recover  from machine checks "Luck, Tony" <tony.luck@intel.com> - 2015-11-12 21:10 +0100
  Re: [RFC PATCH 0/3] Machine check recovery when kernel accesses  poison Borislav Petkov <bp@alien8.de> - 2015-11-10 12:30 +0100
    Re: [RFC PATCH 0/3] Machine check recovery when kernel accesses  poison "Luck, Tony" <tony.luck@intel.com> - 2015-11-10 23:00 +0100
      Re: [RFC PATCH 0/3] Machine check recovery when kernel accesses  poison Borislav Petkov <bp@alien8.de> - 2015-11-11 21:50 +0100
        Re: [RFC PATCH 0/3] Machine check recovery when kernel accesses  poison "Luck, Tony" <tony.luck@intel.com> - 2015-11-11 22:50 +0100
          Re: [RFC PATCH 0/3] Machine check recovery when kernel accesses  poison Borislav Petkov <bp@alien8.de> - 2015-11-11 23:30 +0100
            RE: [RFC PATCH 0/3] Machine check recovery when kernel accesses  poison "Luck, Tony" <tony.luck@intel.com> - 2015-11-11 23:40 +0100

csiph-web