Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1157902

[PATCHv6 29/36] thp: implement split_huge_pmd()

From "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Newsgroups linux.kernel
Subject [PATCHv6 29/36] thp: implement split_huge_pmd()
Date 2015-06-03 19:20 +0200
Message-ID <pxkuD-7Ey-33@gated-at.bofh.it> (permalink)
References <pxkkW-7sr-9@gated-at.bofh.it>
Organization linux.* mail to news gateway

Show all headers | View raw

Original split_huge_page() combined two operations: splitting PMDs into
tables of PTEs and splitting underlying compound page. This patch
implements split_huge_pmd() which split given PMD without splitting
other PMDs this page mapped with or underlying compound page.

Without tail page refcounting, implementation of split_huge_pmd() is
pretty straight-forward.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: Sasha Levin <sasha.levin@oracle.com>
---
 include/linux/huge_mm.h |  11 ++++-
 mm/huge_memory.c        | 119 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 129 insertions(+), 1 deletion(-)

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 67b3f3760f4a..9cfce23ab9d0 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -94,7 +94,16 @@ extern unsigned long transparent_hugepage_flags;
 
 #define split_huge_page_to_list(page, list) BUILD_BUG()
 #define split_huge_page(page) BUILD_BUG()
-#define split_huge_pmd(__vma, __pmd, __address) BUILD_BUG()
+
+void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
+		unsigned long address);
+
+#define split_huge_pmd(__vma, __pmd, __address)				\
+	do {								\
+		pmd_t *____pmd = (__pmd);				\
+		if (pmd_trans_huge(*____pmd))				\
+			__split_huge_pmd(__vma, __pmd, __address);	\
+	}  while (0)
 
 #if HPAGE_PMD_ORDER >= MAX_ORDER
 #error "hugepages can't be allocated by the buddy allocator"
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 5b0a13d2f28c..0f1f5731a893 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2547,6 +2547,125 @@ static int khugepaged(void *none)
 	return 0;
 }
 
+static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
+		unsigned long haddr, pmd_t *pmd)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	pgtable_t pgtable;
+	pmd_t _pmd;
+	int i;
+
+	/* leave pmd empty until pte is filled */
+	pmdp_huge_clear_flush_notify(vma, haddr, pmd);
+
+	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
+	pmd_populate(mm, &_pmd, pgtable);
+
+	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
+		pte_t *pte, entry;
+		entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
+		entry = pte_mkspecial(entry);
+		pte = pte_offset_map(&_pmd, haddr);
+		VM_BUG_ON(!pte_none(*pte));
+		set_pte_at(mm, haddr, pte, entry);
+		pte_unmap(pte);
+	}
+	smp_wmb(); /* make pte visible before pmd */
+	pmd_populate(mm, pmd, pgtable);
+	put_huge_zero_page();
+}
+
+static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
+		unsigned long haddr)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	struct page *page;
+	pgtable_t pgtable;
+	pmd_t _pmd;
+	bool young, write;
+	int i;
+
+	VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
+	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
+	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
+	VM_BUG_ON(!pmd_trans_huge(*pmd));
+
+	count_vm_event(THP_SPLIT_PMD);
+
+	if (is_huge_zero_pmd(*pmd))
+		return __split_huge_zero_page_pmd(vma, haddr, pmd);
+
+	page = pmd_page(*pmd);
+	VM_BUG_ON_PAGE(!page_count(page), page);
+	atomic_add(HPAGE_PMD_NR - 1, &page->_count);
+	write = pmd_write(*pmd);
+	young = pmd_young(*pmd);
+
+	/* leave pmd empty until pte is filled */
+	pmdp_huge_clear_flush_notify(vma, haddr, pmd);
+
+	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
+	pmd_populate(mm, &_pmd, pgtable);
+
+	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
+		pte_t entry, *pte;
+		/*
+		 * Note that NUMA hinting access restrictions are not
+		 * transferred to avoid any possibility of altering
+		 * permissions across VMAs.
+		 */
+		entry = mk_pte(page + i, vma->vm_page_prot);
+		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
+		if (!write)
+			entry = pte_wrprotect(entry);
+		if (!young)
+			entry = pte_mkold(entry);
+		pte = pte_offset_map(&_pmd, haddr);
+		BUG_ON(!pte_none(*pte));
+		set_pte_at(mm, haddr, pte, entry);
+		atomic_inc(&page[i]._mapcount);
+		pte_unmap(pte);
+	}
+
+	if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
+		/* Last compound_mapcount is gone. */
+		__dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
+		if (PageDoubleMap(page)) {
+			/* No need in mapcount reference anymore */
+			ClearPageDoubleMap(page);
+			for (i = 0; i < HPAGE_PMD_NR; i++)
+				atomic_dec(&page[i]._mapcount);
+		}
+	} else if (!TestSetPageDoubleMap(page)) {
+		/*
+		 * The first PMD split for the compound page and we still
+		 * have other PMD mapping of the page: bump _mapcount in
+		 * every small page.
+		 * This reference will go away with last compound_mapcount.
+		 */
+		for (i = 0; i < HPAGE_PMD_NR; i++)
+			atomic_inc(&page[i]._mapcount);
+	}
+
+	smp_wmb(); /* make pte visible before pmd */
+	pmd_populate(mm, pmd, pgtable);
+}
+
+void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
+		unsigned long address)
+{
+	spinlock_t *ptl;
+	struct mm_struct *mm = vma->vm_mm;
+	unsigned long haddr = address & HPAGE_PMD_MASK;
+
+	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
+	ptl = pmd_lock(mm, pmd);
+	if (likely(pmd_trans_huge(*pmd)))
+		__split_huge_pmd_locked(vma, pmd, haddr);
+	spin_unlock(ptl);
+	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
+}
+
 static void split_huge_pmd_address(struct vm_area_struct *vma,
 				    unsigned long address)
 {
-- 
2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[PATCHv6 00/36] THP refcounting redesign "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:10 +0200
  [PATCHv6 12/36] thp: drop all split_huge_page()-related code "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:10 +0200
  [PATCHv6 18/36] arm, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:10 +0200
  [PATCHv6 01/36] mm, proc: adjust PSS calculation "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:10 +0200
  [PATCHv6 28/36] mm, numa: skip PTE-mapped THP on numa fault "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:10 +0200
  [PATCHv6 02/36] rmap: add argument to charge compound page "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:10 +0200
  [PATCHv6 24/36] x86, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 08/36] khugepaged: ignore pmd tables with THP mapped with ptes "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 19/36] mips, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 33/36] migrate_pages: try to split pages on qeueuing "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 13/36] mm: drop tail page refcounting "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 27/36] mm: differentiate page_mapped() from page_mapcount() for compound pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 32/36] thp: reintroduce split_huge_page() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 21/36] s390, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 29/36] thp: implement split_huge_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 20/36] powerpc, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 17/36] arm64, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 34/36] thp: introduce deferred_split_huge_page() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 30/36] thp: add option to setup migration entiries during PMD split "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 10/36] mm, vmstats: new THP splitting event "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 23/36] tile, thp: remove infrastructure for handling splitting PMDs "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 06/36] mm: handle PTE-mapped tail pages in gerneric fast gup implementaiton "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 31/36] thp, mm: split_huge_page(): caller need to lock page "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 35/36] mm: re-enable THP "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200
  [PATCHv6 16/36] mm, thp: remove compound_lock "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2015-06-03 19:20 +0200

csiph-web