Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1416038

[PATCHv9-rebased 18/32] radix-tree: implement radix_tree_maybe_preload_order()

From "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Newsgroups linux.kernel
Subject [PATCHv9-rebased 18/32] radix-tree: implement radix_tree_maybe_preload_order()
Date 2016-06-07 13:20 +0200
Message-ID <rHnd8-1YN-27@gated-at.bofh.it> (permalink)
References <rH3o5-5Yv-3@gated-at.bofh.it> <rHn3r-1Vt-11@gated-at.bofh.it>
Organization linux.* mail to news gateway

Show all headers | View raw

The new helper is similar to radix_tree_maybe_preload(), but tries to
preload number of nodes required to insert (1 << order) continuous
naturally-aligned elements.

This is required to push huge pages into pagecache.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
---
 include/linux/radix-tree.h |  1 +
 lib/radix-tree.c           | 84 +++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 80 insertions(+), 5 deletions(-)

diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h
index cb4b7e8cee81..084965cf90fc 100644
--- a/include/linux/radix-tree.h
+++ b/include/linux/radix-tree.h
@@ -291,6 +291,7 @@ unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
 			unsigned long first_index, unsigned int max_items);
 int radix_tree_preload(gfp_t gfp_mask);
 int radix_tree_maybe_preload(gfp_t gfp_mask);
+int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order);
 void radix_tree_init(void);
 void *radix_tree_tag_set(struct radix_tree_root *root,
 			unsigned long index, unsigned int tag);
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 8b7d8459bb9d..61b8fb529cef 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -38,6 +38,9 @@
 #include <linux/preempt.h>		/* in_interrupt() */
 
 
+/* Number of nodes in fully populated tree of given height */
+static unsigned long height_to_maxnodes[RADIX_TREE_MAX_PATH + 1] __read_mostly;
+
 /*
  * Radix tree node cache.
  */
@@ -342,7 +345,7 @@ radix_tree_node_free(struct radix_tree_node *node)
  * To make use of this facility, the radix tree must be initialised without
  * __GFP_DIRECT_RECLAIM being passed to INIT_RADIX_TREE().
  */
-static int __radix_tree_preload(gfp_t gfp_mask)
+static int __radix_tree_preload(gfp_t gfp_mask, int nr)
 {
 	struct radix_tree_preload *rtp;
 	struct radix_tree_node *node;
@@ -350,14 +353,14 @@ static int __radix_tree_preload(gfp_t gfp_mask)
 
 	preempt_disable();
 	rtp = this_cpu_ptr(&radix_tree_preloads);
-	while (rtp->nr < RADIX_TREE_PRELOAD_SIZE) {
+	while (rtp->nr < nr) {
 		preempt_enable();
 		node = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
 		if (node == NULL)
 			goto out;
 		preempt_disable();
 		rtp = this_cpu_ptr(&radix_tree_preloads);
-		if (rtp->nr < RADIX_TREE_PRELOAD_SIZE) {
+		if (rtp->nr < nr) {
 			node->private_data = rtp->nodes;
 			rtp->nodes = node;
 			rtp->nr++;
@@ -383,7 +386,7 @@ int radix_tree_preload(gfp_t gfp_mask)
 {
 	/* Warn on non-sensical use... */
 	WARN_ON_ONCE(!gfpflags_allow_blocking(gfp_mask));
-	return __radix_tree_preload(gfp_mask);
+	return __radix_tree_preload(gfp_mask, RADIX_TREE_PRELOAD_SIZE);
 }
 EXPORT_SYMBOL(radix_tree_preload);
 
@@ -395,7 +398,7 @@ EXPORT_SYMBOL(radix_tree_preload);
 int radix_tree_maybe_preload(gfp_t gfp_mask)
 {
 	if (gfpflags_allow_blocking(gfp_mask))
-		return __radix_tree_preload(gfp_mask);
+		return __radix_tree_preload(gfp_mask, RADIX_TREE_PRELOAD_SIZE);
 	/* Preloading doesn't help anything with this gfp mask, skip it */
 	preempt_disable();
 	return 0;
@@ -403,6 +406,51 @@ int radix_tree_maybe_preload(gfp_t gfp_mask)
 EXPORT_SYMBOL(radix_tree_maybe_preload);
 
 /*
+ * The same as function above, but preload number of nodes required to insert
+ * (1 << order) continuous naturally-aligned elements.
+ */
+int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order)
+{
+	unsigned long nr_subtrees;
+	int nr_nodes, subtree_height;
+
+	/* Preloading doesn't help anything with this gfp mask, skip it */
+	if (!gfpflags_allow_blocking(gfp_mask)) {
+		preempt_disable();
+		return 0;
+	}
+
+	/*
+	 * Calculate number and height of fully populated subtrees it takes to
+	 * store (1 << order) elements.
+	 */
+	nr_subtrees = 1 << order;
+	for (subtree_height = 0; nr_subtrees > RADIX_TREE_MAP_SIZE;
+			subtree_height++)
+		nr_subtrees >>= RADIX_TREE_MAP_SHIFT;
+
+	/*
+	 * The worst case is zero height tree with a single item at index 0 and
+	 * then inserting items starting at ULONG_MAX - (1 << order).
+	 *
+	 * This requires RADIX_TREE_MAX_PATH nodes to build branch from root to
+	 * 0-index item.
+	 */
+	nr_nodes = RADIX_TREE_MAX_PATH;
+
+	/* Plus branch to fully populated subtrees. */
+	nr_nodes += RADIX_TREE_MAX_PATH - subtree_height;
+
+	/* Root node is shared. */
+	nr_nodes--;
+
+	/* Plus nodes required to build subtrees. */
+	nr_nodes += nr_subtrees * height_to_maxnodes[subtree_height];
+
+	return __radix_tree_preload(gfp_mask, nr_nodes);
+}
+
+/*
  * The maximum index which can be stored in a radix tree
  */
 static inline unsigned long shift_maxindex(unsigned int shift)
@@ -1571,6 +1619,31 @@ radix_tree_node_ctor(void *arg)
 	INIT_LIST_HEAD(&node->private_list);
 }
 
+static __init unsigned long __maxindex(unsigned int height)
+{
+	unsigned int width = height * RADIX_TREE_MAP_SHIFT;
+	int shift = RADIX_TREE_INDEX_BITS - width;
+
+	if (shift < 0)
+		return ~0UL;
+	if (shift >= BITS_PER_LONG)
+		return 0UL;
+	return ~0UL >> shift;
+}
+
+static __init void radix_tree_init_maxnodes(void)
+{
+	unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH + 1];
+	unsigned int i, j;
+
+	for (i = 0; i < ARRAY_SIZE(height_to_maxindex); i++)
+		height_to_maxindex[i] = __maxindex(i);
+	for (i = 0; i < ARRAY_SIZE(height_to_maxnodes); i++) {
+		for (j = i; j > 0; j--)
+			height_to_maxnodes[i] += height_to_maxindex[j - 1] + 1;
+	}
+}
+
 static int radix_tree_callback(struct notifier_block *nfb,
 				unsigned long action, void *hcpu)
 {
@@ -1597,5 +1670,6 @@ void __init radix_tree_init(void)
 			sizeof(struct radix_tree_node), 0,
 			SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
 			radix_tree_node_ctor);
+	radix_tree_init_maxnodes();
 	hotcpu_notifier(radix_tree_callback, 0);
 }
-- 
2.8.1

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[PATCHv9 00/32] THP-enabled tmpfs/shmem using compound pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-06 16:20 +0200
  [PATCHv9 11/32] thp: skip file huge pmd on copy_huge_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-06 16:20 +0200
  [PATCHv9 09/32] thp: handle file pages in split_huge_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-06 16:30 +0200
  [PATCHv9 06/32] mm: introduce do_set_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-06 16:30 +0200
  [PATCHv9 13/32] thp: run vma_adjust_trans_huge() outside i_mmap_rwsem "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-06 16:30 +0200
  [PATCHv9 04/32] mm: postpone page table allocation until we have page to map "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-06 16:30 +0200
  [PATCHv9-rebased 24/32] shmem: add huge pages support "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 25/32] shmem, thp: respect MADV_{NO,}HUGEPAGE for file mappings "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 06/32] mm: introduce do_set_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 16/32] vmscan: split file huge pages before paging them out "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 07/32] thp, vmstats: add counters for huge file pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 23/32] shmem: get_unmapped_area align huge page "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 19/32] filemap: prepare find and delete operations for huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 30/32] thp: introduce CONFIG_TRANSPARENT_HUGE_PAGECACHE "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 01/32] thp, mlock: update unevictable-lru.txt "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 13/32] thp: run vma_adjust_trans_huge() outside i_mmap_rwsem "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 04/32] mm: postpone page table allocation until we have page to map "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 28/32] shmem: make shmem_inode_info::lock irq-safe "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 32/32] thp: update Documentation/{vm/transhuge,filesystems/proc}.txt "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 27/32] khugepaged: move up_read(mmap_sem) out of khugepaged_alloc_page() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 10/32] thp: handle file COW faults "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 29/32] khugepaged: add support of collapse for tmpfs/shmem pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 31/32] shmem: split huge pages beyond i_size under memory pressure "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:10 +0200
  [PATCHv9-rebased 02/32] mm: do not pass mm_struct into handle_mm_fault "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
  [PATCHv9-rebased 21/32] mm, rmap: account shmem thp pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
  [PATCHv9-rebased 12/32] thp: prepare change_huge_pmd() for file thp "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
  [PATCHv9-rebased 08/32] thp: support file pages in zap_huge_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
  [PATCHv9-rebased 05/32] rmap: support file thp "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
  [PATCHv9-rebased 11/32] thp: skip file huge pmd on copy_huge_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
  [PATCHv9-rebased 00/32] THP-enabled tmpfs/shmem using compound pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 22/32] shmem: prepare huge= mount option and sysfs knob "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 15/32] thp, mlock: do not mlock PTE-mapped file huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 17/32] page-flags: relax policy for PG_mappedtodisk and PG_reclaim "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 20/32] truncate: handle file thp "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 18/32] radix-tree: implement radix_tree_maybe_preload_order() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 09/32] thp: handle file pages in split_huge_pmd() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200
    [PATCHv9-rebased 14/32] thp: file pages support for split_huge_page() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-06-07 13:20 +0200

csiph-web