Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1601976

[PATCH v4 02/13] mm, devm_memremap_pages: use multi-order radix for ZONE_DEVICE lookups

Show all headers | View raw

devm_memremap_pages() records mapped ranges in pgmap_radix with an entry
per section's worth of memory (128MB).  The key for each of those
entries is a section number.

This leads to false positives when devm_memremap_pages() is passed a
section-unaligned range as lookups in the misalignment fail to return
NULL. We can close this hole by using the pfn as the key for entries in
the tree.  The number of entries required to describe a remapped range
is reduced by leveraging multi-order entries.

In practice this approach usually yields just one entry in the tree if
the size and starting address are of the same power-of-2 alignment.
Previously we always needed nr_entries = mapping_size / 128MB.

Link: https://lists.01.org/pipermail/linux-nvdimm/2016-August/006666.html
Reported-by: Toshi Kani <toshi.kani@hpe.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
 kernel/memremap.c |   52 ++++++++++++++++++++++++++++++++++++++--------------
 mm/Kconfig        |    1 +
 2 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/kernel/memremap.c b/kernel/memremap.c
index 07e85e5229da..72e93754c0f4 100644
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -194,18 +194,41 @@ void put_zone_device_page(struct page *page)
 }
 EXPORT_SYMBOL(put_zone_device_page);
 
-static void pgmap_radix_release(struct resource *res)
+static unsigned long order_at(struct resource *res, unsigned long pgoff)
 {
-	resource_size_t key, align_start, align_size, align_end;
+	unsigned long phys_pgoff = PHYS_PFN(res->start) + pgoff;
+	unsigned long nr_pages, mask;
 
-	align_start = res->start & ~(SECTION_SIZE - 1);
-	align_size = ALIGN(resource_size(res), SECTION_SIZE);
-	align_end = align_start + align_size - 1;
+	nr_pages = PHYS_PFN(resource_size(res));
+	if (nr_pages == pgoff)
+		return ULONG_MAX;
+
+	/*
+	 * What is the largest aligned power-of-2 range available from
+	 * this resource pgoff to the end of the resource range,
+	 * considering the alignment of the current pgoff?
+	 */
+	mask = phys_pgoff | rounddown_pow_of_two(nr_pages - pgoff);
+	if (!mask)
+		return ULONG_MAX;
+
+	return find_first_bit(&mask, BITS_PER_LONG);
+}
+
+#define foreach_order_pgoff(res, order, pgoff) \
+	for (pgoff = 0, order = order_at((res), pgoff); order < ULONG_MAX; \
+			pgoff += 1UL << order, order = order_at((res), pgoff))
+
+static void pgmap_radix_release(struct resource *res)
+{
+	unsigned long pgoff, order;
 
 	mutex_lock(&pgmap_lock);
-	for (key = res->start; key <= res->end; key += SECTION_SIZE)
-		radix_tree_delete(&pgmap_radix, key >> PA_SECTION_SHIFT);
+	foreach_order_pgoff(res, order, pgoff)
+		radix_tree_delete(&pgmap_radix, PHYS_PFN(res->start) + pgoff);
 	mutex_unlock(&pgmap_lock);
+
+	synchronize_rcu();
 }
 
 static unsigned long pfn_first(struct page_map *page_map)
@@ -264,7 +287,7 @@ struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
 
 	WARN_ON_ONCE(!rcu_read_lock_held());
 
-	page_map = radix_tree_lookup(&pgmap_radix, phys >> PA_SECTION_SHIFT);
+	page_map = radix_tree_lookup(&pgmap_radix, PHYS_PFN(phys));
 	return page_map ? &page_map->pgmap : NULL;
 }
 
@@ -286,12 +309,12 @@ struct dev_pagemap *find_dev_pagemap(resource_size_t phys)
 void *devm_memremap_pages(struct device *dev, struct resource *res,
 		struct percpu_ref *ref, struct vmem_altmap *altmap)
 {
-	resource_size_t key, align_start, align_size, align_end;
+	resource_size_t align_start, align_size, align_end;
+	unsigned long pfn, pgoff, order;
 	pgprot_t pgprot = PAGE_KERNEL;
 	struct dev_pagemap *pgmap;
 	struct page_map *page_map;
 	int error, nid, is_ram;
-	unsigned long pfn;
 
 	align_start = res->start & ~(SECTION_SIZE - 1);
 	align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
@@ -330,11 +353,12 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
 	mutex_lock(&pgmap_lock);
 	error = 0;
 	align_end = align_start + align_size - 1;
-	for (key = align_start; key <= align_end; key += SECTION_SIZE) {
+
+	foreach_order_pgoff(res, order, pgoff) {
 		struct dev_pagemap *dup;
 
 		rcu_read_lock();
-		dup = find_dev_pagemap(key);
+		dup = find_dev_pagemap(res->start + PFN_PHYS(pgoff));
 		rcu_read_unlock();
 		if (dup) {
 			dev_err(dev, "%s: %pr collides with mapping for %s\n",
@@ -342,8 +366,8 @@ void *devm_memremap_pages(struct device *dev, struct resource *res,
 			error = -EBUSY;
 			break;
 		}
-		error = radix_tree_insert(&pgmap_radix, key >> PA_SECTION_SHIFT,
-				page_map);
+		error = __radix_tree_insert(&pgmap_radix,
+				PHYS_PFN(res->start) + pgoff, order, page_map);
 		if (error) {
 			dev_err(dev, "%s: failed: %d\n", __func__, error);
 			break;
diff --git a/mm/Kconfig b/mm/Kconfig
index 9b8fccb969dc..4aff67e042f1 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -690,6 +690,7 @@ config ZONE_DEVICE
 	depends on MEMORY_HOTREMOVE
 	depends on SPARSEMEM_VMEMMAP
 	depends on X86_64 #arch_add_memory() comprehends device memory
+	select RADIX_TREE_MULTIORDER
 
 	help
 	  Device memory hotplug support allows for establishing pmem,

Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread

Thread

[PATCH v4 00/13] mm: sub-section memory hotplug support Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 07/13] mm: fix register_new_memory() zone type detection Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 10/13] mm: prepare for hot-{add,  remove} of sub-section ranges Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 06/13] mm: track active portions of a section at boot Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 12/13] mm: enable section-unaligned devm_memremap_pages() Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 11/13] mm: support section-unaligned ZONE_DEVICE memory  ranges Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 05/13] mm: cleanup sparse_init_one_section() return value Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 01/13] mm: fix type width of section to/from pfn  conversion macros Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 09/13] mm: convert kmalloc_section_memmap() to  populate_section_memmap() Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 02/13] mm,  devm_memremap_pages: use multi-order radix for ZONE_DEVICE lookups Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 03/13] mm: introduce struct mem_section_usage to track  partial population of a section Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 04/13] mm: introduce common definitions for the size and  mask of a section Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 13/13] libnvdimm, pfn,  dax: stop padding pmem namespaces to section alignment Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  [PATCH v4 08/13] x86,  kasan: clarify kasan's dependency on vmemmap_populate_hugepages() Dan Williams <dan.j.williams@intel.com> - 2017-03-16 07:20 +0100
  Re: [PATCH v4 00/13] mm: sub-section memory hotplug support Michal Hocko <mhocko@kernel.org> - 2017-03-16 18:50 +0100
    Re: [PATCH v4 00/13] mm: sub-section memory hotplug support Dan Williams <dan.j.williams@intel.com> - 2017-03-16 20:10 +0100
      Re: [PATCH v4 00/13] mm: sub-section memory hotplug support Michal Hocko <mhocko@kernel.org> - 2017-03-19 18:00 +0100

csiph-web