Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1602515
| From | Jérôme Glisse <jglisse@redhat.com> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [HMM 08/16] mm/migrate: migrate_vma() unmap page from vma while collecting pages |
| Date | 2017-03-16 16:20 +0100 |
| Message-ID | <tlFm3-cU-57@gated-at.bofh.it> (permalink) |
| References | <tlFm1-cU-3@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
Common case for migration of virtual address range is page are map
only once inside the vma in which migration is taking place. Because
we already walk the CPU page table for that range we can directly do
the unmap there and setup special migration swap entry.
Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
Signed-off-by: Evgeny Baskakov <ebaskakov@nvidia.com>
Signed-off-by: John Hubbard <jhubbard@nvidia.com>
Signed-off-by: Mark Hairgrove <mhairgrove@nvidia.com>
Signed-off-by: Sherry Cheung <SCheung@nvidia.com>
Signed-off-by: Subhash Gutti <sgutti@nvidia.com>
---
mm/migrate.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 95 insertions(+), 16 deletions(-)
diff --git a/mm/migrate.c b/mm/migrate.c
index e37d796..5a14b4ec 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2125,9 +2125,10 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
{
struct migrate_vma *migrate = walk->private;
struct mm_struct *mm = walk->vma->vm_mm;
- unsigned long addr = start;
+ unsigned long addr = start, unmapped = 0;
spinlock_t *ptl;
pte_t *ptep;
+ int ret = 0;
if (pmd_none(*pmdp) || pmd_trans_unstable(pmdp)) {
/* FIXME support THP */
@@ -2135,9 +2136,12 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
}
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
+ arch_enter_lazy_mmu_mode();
+
for (; addr < end; addr += PAGE_SIZE, ptep++) {
unsigned long flags, pfn;
struct page *page;
+ swp_entry_t entry;
pte_t pte;
int ret;
@@ -2170,17 +2174,50 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
flags = MIGRATE_PFN_VALID | MIGRATE_PFN_MIGRATE;
flags |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
+ /*
+ * Optimize for the common case where page is only mapped once
+ * in one process. If we can lock the page, then we can safely
+ * set up a special migration page table entry now.
+ */
+ if (trylock_page(page)) {
+ pte_t swp_pte;
+
+ flags |= MIGRATE_PFN_LOCKED;
+ ptep_get_and_clear(mm, addr, ptep);
+
+ /* Setup special migration page table entry */
+ entry = make_migration_entry(page, pte_write(pte));
+ swp_pte = swp_entry_to_pte(entry);
+ if (pte_soft_dirty(pte))
+ swp_pte = pte_swp_mksoft_dirty(swp_pte);
+ set_pte_at(mm, addr, ptep, swp_pte);
+
+ /*
+ * This is like regular unmap: we remove the rmap and
+ * drop page refcount. Page won't be freed, as we took
+ * a reference just above.
+ */
+ page_remove_rmap(page, false);
+ put_page(page);
+ unmapped++;
+ }
+
next:
migrate->src[migrate->npages++] = pfn | flags;
ret = migrate_vma_array_full(migrate);
if (ret) {
- pte_unmap_unlock(ptep, ptl);
- return ret;
+ ptep++;
+ break;
}
}
+ arch_leave_lazy_mmu_mode();
pte_unmap_unlock(ptep - 1, ptl);
- return 0;
+ /* Only flush the TLB if we actually modified any entries */
+ if (unmapped)
+ flush_tlb_range(walk->vma, start, end);
+
+ return ret;
}
/*
@@ -2204,7 +2241,13 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
mm_walk.mm = migrate->vma->vm_mm;
mm_walk.private = migrate;
+ mmu_notifier_invalidate_range_start(mm_walk.mm,
+ migrate->start,
+ migrate->end);
walk_page_range(migrate->start, migrate->end, &mm_walk);
+ mmu_notifier_invalidate_range_end(mm_walk.mm,
+ migrate->start,
+ migrate->end);
migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
}
@@ -2251,21 +2294,27 @@ static bool migrate_vma_check_page(struct page *page)
*/
static void migrate_vma_prepare(struct migrate_vma *migrate)
{
- unsigned long addr = migrate->start, i, size;
+ unsigned long addr = migrate->start, i, size, restore = 0;
const unsigned long npages = migrate->npages;
+ const unsigned long start = migrate->start;
bool allow_drain = true;
lru_add_drain();
- for (i = 0; i < npages && migrate->cpages; i++, addr += size) {
+ for (addr = start, i = 0; i < npages; i++, addr += size) {
struct page *page = migrate_pfn_to_page(migrate->src[i]);
+ bool remap = true;
+
size = migrate_pfn_size(migrate->src[i]);
if (!page)
continue;
- lock_page(page);
- migrate->src[i] |= MIGRATE_PFN_LOCKED;
+ if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
+ remap = false;
+ lock_page(page);
+ migrate->src[i] |= MIGRATE_PFN_LOCKED;
+ }
if (!PageLRU(page) && allow_drain) {
/* Drain CPU's pagevec */
@@ -2274,10 +2323,16 @@ static void migrate_vma_prepare(struct migrate_vma *migrate)
}
if (isolate_lru_page(page)) {
- migrate->src[i] = 0;
- unlock_page(page);
- migrate->cpages--;
- put_page(page);
+ if (remap) {
+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
+ migrate->cpages--;
+ restore++;
+ } else {
+ migrate->src[i] = 0;
+ unlock_page(page);
+ migrate->cpages--;
+ put_page(page);
+ }
continue;
}
@@ -2285,13 +2340,37 @@ static void migrate_vma_prepare(struct migrate_vma *migrate)
put_page(page);
if (!migrate_vma_check_page(page)) {
- migrate->src[i] = 0;
- unlock_page(page);
- migrate->cpages--;
+ if (remap) {
+ migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
+ migrate->cpages--;
+ restore++;
- putback_lru_page(page);
+ get_page(page);
+ putback_lru_page(page);
+ } else {
+ migrate->src[i] = 0;
+ unlock_page(page);
+ migrate->cpages--;
+
+ putback_lru_page(page);
+ }
}
}
+
+ for (i = 0, addr = start; i < npages && restore; i++, addr += size) {
+ struct page *page = migrate_pfn_to_page(migrate->src[i]);
+ size = migrate_pfn_size(migrate->src[i]);
+
+ if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
+ continue;
+
+ remove_migration_pte(page, migrate->vma, addr, page);
+
+ migrate->src[i] = 0;
+ unlock_page(page);
+ put_page(page);
+ restore--;
+ }
}
/*
--
2.4.11
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[HMM 00/16] HMM (Heterogeneous Memory Management) v18 Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
[HMM 14/16] mm/migrate: allow migrate_vma() to alloc new page on empty entry Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
[HMM 10/16] mm/hmm/mirror: mirror process address space on device with HMM helpers Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 10/16] mm/hmm/mirror: mirror process address space on device with HMM helpers Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
[HMM 15/16] mm/hmm/devmem: device memory hotplug using ZONE_DEVICE Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
[HMM 09/16] mm/hmm: heterogeneous memory management (HMM for short) Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 09/16] mm/hmm: heterogeneous memory management (HMM for short) Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
[HMM 11/16] mm/hmm/mirror: helper to snapshot CPU page table v2 Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 11/16] mm/hmm/mirror: helper to snapshot CPU page table v2 Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:20 +0100
[HMM 03/16] mm/ZONE_DEVICE/free-page: callback when page is freed v3 Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 03/16] mm/ZONE_DEVICE/free-page: callback when page is freed v3 Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
[HMM 04/16] mm/ZONE_DEVICE/unaddressable: add support for un-addressable device memory v3 Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 04/16] mm/ZONE_DEVICE/unaddressable: add support for un-addressable device memory v3 Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
[HMM 16/16] mm/hmm/devmem: dummy HMM device for ZONE_DEVICE memory v2 Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 16/16] mm/hmm/devmem: dummy HMM device for ZONE_DEVICE memory v2 Bob Liu <liubo95@huawei.com> - 2017-03-17 08:10 +0100
Re: [HMM 16/16] mm/hmm/devmem: dummy HMM device for ZONE_DEVICE memory v2 Jerome Glisse <jglisse@redhat.com> - 2017-03-17 18:00 +0100
[HMM 01/16] mm/memory/hotplug: convert device bool to int to allow for more flags v3 Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
Re: [HMM 01/16] mm/memory/hotplug: convert device bool to int to allow for more flags v3 Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
[HMM 08/16] mm/migrate: migrate_vma() unmap page from vma while collecting pages Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
[HMM 13/16] mm/hmm/migrate: support un-addressable ZONE_DEVICE page in migration Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:20 +0100
[HMM 05/16] mm/ZONE_DEVICE/x86: add support for un-addressable device memory Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:30 +0100
[HMM 02/16] mm/put_page: move ref decrement to put_zone_device_page() Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:30 +0100
Re: [HMM 02/16] mm/put_page: move ref decrement to put_zone_device_page() Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
[HMM 06/16] mm/migrate: add new boolean copy flag to migratepage() callback Jérôme Glisse <jglisse@redhat.com> - 2017-03-16 16:30 +0100
Re: [HMM 06/16] mm/migrate: add new boolean copy flag to migratepage() callback Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:10 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Andrew Morton <akpm@linux-foundation.org> - 2017-03-16 21:50 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Jerome Glisse <jglisse@redhat.com> - 2017-03-17 01:00 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Bob Liu <liubo95@huawei.com> - 2017-03-17 09:30 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Jerome Glisse <jglisse@redhat.com> - 2017-03-17 17:00 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Bob Liu <liubo95@huawei.com> - 2017-03-17 09:50 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Jerome Glisse <jglisse@redhat.com> - 2017-03-17 17:20 +0100
Re: [HMM 00/16] HMM (Heterogeneous Memory Management) v18 Mel Gorman <mgorman@techsingularity.net> - 2017-03-19 21:40 +0100
csiph-web