Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1353346
| From | Jérôme Glisse <jglisse@redhat.com> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [PATCH v12 10/29] HMM: use CPU page table during invalidation. |
| Date | 2016-03-08 20:50 +0100 |
| Message-ID | <ravNM-5ru-19@gated-at.bofh.it> (permalink) |
| References | <ravNL-5ru-3@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
Once we store the dma mapping inside the secondary page table we can
no longer easily find back the page backing an address. Instead use
the cpu page table which still has the proper information, except for
the invalidate_page() case which is handled by using the page passed
by the mmu_notifier layer.
Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
---
mm/hmm.c | 53 +++++++++++++++++++++++++++++++++++------------------
1 file changed, 35 insertions(+), 18 deletions(-)
diff --git a/mm/hmm.c b/mm/hmm.c
index 74e429a..7b6ba6a 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -47,9 +47,11 @@
static struct mmu_notifier_ops hmm_notifier_ops;
static void hmm_mirror_kill(struct hmm_mirror *mirror);
static inline int hmm_mirror_update(struct hmm_mirror *mirror,
- struct hmm_event *event);
+ struct hmm_event *event,
+ struct page *page);
static void hmm_mirror_update_pt(struct hmm_mirror *mirror,
- struct hmm_event *event);
+ struct hmm_event *event,
+ struct page *page);
/* hmm_event - use to track information relating to an event.
@@ -223,7 +225,9 @@ again:
}
}
-static void hmm_update(struct hmm *hmm, struct hmm_event *event)
+static void hmm_update(struct hmm *hmm,
+ struct hmm_event *event,
+ struct page *page)
{
struct hmm_mirror *mirror;
@@ -236,7 +240,7 @@ static void hmm_update(struct hmm *hmm, struct hmm_event *event)
again:
down_read(&hmm->rwsem);
hlist_for_each_entry(mirror, &hmm->mirrors, mlist)
- if (hmm_mirror_update(mirror, event)) {
+ if (hmm_mirror_update(mirror, event, page)) {
mirror = hmm_mirror_ref(mirror);
up_read(&hmm->rwsem);
hmm_mirror_kill(mirror);
@@ -304,7 +308,7 @@ static void hmm_notifier_release(struct mmu_notifier *mn, struct mm_struct *mm)
/* Make sure everything is unmapped. */
hmm_event_init(&event, mirror->hmm, 0, -1UL, HMM_MUNMAP);
- hmm_mirror_update(mirror, &event);
+ hmm_mirror_update(mirror, &event, NULL);
mirror->device->ops->release(mirror);
hmm_mirror_unref(&mirror);
@@ -338,9 +342,10 @@ static void hmm_mmu_mprot_to_etype(struct mm_struct *mm,
*etype = HMM_NONE;
}
-static void hmm_notifier_invalidate_range_start(struct mmu_notifier *mn,
- struct mm_struct *mm,
- const struct mmu_notifier_range *range)
+static void hmm_notifier_invalidate(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ struct page *page,
+ const struct mmu_notifier_range *range)
{
struct hmm_event event;
unsigned long start = range->start, end = range->end;
@@ -382,7 +387,14 @@ static void hmm_notifier_invalidate_range_start(struct mmu_notifier *mn,
hmm_event_init(&event, hmm, start, end, event.etype);
- hmm_update(hmm, &event);
+ hmm_update(hmm, &event, page);
+}
+
+static void hmm_notifier_invalidate_range_start(struct mmu_notifier *mn,
+ struct mm_struct *mm,
+ const struct mmu_notifier_range *range)
+{
+ hmm_notifier_invalidate(mn, mm, NULL, range);
}
static void hmm_notifier_invalidate_page(struct mmu_notifier *mn,
@@ -396,7 +408,7 @@ static void hmm_notifier_invalidate_page(struct mmu_notifier *mn,
range.start = addr & PAGE_MASK;
range.end = range.start + PAGE_SIZE;
range.event = mmu_event;
- hmm_notifier_invalidate_range_start(mn, mm, &range);
+ hmm_notifier_invalidate(mn, mm, page, &range);
}
static struct mmu_notifier_ops hmm_notifier_ops = {
@@ -554,23 +566,27 @@ void hmm_mirror_unref(struct hmm_mirror **mirror)
EXPORT_SYMBOL(hmm_mirror_unref);
static inline int hmm_mirror_update(struct hmm_mirror *mirror,
- struct hmm_event *event)
+ struct hmm_event *event,
+ struct page *page)
{
struct hmm_device *device = mirror->device;
int ret = 0;
ret = device->ops->update(mirror, event);
- hmm_mirror_update_pt(mirror, event);
+ hmm_mirror_update_pt(mirror, event, page);
return ret;
}
static void hmm_mirror_update_pt(struct hmm_mirror *mirror,
- struct hmm_event *event)
+ struct hmm_event *event,
+ struct page *page)
{
unsigned long addr;
struct hmm_pt_iter iter;
+ struct mm_pt_iter mm_iter;
hmm_pt_iter_init(&iter, &mirror->pt);
+ mm_pt_iter_init(&mm_iter, mirror->hmm->mm);
for (addr = event->start; addr != event->end;) {
unsigned long next = event->end;
dma_addr_t *hmm_pte;
@@ -591,10 +607,10 @@ static void hmm_mirror_update_pt(struct hmm_mirror *mirror,
continue;
if (hmm_pte_test_and_clear_dirty(hmm_pte) &&
hmm_pte_test_write(hmm_pte)) {
- struct page *page;
-
- page = pfn_to_page(hmm_pte_pfn(*hmm_pte));
- set_page_dirty(page);
+ page = page ? : mm_pt_iter_page(&mm_iter, addr);
+ if (page)
+ set_page_dirty(page);
+ page = NULL;
}
*hmm_pte &= event->pte_mask;
if (hmm_pte_test_valid_pfn(hmm_pte))
@@ -604,6 +620,7 @@ static void hmm_mirror_update_pt(struct hmm_mirror *mirror,
hmm_pt_iter_directory_unlock(&iter);
}
hmm_pt_iter_fini(&iter);
+ mm_pt_iter_fini(&mm_iter);
}
static inline bool hmm_mirror_is_dead(struct hmm_mirror *mirror)
@@ -1004,7 +1021,7 @@ static void hmm_mirror_kill(struct hmm_mirror *mirror)
/* Make sure everything is unmapped. */
hmm_event_init(&event, mirror->hmm, 0, -1UL, HMM_MUNMAP);
- hmm_mirror_update(mirror, &event);
+ hmm_mirror_update(mirror, &event, NULL);
device->ops->release(mirror);
hmm_mirror_unref(&mirror);
--
2.4.3
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
HMM (Heterogeneous Memory Management) Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 11/29] HMM: add discard range helper (to clear and free resources for a range). Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 19/29] HMM: add new HMM page table flag (select flag). Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 03/29] mmu_notifier: pass page pointer to mmu_notifier_invalidate_page() v2 Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 05/29] HMM: introduce heterogeneous memory management v5. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 06/29] HMM: add HMM page table v4. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 10/29] HMM: use CPU page table during invalidation. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 29/29] HMM: add mirror fault support for system to device memory migration v3. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 24/29] HMM: allow to get pointer to spinlock protecting a directory. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 07/29] HMM: add per mirror page table v4. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 15/29] HMM: add documentation explaining HMM internals and how to use it. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 17/29] HMM: add special swap filetype for memory migrated to device v2. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 21/29] HMM: mm add helper to update page table when migrating memory back v2. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 20:50 +0100 [PATCH v12 23/29] HMM: new callback for copying memory from and to device memory v2. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 13/29] HMM: DMA map memory on behalf of device driver v2. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 14/29] HMM: Add support for hugetlb. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 28/29] HMM: CPU page fault on migrated memory. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 26/29] HMM: add helpers for migration back to system memory v3. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 25/29] HMM: split DMA mapping function in two. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 08/29] HMM: add device page fault support v6. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 18/29] HMM: add new HMM page table flag (valid device memory). Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 12/29] HMM: add dirty range helper (toggle dirty bit inside mirror page table) v2. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100 [PATCH v12 20/29] HMM: handle HMM device page table entry on mirror page table fault and update. Jérôme Glisse <jglisse@redhat.com> - 2016-03-08 21:00 +0100
csiph-web