168984
From 94f984ff563d1777652b822d7a282cacc1e481c2 Mon Sep 17 00:00:00 2001
168984
From: Andrea Arcangeli <aarcange@redhat.com>
168984
Date: Wed, 27 Apr 2016 12:04:46 -0500
168984
Subject: [PATCH] mm: thp: kvm: fix memory corruption in KVM with THP enabled
168984
168984
After the THP refcounting change, obtaining a compound pages from
168984
get_user_pages() no longer allows us to assume the entire compound
168984
page is immediately mappable from a secondary MMU.
168984
168984
A secondary MMU doesn't want to call get_user_pages() more than once
168984
for each compound page, in order to know if it can map the whole
168984
compound page. So a secondary MMU needs to know from a single
168984
get_user_pages() invocation when it can map immediately the entire
168984
compound page to avoid a flood of unnecessary secondary MMU faults and
168984
spurious atomic_inc()/atomic_dec() (pages don't have to be pinned by
168984
MMU notifier users).
168984
168984
Ideally instead of the page->_mapcount < 1 check, get_user_pages()
168984
should return the granularity of the "page" mapping in the "mm" passed
168984
to get_user_pages(). However it's non trivial change to pass the "pmd"
168984
status belonging to the "mm" walked by get_user_pages up the stack (up
168984
to the caller of get_user_pages). So the fix just checks if there is
168984
not a single pte mapping on the page returned by get_user_pages, and
168984
in turn if the caller can assume that the whole compound page is
168984
mapped in the current "mm" (in a pmd_trans_huge()). In such case the
168984
entire compound page is safe to map into the secondary MMU without
168984
additional get_user_pages() calls on the surrounding tail/head
168984
pages. In addition of being faster, not having to run other
168984
get_user_pages() calls also reduces the memory footprint of the
168984
secondary MMU fault in case the pmd split happened as result of memory
168984
pressure.
168984
168984
Without this fix after a MADV_DONTNEED (like invoked by QEMU during
168984
postcopy live migration or balloning) or after generic swapping (with
168984
a failure in split_huge_page() that would only result in pmd splitting
168984
and not a physical page split), KVM would map the whole compound page
168984
into the shadow pagetables, despite regular faults or userfaults (like
168984
UFFDIO_COPY) may map regular pages into the primary MMU as result of
168984
the pte faults, leading to the guest mode and userland mode going out
168984
of sync and not working on the same memory at all times.
168984
168984
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
168984
---
168984
 arch/arm/kvm/mmu.c         |  2 +-
168984
 arch/x86/kvm/mmu.c         |  4 ++--
168984
 include/linux/page-flags.h | 22 ++++++++++++++++++++++
168984
 3 files changed, 25 insertions(+), 3 deletions(-)
168984
168984
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
168984
index aba61fd..8dafe97 100644
168984
--- a/arch/arm/kvm/mmu.c
168984
+++ b/arch/arm/kvm/mmu.c
168984
@@ -997,7 +997,7 @@ static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap)
168984
 	kvm_pfn_t pfn = *pfnp;
168984
 	gfn_t gfn = *ipap >> PAGE_SHIFT;
168984
168984
-	if (PageTransCompound(pfn_to_page(pfn))) {
168984
+	if (PageTransCompoundMap(pfn_to_page(pfn))) {
168984
 		unsigned long mask;
168984
 		/*
168984
 		 * The address we faulted on is backed by a transparent huge
168984
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
168984
index 1e7a49b..3a371f7 100644
168984
--- a/arch/x86/kvm/mmu.c
168984
+++ b/arch/x86/kvm/mmu.c
168984
@@ -2767,7 +2767,7 @@ static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
168984
 	 */
168984
 	if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn) &&
168984
 	    level == PT_PAGE_TABLE_LEVEL &&
168984
-	    PageTransCompound(pfn_to_page(pfn)) &&
168984
+	    PageTransCompoundMap(pfn_to_page(pfn)) &&
168984
 	    !has_wrprotected_page(vcpu, gfn, PT_DIRECTORY_LEVEL)) {
168984
 		unsigned long mask;
168984
 		/*
168984
@@ -4621,7 +4621,7 @@ restart:
168984
 		 */
168984
 		if (sp->role.direct &&
168984
 			!kvm_is_reserved_pfn(pfn) &&
168984
-			PageTransCompound(pfn_to_page(pfn))) {
168984
+			PageTransCompoundMap(pfn_to_page(pfn))) {
168984
 			drop_spte(kvm, sptep);
168984
 			need_tlb_flush = 1;
168984
 			goto restart;
168984
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
168984
index 19724e6..522bd6d 100644
168984
--- a/include/linux/page-flags.h
168984
+++ b/include/linux/page-flags.h
168984
@@ -517,6 +517,27 @@ static inline int PageTransCompound(struct page *page)
168984
 }
168984
168984
 /*
168984
+ * PageTransCompoundMap is the same as PageTransCompound, but it also
168984
+ * guarantees the primary MMU has the entire compound page mapped
168984
+ * through pmd_trans_huge, which in turn guarantees the secondary MMUs
168984
+ * can also map the entire compound page. This allows the secondary
168984
+ * MMUs to call get_user_pages() only once for each compound page and
168984
+ * to immediately map the entire compound page with a single secondary
168984
+ * MMU fault. If there will be a pmd split later, the secondary MMUs
168984
+ * will get an update through the MMU notifier invalidation through
168984
+ * split_huge_pmd().
168984
+ *
168984
+ * Unlike PageTransCompound, this is safe to be called only while
168984
+ * split_huge_pmd() cannot run from under us, like if protected by the
168984
+ * MMU notifier, otherwise it may result in page->_mapcount < 0 false
168984
+ * positives.
168984
+ */
168984
+static inline int PageTransCompoundMap(struct page *page)
168984
+{
168984
+	return PageTransCompound(page) && atomic_read(&page->_mapcount) < 0;
168984
+}
168984
+
168984
+/*
168984
  * PageTransTail returns true for both transparent huge pages
168984
  * and hugetlbfs pages, so it should only be called when it's known
168984
  * that hugetlbfs pages aren't involved.
168984
@@ -559,6 +580,7 @@ static inline int TestClearPageDoubleMap(struct page *page)
168984
 #else
168984
 TESTPAGEFLAG_FALSE(TransHuge)
168984
 TESTPAGEFLAG_FALSE(TransCompound)
168984
+TESTPAGEFLAG_FALSE(TransCompoundMap)
168984
 TESTPAGEFLAG_FALSE(TransTail)
168984
 TESTPAGEFLAG_FALSE(DoubleMap)
168984
 	TESTSETFLAG_FALSE(DoubleMap)
168984
-- 
168984
2.7.4
168984