From 5cfd1c393d0f297a5b115650de99fe5a1ac5519c Mon Sep 17 00:00:00 2001
From: Nekotekina <nekotekina@gmail.com>
Date: Thu, 14 Jan 2021 19:59:52 +0300
Subject: [PATCH] Memory: use MADV_DONTDUMP if available (Linux)

Also refactor other madvise() usage
---
 rpcs3/util/vm_native.cpp | 72 +++++++++++++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 19 deletions(-)

diff --git a/rpcs3/util/vm_native.cpp b/rpcs3/util/vm_native.cpp
index 342e94db9ad3..b1f64158d4cd 100644
--- a/rpcs3/util/vm_native.cpp
+++ b/rpcs3/util/vm_native.cpp
@@ -34,17 +34,37 @@ static int memfd_create_(const char *name, uint flags)
 namespace utils
 {
 #ifdef MAP_NORESERVE
-	constexpr auto c_map_noreserve = MAP_NORESERVE;
+	constexpr int c_map_noreserve = MAP_NORESERVE;
 #else
 	constexpr int c_map_noreserve = 0;
 #endif
 
 #ifdef MADV_FREE
-	constexpr auto c_madv_free = MADV_FREE;
+	constexpr int c_madv_free = MADV_FREE;
 #elif defined(MADV_DONTNEED)
-	constexpr auto c_madv_free = MADV_DONTNEED;
+	constexpr int c_madv_free = MADV_DONTNEED;
 #else
-	constexpr auto c_madv_free = 0;
+	constexpr int c_madv_free = 0;
+#endif
+
+#ifdef MADV_HUGEPAGE
+	constexpr int c_madv_hugepage = MADV_HUGEPAGE;
+#else
+	constexpr int c_madv_hugepage = 0;
+#endif
+
+#if defined(MADV_DONTDUMP) && defined(MADV_DODUMP)
+	constexpr int c_madv_no_dump = MADV_DONTDUMP;
+	constexpr int c_madv_dump = MADV_DODUMP;
+#else
+	constexpr int c_madv_no_dump = 0;
+	constexpr int c_madv_dump = 0;
+#endif
+
+#if defined(MFD_HUGETLB) && defined(MFD_HUGE_2MB)
+	constexpr int c_mfd_huge_2mb = MFD_HUGETLB | MFD_HUGE_2MB;
+#else
+	constexpr int c_mfd_huge_2mb = 0;
 #endif
 
 #ifdef _WIN32
@@ -90,7 +110,7 @@ namespace utils
 			return nullptr;
 		}
 
-		[[maybe_unused]] const auto orig_size = size;
+		const auto orig_size = size;
 
 		if (!use_addr)
 		{
@@ -125,10 +145,16 @@ namespace utils
 			ptr = static_cast<u8*>(ptr) + (0x10000 - misalign);
 		}
 
-#ifdef MADV_HUGEPAGE
 		if (orig_size % 0x200000 == 0)
-			::madvise(ptr, orig_size, MADV_HUGEPAGE);
-#endif
+		{
+			if constexpr (c_madv_hugepage)
+				::madvise(ptr, orig_size, c_madv_hugepage);
+		}
+
+		if constexpr (c_madv_no_dump)
+		{
+			ensure(::madvise(ptr, orig_size, c_madv_no_dump) != -1);
+		}
 
 		return ptr;
 #endif
@@ -141,7 +167,7 @@ namespace utils
 #else
 		const u64 ptr64 = reinterpret_cast<u64>(pointer);
 		ensure(::mprotect(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), +prot) != -1);
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED) != -1);
+		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED | c_madv_dump) != -1);
 #endif
 	}
 
@@ -152,7 +178,11 @@ namespace utils
 #else
 		const u64 ptr64 = reinterpret_cast<u64>(pointer);
 		ensure(::mmap(pointer, size, PROT_NONE, MAP_FIXED | MAP_ANON | MAP_PRIVATE | c_map_noreserve, -1, 0) != reinterpret_cast<void*>(-1));
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_free) != -1);
+
+		if constexpr (c_madv_no_dump)
+		{
+			ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_no_dump) != -1);
+		}
 #endif
 	}
 
@@ -163,14 +193,15 @@ namespace utils
 		memory_commit(pointer, size, prot);
 #else
 		const u64 ptr64 = reinterpret_cast<u64>(pointer);
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_free) != -1);
 		ensure(::mmap(pointer, size, +prot, MAP_FIXED | MAP_ANON | MAP_PRIVATE, -1, 0) != reinterpret_cast<void*>(-1));
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED) != -1);
 
-#ifdef MADV_HUGEPAGE
 		if (size % 0x200000 == 0)
-			::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_HUGEPAGE);
-#endif
+		{
+			if constexpr (c_madv_hugepage)
+				::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_hugepage);
+		}
+
+		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED) != -1);
 #endif
 	}
 
@@ -225,13 +256,16 @@ namespace utils
 		ensure(m_handle != INVALID_HANDLE_VALUE);
 #elif __linux__
 		m_file = -1;
-#ifdef MFD_HUGETLB
+
 		// Try to use 2MB pages for 2M-aligned shm
-		if (m_size % 0x200000 == 0 && flags & 2)
+		if constexpr (c_mfd_huge_2mb)
 		{
-			m_file = ::memfd_create_("2M", MFD_HUGETLB | MFD_HUGE_2MB);
+			if (m_size % 0x200000 == 0 && flags & 2)
+			{
+				m_file = ::memfd_create_("2M", c_mfd_huge_2mb);
+			}
 		}
-#endif
+
 		if (m_file == -1)
 		{
 			m_file = ::memfd_create_("", 0);