Memory: use MADV_DONTDUMP if available (Linux)

Also refactor other madvise() usage

rpcs3/util/vm_native.cpp

@@ -34,17 +34,37 @@ static int memfd_create_(const char *name, uint flags)
 namespace utils
 {
 #ifdef MAP_NORESERVE
-	constexpr auto c_map_noreserve = MAP_NORESERVE;
+	constexpr int c_map_noreserve = MAP_NORESERVE;
 #else
 	constexpr int c_map_noreserve = 0;
 #endif
 
 #ifdef MADV_FREE
-	constexpr auto c_madv_free = MADV_FREE;
+	constexpr int c_madv_free = MADV_FREE;
 #elif defined(MADV_DONTNEED)
-	constexpr auto c_madv_free = MADV_DONTNEED;
+	constexpr int c_madv_free = MADV_DONTNEED;
 #else
-	constexpr auto c_madv_free = 0;
+	constexpr int c_madv_free = 0;
+#endif
+
+#ifdef MADV_HUGEPAGE
+	constexpr int c_madv_hugepage = MADV_HUGEPAGE;
+#else
+	constexpr int c_madv_hugepage = 0;
+#endif
+
+#if defined(MADV_DONTDUMP) && defined(MADV_DODUMP)
+	constexpr int c_madv_no_dump = MADV_DONTDUMP;
+	constexpr int c_madv_dump = MADV_DODUMP;
+#else
+	constexpr int c_madv_no_dump = 0;
+	constexpr int c_madv_dump = 0;
+#endif
+
+#if defined(MFD_HUGETLB) && defined(MFD_HUGE_2MB)
+	constexpr int c_mfd_huge_2mb = MFD_HUGETLB | MFD_HUGE_2MB;
+#else
+	constexpr int c_mfd_huge_2mb = 0;
 #endif
 
 #ifdef _WIN32
@@ -90,7 +110,7 @@ namespace utils
 			return nullptr;
 		}
 
-		[[maybe_unused]] const auto orig_size = size;
+		const auto orig_size = size;
 
 		if (!use_addr)
 		{
@@ -125,10 +145,16 @@ namespace utils
 			ptr = static_cast<u8*>(ptr) + (0x10000 - misalign);
 		}
 
-#ifdef MADV_HUGEPAGE
 		if (orig_size % 0x200000 == 0)
-			::madvise(ptr, orig_size, MADV_HUGEPAGE);
-#endif
+		{
+			if constexpr (c_madv_hugepage)
+				::madvise(ptr, orig_size, c_madv_hugepage);
+		}
+
+		if constexpr (c_madv_no_dump)
+		{
+			ensure(::madvise(ptr, orig_size, c_madv_no_dump) != -1);
+		}
 
 		return ptr;
 #endif
@@ -141,7 +167,7 @@ namespace utils
 #else
 		const u64 ptr64 = reinterpret_cast<u64>(pointer);
 		ensure(::mprotect(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), +prot) != -1);
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED) != -1);
+		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED | c_madv_dump) != -1);
 #endif
 	}
 
@@ -152,7 +178,11 @@ namespace utils
 #else
 		const u64 ptr64 = reinterpret_cast<u64>(pointer);
 		ensure(::mmap(pointer, size, PROT_NONE, MAP_FIXED | MAP_ANON | MAP_PRIVATE | c_map_noreserve, -1, 0) != reinterpret_cast<void*>(-1));
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_free) != -1);
+
+		if constexpr (c_madv_no_dump)
+		{
+			ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_no_dump) != -1);
+		}
 #endif
 	}
 
@@ -163,14 +193,15 @@ namespace utils
 		memory_commit(pointer, size, prot);
 #else
 		const u64 ptr64 = reinterpret_cast<u64>(pointer);
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_free) != -1);
 		ensure(::mmap(pointer, size, +prot, MAP_FIXED | MAP_ANON | MAP_PRIVATE, -1, 0) != reinterpret_cast<void*>(-1));
-		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED) != -1);
 
-#ifdef MADV_HUGEPAGE
 		if (size % 0x200000 == 0)
-			::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_HUGEPAGE);
-#endif
+		{
+			if constexpr (c_madv_hugepage)
+				::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), c_madv_hugepage);
+		}
+
+		ensure(::madvise(reinterpret_cast<void*>(ptr64 & -4096), size + (ptr64 & 4095), MADV_WILLNEED) != -1);
 #endif
 	}
 
@@ -225,13 +256,16 @@ namespace utils
 		ensure(m_handle != INVALID_HANDLE_VALUE);
 #elif __linux__
 		m_file = -1;
-#ifdef MFD_HUGETLB
+
 		// Try to use 2MB pages for 2M-aligned shm
-		if (m_size % 0x200000 == 0 && flags & 2)
+		if constexpr (c_mfd_huge_2mb)
 		{
-			m_file = ::memfd_create_("2M", MFD_HUGETLB | MFD_HUGE_2MB);
+			if (m_size % 0x200000 == 0 && flags & 2)
+			{
+				m_file = ::memfd_create_("2M", c_mfd_huge_2mb);
+			}
 		}
-#endif
+
 		if (m_file == -1)
 		{
 			m_file = ::memfd_create_("", 0);