8315923: pretouch_memory by atomic-add-0 fragments huge pages unexpec…

…tedly

8324781: runtime/Thread/TestAlwaysPreTouchStacks.java failed with Expected a higher ratio between stack committed and reserved
8325218: gc/parallel/TestAlwaysPreTouchBehavior.java fails

Reviewed-by: shade
Backport-of: a65a89522d2f24b1767e1c74f6689a22ea32ca6a

src/hotspot/os/aix/os_aix.cpp

@@ -1871,6 +1871,10 @@ void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) {
 void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
 }
 
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes) {
 }
 
@@ -3006,4 +3010,3 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {}
 void os::jfr_report_memory_info() {}
 
 #endif // INCLUDE_JFR
-

src/hotspot/os/bsd/os_bsd.cpp

@@ -1573,6 +1573,10 @@ void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
   ::madvise(addr, bytes, MADV_DONTNEED);
 }
 
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes) {
 }
 

src/hotspot/os/linux/globals_linux.hpp

@@ -98,7 +98,9 @@
   develop(bool, DelayThreadStartALot, false,                            \
           "Artificially delay thread starts randomly for testing.")     \
                                                                         \
-
+  product(bool, UseMadvPopulateWrite, true, DIAGNOSTIC,                 \
+          "Use MADV_POPULATE_WRITE in os::pd_pretouch_memory.")         \
+                                                                        \
 
 // end of RUNTIME_OS_FLAGS
 

src/hotspot/os/linux/os_linux.cpp

@@ -2830,6 +2830,15 @@ void os::pd_commit_memory_or_exit(char* addr, size_t size, bool exec,
   #define MADV_HUGEPAGE 14
 #endif
 
+// Define MADV_POPULATE_WRITE here so we can build HotSpot on old systems.
+#define MADV_POPULATE_WRITE_value 23
+#ifndef MADV_POPULATE_WRITE
+  #define MADV_POPULATE_WRITE MADV_POPULATE_WRITE_value
+#else
+  // Sanity-check our assumed default value if we build with a new enough libc.
+  static_assert(MADV_POPULATE_WRITE == MADV_POPULATE_WRITE_value);
+#endif
+
 int os::Linux::commit_memory_impl(char* addr, size_t size,
                                   size_t alignment_hint, bool exec) {
   int err = os::Linux::commit_memory_impl(addr, size, exec);
@@ -2875,6 +2884,31 @@ void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
   }
 }
 
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  const size_t len = pointer_delta(last, first, sizeof(char)) + page_size;
+  // Use madvise to pretouch on Linux when THP is used, and fallback to the
+  // common method if unsupported. THP can form right after madvise rather than
+  // being assembled later.
+  if (HugePages::thp_mode() == THPMode::always || UseTransparentHugePages) {
+    int err = 0;
+    if (UseMadvPopulateWrite &&
+        ::madvise(first, len, MADV_POPULATE_WRITE) == -1) {
+      err = errno;
+    }
+    if (!UseMadvPopulateWrite || err == EINVAL) { // Not to use or not supported
+      // When using THP we need to always pre-touch using small pages as the
+      // OS will initially always use small pages.
+      return os::vm_page_size();
+    } else if (err != 0) {
+      log_info(gc, os)("::madvise(" PTR_FORMAT ", " SIZE_FORMAT ", %d) failed; "
+                       "error='%s' (errno=%d)", p2i(first), len,
+                       MADV_POPULATE_WRITE, os::strerror(err), err);
+    }
+    return 0;
+  }
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes) {
   Linux::numa_interleave_memory(addr, bytes);
 }
@@ -4441,6 +4475,9 @@ void os::init(void) {
 
   check_pax();
 
+  // Check the availability of MADV_POPULATE_WRITE.
+  FLAG_SET_DEFAULT(UseMadvPopulateWrite, (::madvise(0, 0, MADV_POPULATE_WRITE) == 0));
+
   os::Posix::init();
 }
 

src/hotspot/os/windows/os_windows.cpp

@@ -3785,6 +3785,11 @@ bool os::unguard_memory(char* addr, size_t bytes) {
 
 void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) { }
 void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) { }
+
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes)    { }
 void os::numa_make_local(char *addr, size_t bytes, int lgrp_hint)    { }
 bool os::numa_topology_changed()                       { return false; }

src/hotspot/share/gc/shared/pretouchTask.cpp

@@ -68,12 +68,6 @@ void PretouchTask::pretouch(const char* task_name, char* start_address, char* en
   // Page-align the chunk size, so if start_address is also page-aligned (as
   // is common) then there won't be any pages shared by multiple chunks.
   size_t chunk_size = align_down_bounded(PretouchTask::chunk_size(), page_size);
-#ifdef LINUX
-  // When using THP we need to always pre-touch using small pages as the OS will
-  // initially always use small pages.
-  page_size = UseTransparentHugePages ? (size_t)os::vm_page_size() : page_size;
-#endif
-
   PretouchTask task(task_name, start_address, end_address, page_size, chunk_size);
   size_t total_bytes = pointer_delta(end_address, start_address, sizeof(char));
 

src/hotspot/share/runtime/os.cpp

@@ -1870,14 +1870,18 @@ void os::pretouch_memory(void* start, void* end, size_t page_size) {
     // We're doing concurrent-safe touch and memory state has page
     // granularity, so we can touch anywhere in a page.  Touch at the
     // beginning of each page to simplify iteration.
-    char* cur = static_cast<char*>(align_down(start, page_size));
+    void* first = align_down(start, page_size);
     void* last = align_down(static_cast<char*>(end) - 1, page_size);
-    assert(cur <= last, "invariant");
-    // Iterate from first page through last (inclusive), being careful to
-    // avoid overflow if the last page abuts the end of the address range.
-    for ( ; true; cur += page_size) {
-      Atomic::add(reinterpret_cast<int*>(cur), 0, memory_order_relaxed);
-      if (cur >= last) break;
+    assert(first <= last, "invariant");
+    const size_t pd_page_size = pd_pretouch_memory(first, last, page_size);
+    if (pd_page_size > 0) {
+      // Iterate from first page through last (inclusive), being careful to
+      // avoid overflow if the last page abuts the end of the address range.
+      last = align_down(static_cast<char*>(end) - 1, pd_page_size);
+      for (char* cur = static_cast<char*>(first); /* break */; cur += pd_page_size) {
+        Atomic::add(reinterpret_cast<int*>(cur), 0, memory_order_relaxed);
+        if (cur >= last) break;
+      }
     }
   }
 }

src/hotspot/share/runtime/os.hpp

@@ -219,6 +219,10 @@ class os: AllStatic {
   static void   pd_free_memory(char *addr, size_t bytes, size_t alignment_hint);
   static void   pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint);
 
+  // Returns 0 if pretouch is done via platform dependent method, or otherwise
+  // returns page_size that should be used for the common method.
+  static size_t pd_pretouch_memory(void* first, void* last, size_t page_size);
+
   static char*  pd_reserve_memory_special(size_t size, size_t alignment, size_t page_size,
 
                                           char* addr, bool executable);

test/hotspot/gtest/runtime/test_os_linux.cpp

@@ -427,6 +427,40 @@ TEST_VM(os_linux, reserve_memory_special_concurrent) {
   }
 }
 
+TEST_VM(os_linux, pretouch_thp_and_use_concurrent) {
+  // Explicitly enable thp to test cocurrent system calls.
+  const size_t size = 1 * G;
+  const bool useThp = UseTransparentHugePages;
+  UseTransparentHugePages = true;
+  char* const heap = os::reserve_memory(size, false, mtInternal);
+  EXPECT_NE(heap, nullptr);
+  EXPECT_TRUE(os::commit_memory(heap, size, false));
+
+  {
+    auto pretouch = [heap, size](Thread*, int) {
+      os::pretouch_memory(heap, heap + size, os::vm_page_size());
+    };
+    auto useMemory = [heap, size](Thread*, int) {
+      int* iptr = reinterpret_cast<int*>(heap);
+      for (int i = 0; i < 1000; i++) *iptr++ = i;
+    };
+    TestThreadGroup<decltype(pretouch)> pretouchThreads{pretouch, 4};
+    TestThreadGroup<decltype(useMemory)> useMemoryThreads{useMemory, 4};
+    useMemoryThreads.doit();
+    pretouchThreads.doit();
+    useMemoryThreads.join();
+    pretouchThreads.join();
+  }
+
+  int* iptr = reinterpret_cast<int*>(heap);
+  for (int i = 0; i < 1000; i++)
+    EXPECT_EQ(*iptr++, i);
+
+  EXPECT_TRUE(os::uncommit_memory(heap, size, false));
+  EXPECT_TRUE(os::release_memory(heap, size));
+  UseTransparentHugePages = useThp;
+}
+
 // Check that method JNI_CreateJavaVM is found.
 TEST(os_linux, addr_to_function_valid) {
   char buf[128] = "";

test/hotspot/jtreg/gc/parallel/TestAlwaysPreTouchBehavior.java

@@ -31,7 +31,7 @@
  * @requires os.family == "linux"
  * @requires os.maxMemory > 2G
  * @library /test/lib
- * @run main/othervm -Xmx1g -Xms1g -XX:+UseParallelGC -XX:+AlwaysPreTouch gc.parallel.TestAlwaysPreTouchBehavior
+ * @run main/othervm -Xmx1g -Xms1g -XX:+UseParallelGC -XX:+AlwaysPreTouch -XX:+UnlockDiagnosticVMOptions -XX:-UseMadvPopulateWrite gc.parallel.TestAlwaysPreTouchBehavior
  */
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
@@ -77,4 +77,3 @@ public static void main(String [] args) {
     Asserts.assertGreaterThanOrEqual(rss, committedMemory, "RSS of this process(" + rss + "kb) should be bigger than or equal to committed heap mem(" + committedMemory + "kb)");
    }
 }
-

test/hotspot/jtreg/runtime/Thread/TestAlwaysPreTouchStacks.java

@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.concurrent.CyclicBarrier;
@@ -89,14 +90,17 @@ public static void main(String[] args) throws Exception {
             // should show up with fully - or almost fully - committed thread stacks.
 
         } else {
-
-            ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(
+            ArrayList<String> vmArgs = new ArrayList<>();
+            Collections.addAll(vmArgs,
                     "-XX:+UnlockDiagnosticVMOptions",
                     "-Xmx100M",
                     "-XX:+AlwaysPreTouchStacks",
-                    "-XX:NativeMemoryTracking=summary", "-XX:+PrintNMTStatistics",
-                    "TestAlwaysPreTouchStacks",
-                    "test");
+                    "-XX:NativeMemoryTracking=summary", "-XX:+PrintNMTStatistics");
+            if (System.getProperty("os.name").contains("Linux")) {
+                vmArgs.add("-XX:-UseMadvPopulateWrite");
+            }
+            Collections.addAll(vmArgs, "TestAlwaysPreTouchStacks", "test");
+            ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(vmArgs);
             OutputAnalyzer output = new OutputAnalyzer(pb.start());
             output.reportDiagnosticSummary();