8315923: pretouch_memory by atomic-add-0 fragments huge pages unexpec…

…tedly

Reviewed-by: jsjolen, stuefe

src/hotspot/os/aix/os_aix.cpp

@@ -1886,6 +1886,10 @@ void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) {
 void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
 }
 
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes) {
 }
 
@@ -3027,4 +3031,3 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {}
 void os::jfr_report_memory_info() {}
 
 #endif // INCLUDE_JFR
-

src/hotspot/os/bsd/os_bsd.cpp

@@ -1637,6 +1637,10 @@ void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
   ::madvise(addr, bytes, MADV_DONTNEED);
 }
 
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes) {
 }
 

src/hotspot/os/linux/globals_linux.hpp

@@ -92,7 +92,9 @@
   develop(bool, DelayThreadStartALot, false,                            \
           "Artificially delay thread starts randomly for testing.")     \
                                                                         \
-
+  product(bool, UseMadvPopulateWrite, true, DIAGNOSTIC,                 \
+          "Use MADV_POPULATE_WRITE in os::pd_pretouch_memory.")         \
+                                                                        \
 
 // end of RUNTIME_OS_FLAGS
 

src/hotspot/os/linux/os_linux.cpp

@@ -2898,6 +2898,15 @@ void os::pd_commit_memory_or_exit(char* addr, size_t size, bool exec,
   #define MADV_HUGEPAGE 14
 #endif
 
+// Define MADV_POPULATE_WRITE here so we can build HotSpot on old systems.
+#define MADV_POPULATE_WRITE_value 23
+#ifndef MADV_POPULATE_WRITE
+  #define MADV_POPULATE_WRITE MADV_POPULATE_WRITE_value
+#else
+  // Sanity-check our assumed default value if we build with a new enough libc.
+  static_assert(MADV_POPULATE_WRITE == MADV_POPULATE_WRITE_value);
+#endif
+
 // Note that the value for MAP_FIXED_NOREPLACE differs between architectures, but all architectures
 // supported by OpenJDK share the same flag value.
 #define MAP_FIXED_NOREPLACE_value 0x100000
@@ -2957,6 +2966,31 @@ void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) {
   }
 }
 
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  const size_t len = pointer_delta(last, first, sizeof(char)) + page_size;
+  // Use madvise to pretouch on Linux when THP is used, and fallback to the
+  // common method if unsupported. THP can form right after madvise rather than
+  // being assembled later.
+  if (HugePages::thp_mode() == THPMode::always || UseTransparentHugePages) {
+    int err = 0;
+    if (UseMadvPopulateWrite &&
+        ::madvise(first, len, MADV_POPULATE_WRITE) == -1) {
+      err = errno;
+    }
+    if (!UseMadvPopulateWrite || err == EINVAL) { // Not to use or not supported
+      // When using THP we need to always pre-touch using small pages as the
+      // OS will initially always use small pages.
+      return os::vm_page_size();
+    } else if (err != 0) {
+      log_info(gc, os)("::madvise(" PTR_FORMAT ", " SIZE_FORMAT ", %d) failed; "
+                       "error='%s' (errno=%d)", p2i(first), len,
+                       MADV_POPULATE_WRITE, os::strerror(err), err);
+    }
+    return 0;
+  }
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes) {
   Linux::numa_interleave_memory(addr, bytes);
 }
@@ -4379,6 +4413,9 @@ void os::init(void) {
 
   check_pax();
 
+  // Check the availability of MADV_POPULATE_WRITE.
+  FLAG_SET_DEFAULT(UseMadvPopulateWrite, (::madvise(0, 0, MADV_POPULATE_WRITE) == 0));
+
   os::Posix::init();
 }
 

src/hotspot/os/windows/os_windows.cpp

@@ -3796,6 +3796,11 @@ bool os::unguard_memory(char* addr, size_t bytes) {
 
 void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) { }
 void os::pd_free_memory(char *addr, size_t bytes, size_t alignment_hint) { }
+
+size_t os::pd_pretouch_memory(void* first, void* last, size_t page_size) {
+  return page_size;
+}
+
 void os::numa_make_global(char *addr, size_t bytes)    { }
 void os::numa_make_local(char *addr, size_t bytes, int lgrp_hint)    { }
 bool os::numa_topology_changed()                       { return false; }

src/hotspot/share/gc/shared/pretouchTask.cpp

@@ -68,12 +68,6 @@ void PretouchTask::pretouch(const char* task_name, char* start_address, char* en
   // Page-align the chunk size, so if start_address is also page-aligned (as
   // is common) then there won't be any pages shared by multiple chunks.
   size_t chunk_size = align_down_bounded(PretouchTask::chunk_size(), page_size);
-#ifdef LINUX
-  // When using THP we need to always pre-touch using small pages as the OS will
-  // initially always use small pages.
-  page_size = UseTransparentHugePages ? (size_t)os::vm_page_size() : page_size;
-#endif
-
   PretouchTask task(task_name, start_address, end_address, page_size, chunk_size);
   size_t total_bytes = pointer_delta(end_address, start_address, sizeof(char));
 

src/hotspot/share/runtime/os.cpp

@@ -2114,14 +2114,18 @@ void os::pretouch_memory(void* start, void* end, size_t page_size) {
     // We're doing concurrent-safe touch and memory state has page
     // granularity, so we can touch anywhere in a page.  Touch at the
     // beginning of each page to simplify iteration.
-    char* cur = static_cast<char*>(align_down(start, page_size));
+    void* first = align_down(start, page_size);
     void* last = align_down(static_cast<char*>(end) - 1, page_size);
-    assert(cur <= last, "invariant");
-    // Iterate from first page through last (inclusive), being careful to
-    // avoid overflow if the last page abuts the end of the address range.
-    for ( ; true; cur += page_size) {
-      Atomic::add(reinterpret_cast<int*>(cur), 0, memory_order_relaxed);
-      if (cur >= last) break;
+    assert(first <= last, "invariant");
+    const size_t pd_page_size = pd_pretouch_memory(first, last, page_size);
+    if (pd_page_size > 0) {
+      // Iterate from first page through last (inclusive), being careful to
+      // avoid overflow if the last page abuts the end of the address range.
+      last = align_down(static_cast<char*>(end) - 1, pd_page_size);
+      for (char* cur = static_cast<char*>(first); /* break */; cur += pd_page_size) {
+        Atomic::add(reinterpret_cast<int*>(cur), 0, memory_order_relaxed);
+        if (cur >= last) break;
+      }
     }
   }
 }

src/hotspot/share/runtime/os.hpp

@@ -224,6 +224,10 @@ class os: AllStatic {
   static void   pd_free_memory(char *addr, size_t bytes, size_t alignment_hint);
   static void   pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint);
 
+  // Returns 0 if pretouch is done via platform dependent method, or otherwise
+  // returns page_size that should be used for the common method.
+  static size_t pd_pretouch_memory(void* first, void* last, size_t page_size);
+
   static char*  pd_reserve_memory_special(size_t size, size_t alignment, size_t page_size,
 
                                           char* addr, bool executable);

test/hotspot/gtest/runtime/test_os_linux.cpp

@@ -350,6 +350,40 @@ TEST_VM(os_linux, reserve_memory_special_concurrent) {
   }
 }
 
+TEST_VM(os_linux, pretouch_thp_and_use_concurrent) {
+  // Explicitly enable thp to test cocurrent system calls.
+  const size_t size = 1 * G;
+  const bool useThp = UseTransparentHugePages;
+  UseTransparentHugePages = true;
+  char* const heap = os::reserve_memory(size, false, mtInternal);
+  EXPECT_NE(heap, (char*)NULL);
+  EXPECT_TRUE(os::commit_memory(heap, size, false));
+
+  {
+    auto pretouch = [heap, size](Thread*, int) {
+      os::pretouch_memory(heap, heap + size, os::vm_page_size());
+    };
+    auto useMemory = [heap, size](Thread*, int) {
+      int* iptr = reinterpret_cast<int*>(heap);
+      for (int i = 0; i < 1000; i++) *iptr++ = i;
+    };
+    TestThreadGroup<decltype(pretouch)> pretouchThreads{pretouch, 4};
+    TestThreadGroup<decltype(useMemory)> useMemoryThreads{useMemory, 4};
+    useMemoryThreads.doit();
+    pretouchThreads.doit();
+    useMemoryThreads.join();
+    pretouchThreads.join();
+  }
+
+  int* iptr = reinterpret_cast<int*>(heap);
+  for (int i = 0; i < 1000; i++)
+    EXPECT_EQ(*iptr++, i);
+
+  EXPECT_TRUE(os::uncommit_memory(heap, size, false));
+  EXPECT_TRUE(os::release_memory(heap, size));
+  UseTransparentHugePages = useThp;
+}
+
 // Check that method JNI_CreateJavaVM is found.
 TEST(os_linux, addr_to_function_valid) {
   char buf[128] = "";

test/hotspot/jtreg/runtime/os/TestTransparentHugePageUsage.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) Ampere Computing and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test TestTransparentHugePageUsage
+ * @bug 8315923
+ * @library /test/lib
+ * @requires vm.gc.Serial & os.family == "linux" & os.maxMemory > 2G
+ * @summary Check that a pretouched java heap appears to use THPs by checking
+ *          AnonHugePages in smaps
+ * @comment Use SerialGC to increase the time window for pretouching
+ * @run driver runtime.os.TestTransparentHugePageUsage
+ */
+
+package runtime.os;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import jdk.test.lib.process.ProcessTools;
+
+public class TestTransparentHugePageUsage {
+  private static final String[] fixedCmdLine = {
+    "-XX:+UseTransparentHugePages", "-XX:+AlwaysPreTouch",
+    "-Xlog:startuptime,pagesize,gc+heap=debug",
+    "-XX:+UseSerialGC", "-Xms1G", "-Xmx1G",
+  };
+
+  public static void main(String[] args) throws Exception {
+    ArrayList<String> cmdLine = new ArrayList<>(Arrays.asList(fixedCmdLine));
+    cmdLine.add("runtime.os.TestTransparentHugePageUsage$CatSmaps");
+    ProcessBuilder builder = ProcessTools.createTestJavaProcessBuilder(cmdLine);
+    checkUsage(new BufferedReader(new InputStreamReader(builder.start().getInputStream())));
+  }
+
+  private static void checkUsage(BufferedReader reader) throws Exception {
+    final Pattern useThp = Pattern.compile(".*\\[info\\]\\[pagesize\\].+UseTransparentHugePages=1.*");
+    // Ensure THP is not disabled by OS.
+    if (reader.lines().filter(line -> useThp.matcher(line).matches()).findFirst().isPresent()) {
+      final Pattern heapAddr = Pattern.compile(".*\\sHeap:\\s.+base=0x0*(\\p{XDigit}+).*");
+      final Optional<Long> addr = reader.lines()
+          .map(line -> new SimpleEntry<String, Matcher>(line, heapAddr.matcher(line)))
+          .filter(e -> e.getValue().matches())
+          .findFirst()
+          .map(e -> Long.valueOf(e.getKey().substring(e.getValue().start(1), e.getValue().end(1)), 16));
+      if (!addr.isPresent()) throw new RuntimeException("Heap base was not found in smaps.");
+      // Match the start of a mapping, for example:
+      // 200000000-800000000 rw-p 00000000 00:00 0
+      final Pattern mapping = Pattern.compile("^(\\p{XDigit}+)-\\p{XDigit}+.*");
+      reader.lines()
+            .filter(line -> {
+                  Matcher matcher = mapping.matcher(line);
+                  if (matcher.matches()) {
+                    Long mappingAddr = Long.valueOf(line.substring(matcher.start(1), matcher.end(1)), 16);
+                    if (mappingAddr.equals(addr.get())) return true;
+                  }
+                  return false;
+                })
+            .findFirst();
+      final Pattern thpUsage = Pattern.compile("^AnonHugePages:\\s+(\\d+)\\skB");
+      final Optional<Long> usage = reader.lines()
+          .map(line -> new SimpleEntry<String, Matcher>(line, thpUsage.matcher(line)))
+          .filter(e -> e.getValue().matches())
+          .findFirst()
+          .map(e -> Long.valueOf(e.getKey().substring(e.getValue().start(1), e.getValue().end(1))));
+      if (!usage.isPresent()) throw new RuntimeException("The usage of THP was not found.");
+      // Even with MADV_POPULATE_WRITE, the usage of THP is still one page less than the whole heap.
+      if (usage.get() < 524288) throw new RuntimeException("The usage of THP is not enough.");
+    }
+  }
+
+  public static class CatSmaps {
+    public static void main(String[] args) throws Exception {
+      new BufferedReader(new FileReader("/proc/self/smaps"))
+          .lines()
+          .forEach(line -> System.out.println(line));
+    }
+  }
+}