8312182: THPs cause huge RSS due to thread start timing issue

src/hotspot/os/linux/globals_linux.hpp

@@ -85,6 +85,15 @@
   product(bool, UseCpuAllocPath, false, DIAGNOSTIC,                     \
           "Use CPU_ALLOC code path in os::active_processor_count ")     \
                                                                         \
+  product(bool, THPStackMitigation, true, DIAGNOSTIC,                   \
+          "If THPs are unconditionally enabled on the system (mode "    \
+          "\"always\"), the JVM will prevent THP from forming in "      \
+          "thread stacks. When disabled, the absence of this mitigation"\
+          "allows THPs to form in thread stacks.")                      \
+                                                                        \
+  develop(bool, DelayThreadStartALot, false,                            \
+          "Artificially delay thread starts randomly for testing.")     \
+                                                                        \
   product(bool, DumpPerfMapAtExit, false, DIAGNOSTIC,                   \
           "Write map file for Linux perf tool at exit")                 \
                                                                         \

src/hotspot/os/linux/hugepages.cpp

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2023, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "hugepages.hpp"
+
+#include "logging/log.hpp"
+#include "logging/logStream.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/ostream.hpp"
+
+// Given a file that contains a single (integral) number, return that number in (*out) and true;
+// in case of an error, return false.
+static bool read_number_file(const char* file, size_t* out) {
+  FILE* f = ::fopen(file, "r");
+  bool rc = false;
+  if (f != NULL) {
+    uint64_t i = 0;
+    if (::fscanf(f, SIZE_FORMAT, out) == 1) {
+      rc = true;
+    }
+    ::fclose(f);
+  }
+  return rc;
+}
+
+THPSupport::THPSupport() :
+    _initialized(false), _mode(thp_never), _pagesize(SIZE_MAX) {}
+
+
+THPMode THPSupport::mode() const {
+  assert(_initialized, "Not initialized");
+  return _mode;
+}
+
+size_t THPSupport::pagesize() const {
+  assert(_initialized, "Not initialized");
+  return _pagesize;
+}
+
+void THPSupport::scan_os() {
+  // Scan /sys/kernel/mm/transparent_hugepage/enabled
+  // see mm/huge_memory.c
+  _mode = thp_never;
+  const char* filename = "/sys/kernel/mm/transparent_hugepage/enabled";
+  FILE* f = ::fopen(filename, "r");
+  if (f != NULL) {
+    char buf[64];
+    char* s = fgets(buf, sizeof(buf), f);
+    assert(s == buf, "Should have worked");
+    if (::strstr(buf, "[madvise]") != NULL) {
+      _mode = thp_madvise;
+    } else if (::strstr(buf, "[always]") != NULL) {
+      _mode = thp_always;
+    } else {
+      assert(::strstr(buf, "[never]") != NULL, "Weird content of %s: %s", filename, buf);
+    }
+    fclose(f);
+  }
+
+  // Scan large page size for THP from hpage_pmd_size
+  _pagesize = 0;
+  if (read_number_file("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", &_pagesize)) {
+    assert(_pagesize > 0, "Expected");
+  }
+  _initialized = true;
+
+  LogTarget(Info, pagesize) lt;
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
+    print_on(&ls);
+  }
+}
+
+void THPSupport::print_on(outputStream* os) {
+  if (_initialized) {
+    os->print_cr("Transparent hugepage (THP) support:");
+    os->print_cr("  THP mode: %s",
+        (_mode == thp_always ? "always" : (_mode == thp_never ? "never" : "madvise")));
+    os->print_cr("  THP pagesize: " SIZE_FORMAT, _pagesize);
+  } else {
+    os->print_cr("  unknown.");
+  }
+}
+
+THPSupport HugePages::_thp_support;
+
+void HugePages::initialize() {
+  _thp_support.scan_os();
+}
+
+void HugePages::print_on(outputStream* os) {
+  _thp_support.print_on(os);
+}

src/hotspot/os/linux/hugepages.hpp

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2023, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef OS_LINUX_HUGEPAGES_HPP
+#define OS_LINUX_HUGEPAGES_HPP
+
+#include "memory/allocation.hpp"
+#include "runtime/os.hpp" // for os::PageSizes
+#include "utilities/globalDefinitions.hpp"
+
+class outputStream;
+
+enum THPMode { thp_always, thp_never, thp_madvise };
+
+// 2) for transparent hugepages
+class THPSupport {
+  bool _initialized;
+
+  // See /sys/kernel/mm/transparent_hugepages/enabled
+  THPMode _mode;
+
+  // Contains the THP page size
+  size_t _pagesize;
+
+public:
+
+  THPSupport();
+
+  // Queries the OS, fills in object
+  void scan_os();
+
+  THPMode mode() const;
+  size_t pagesize() const;
+  void print_on(outputStream* os);
+};
+
+// Umbrella static interface
+class HugePages : public AllStatic {
+
+  static THPSupport _thp_support;
+
+public:
+
+  static const THPSupport& thp_info() { return _thp_support; }
+
+  static THPMode thp_mode()                     { return _thp_support.mode(); }
+  static bool supports_thp()                    { return thp_mode() == thp_madvise || thp_mode() == thp_always; }
+  static size_t thp_pagesize()                  { return _thp_support.pagesize(); }
+
+  static void initialize();
+  static void print_on(outputStream* os);
+};
+
+#endif // OS_LINUX_HUGEPAGES_HPP

src/hotspot/os/linux/os_linux.cpp

@@ -30,6 +30,7 @@
 #include "code/vtableStubs.hpp"
 #include "compiler/compileBroker.hpp"
 #include "compiler/disassembler.hpp"
+#include "hugepages.hpp"
 #include "interpreter/interpreter.hpp"
 #include "jvmtifiles/jvmti.h"
 #include "logging/log.hpp"
@@ -721,6 +722,10 @@ static void *thread_native_entry(Thread *thread) {
 
   assert(osthread->pthread_id() != 0, "pthread_id was not set as expected");
 
+  if (DelayThreadStartALot) {
+    os::naked_short_sleep(100);
+  }
+
   // call one more level start routine
   thread->call_run();
 
@@ -877,13 +882,16 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   }
   assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
 
-  // Add an additional page to the stack size to reduce its chances of getting large page aligned
-  // so that the stack does not get backed by a transparent huge page.
-  size_t default_large_page_size = os::Linux::default_large_page_size();
-  if (default_large_page_size != 0 &&
-      stack_size >= default_large_page_size &&
-      is_aligned(stack_size, default_large_page_size)) {
-    stack_size += os::vm_page_size();
+  if (THPStackMitigation) {
+    // In addition to the glibc guard page that prevents inter-thread-stack hugepage
+    // coalescing (see comment in os::Linux::default_guard_size()), we also make
+    // sure the stack size itself is not huge-page-size aligned; that makes it much
+    // more likely for thread stack boundaries to be unaligned as well and hence
+    // protects thread stacks from being targeted by khugepaged.
+    if (HugePages::thp_pagesize() > 0 &&
+        is_aligned(stack_size, HugePages::thp_pagesize())) {
+      stack_size += os::vm_page_size();
+    }
   }
 
   int status = pthread_attr_setstacksize(&attr, stack_size);
@@ -3135,6 +3143,27 @@ bool os::Linux::libnuma_init() {
 }
 
 size_t os::Linux::default_guard_size(os::ThreadType thr_type) {
+
+  if (THPStackMitigation) {
+    // If THPs are unconditionally enabled, the following scenario can lead to huge RSS
+    // - parent thread spawns, in quick succession, multiple child threads
+    // - child threads are slow to start
+    // - thread stacks of future child threads are adjacent and get merged into one large VMA
+    //   by the kernel, and subsequently transformed into huge pages by khugepaged
+    // - child threads come up, place JVM guard pages, thus splinter the large VMA, splinter
+    //   the huge pages into many (still paged-in) small pages.
+    // The result of that sequence are thread stacks that are fully paged-in even though the
+    // threads did not even start yet.
+    // We prevent that by letting the glibc allocate a guard page, which causes a VMA with different
+    // permission bits to separate two ajacent thread stacks and therefore prevent merging stacks
+    // into one VMA.
+    //
+    // Yes, this means we have two guard sections - the glibc and the JVM one - per thread. But the
+    // cost for that one extra protected page is dwarfed from a large win in performance and memory
+    // that avoiding interference by khugepaged buys us.
+    return os::vm_page_size();
+  }
+
   // Creating guard page is very expensive. Java thread has HotSpot
   // guard pages, only enable glibc guard page for non-Java threads.
   // (Remember: compiler thread is a Java thread, too!)
@@ -3848,7 +3877,25 @@ bool os::Linux::setup_large_page_type(size_t page_size) {
 }
 
 void os::large_page_init() {
-  // Always initialize the default large page size even if large pages are not being used.
+
+  // Query OS information first.
+  HugePages::initialize();
+
+  // If THPs are unconditionally enabled (THP mode "always"), khugepaged may attempt to
+  // coalesce small pages in thread stacks to huge pages. That costs a lot of memory and
+  // is usually unwanted for thread stacks. Therefore we attempt to prevent THP formation in
+  // thread stacks unless the user explicitly allowed THP formation by manually disabling
+  // -XX:-THPStackMitigation.
+  if (HugePages::thp_mode() == thp_always) {
+    if (THPStackMitigation) {
+      log_info(pagesize)("JVM will attempt to prevent THPs in thread stacks.");
+    } else {
+      log_info(pagesize)("JVM will *not* prevent THPs in thread stacks. This may cause high RSS.");
+    }
+  } else {
+    FLAG_SET_ERGO(THPStackMitigation, false); // Mitigation not needed
+  }
+
   size_t default_large_page_size = scan_default_large_page_size();
   os::Linux::_default_large_page_size = default_large_page_size;