8256844: Make NMT late-initializable

Reviewed-by: stuefe
Backport-of: eec64f55870cf51746755d8fa59098a82109e826

Author: Vladimir Kempik

src/hotspot/share/runtime/arguments.cpp

@@ -53,7 +53,7 @@
 #include "runtime/safepointMechanism.hpp"
 #include "runtime/vm_version.hpp"
 #include "services/management.hpp"
-#include "services/memTracker.hpp"
+#include "services/nmtCommon.hpp"
 #include "utilities/align.hpp"
 #include "utilities/defaultStream.hpp"
 #include "utilities/macros.hpp"
@@ -1985,17 +1985,6 @@ bool Arguments::check_vm_args_consistency() {
     status = false;
   }
 
-  if (PrintNMTStatistics) {
-#if INCLUDE_NMT
-    if (MemTracker::tracking_level() == NMT_off) {
-#endif // INCLUDE_NMT
-      warning("PrintNMTStatistics is disabled, because native memory tracking is not enabled");
-      PrintNMTStatistics = false;
-#if INCLUDE_NMT
-    }
-#endif
-  }
-
   status = CompilerConfig::check_args_consistency(status);
 #if INCLUDE_JVMCI
   if (status && EnableJVMCI) {
@@ -3718,29 +3707,6 @@ jint Arguments::match_special_option_and_act(const JavaVMInitArgs* args,
       JVMFlag::printFlags(tty, false);
       vm_exit(0);
     }
-    if (match_option(option, "-XX:NativeMemoryTracking", &tail)) {
-#if INCLUDE_NMT
-      // The launcher did not setup nmt environment variable properly.
-      if (!MemTracker::check_launcher_nmt_support(tail)) {
-        warning("Native Memory Tracking did not setup properly, using wrong launcher?");
-      }
-
-      // Verify if nmt option is valid.
-      if (MemTracker::verify_nmt_option()) {
-        // Late initialization, still in single-threaded mode.
-        if (MemTracker::tracking_level() >= NMT_summary) {
-          MemTracker::init();
-        }
-      } else {
-        vm_exit_during_initialization("Syntax error, expecting -XX:NativeMemoryTracking=[off|summary|detail]", NULL);
-      }
-      continue;
-#else
-      jio_fprintf(defaultStream::error_stream(),
-        "Native Memory Tracking is not supported in this VM\n");
-      return JNI_ERR;
-#endif
-    }
 
 #ifndef PRODUCT
     if (match_option(option, "-XX:+PrintFlagsWithComments")) {
@@ -3992,6 +3958,26 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) {
   no_shared_spaces("CDS Disabled");
 #endif // INCLUDE_CDS
 
+#if INCLUDE_NMT
+  // Verify NMT arguments
+  const NMT_TrackingLevel lvl = NMTUtil::parse_tracking_level(NativeMemoryTracking);
+  if (lvl == NMT_unknown) {
+    jio_fprintf(defaultStream::error_stream(),
+                "Syntax error, expecting -XX:NativeMemoryTracking=[off|summary|detail]", NULL);
+    return JNI_ERR;
+  }
+  if (PrintNMTStatistics && lvl == NMT_off) {
+    warning("PrintNMTStatistics is disabled, because native memory tracking is not enabled");
+    FLAG_SET_DEFAULT(PrintNMTStatistics, false);
+  }
+#else
+  if (!FLAG_IS_DEFAULT(NativeMemoryTracking) || PrintNMTStatistics) {
+    warning("Native Memory Tracking is not supported in this VM");
+    FLAG_SET_DEFAULT(NativeMemoryTracking, "off");
+    FLAG_SET_DEFAULT(PrintNMTStatistics, false);
+  }
+#endif // INCLUDE_NMT
+
   if (TraceDependencies && VerifyDependencies) {
     if (!FLAG_IS_DEFAULT(TraceDependencies)) {
       warning("TraceDependencies results may be inflated by VerifyDependencies");

src/hotspot/share/runtime/os.cpp

@@ -64,6 +64,7 @@
 #include "services/attachListener.hpp"
 #include "services/mallocTracker.hpp"
 #include "services/memTracker.hpp"
+#include "services/nmtPreInit.hpp"
 #include "services/nmtCommon.hpp"
 #include "services/threadService.hpp"
 #include "utilities/align.hpp"
@@ -681,6 +682,15 @@ void* os::malloc(size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));
 
+#if INCLUDE_NMT
+  {
+    void* rc = NULL;
+    if (NMTPreInit::handle_malloc(&rc, size)) {
+      return rc;
+    }
+  }
+#endif
+
   // Since os::malloc can be called when the libjvm.{dll,so} is
   // first loaded and we don't have a thread yet we must accept NULL also here.
   assert(!os::ThreadCrashProtection::is_crash_protected(Thread::current_or_null()),
@@ -740,6 +750,15 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS flags) {
 
 void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
 
+#if INCLUDE_NMT
+  {
+    void* rc = NULL;
+    if (NMTPreInit::handle_realloc(&rc, memblock, size)) {
+      return rc;
+    }
+  }
+#endif
+
   // For the test flag -XX:MallocMaxTestWords
   if (has_reached_max_malloc_test_peak(size)) {
     return NULL;
@@ -790,6 +809,13 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCa
 
 // handles NULL pointers
 void  os::free(void *memblock) {
+
+#if INCLUDE_NMT
+  if (NMTPreInit::handle_free(memblock)) {
+    return;
+  }
+#endif
+
   NOT_PRODUCT(inc_stat_counter(&num_frees, 1));
 #ifdef ASSERT
   if (memblock == NULL) return;

src/hotspot/share/runtime/thread.cpp

@@ -2769,6 +2769,11 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   jint parse_result = Arguments::parse(args);
   if (parse_result != JNI_OK) return parse_result;
 
+#if INCLUDE_NMT
+  // Initialize NMT right after argument parsing to keep the pre-NMT-init window small.
+  MemTracker::initialize();
+#endif // INCLUDE_NMT
+
   os::init_before_ergo();
 
   jint ergo_result = Arguments::apply_ergo();

src/hotspot/share/services/memTracker.cpp

@@ -23,15 +23,20 @@
  */
 #include "precompiled.hpp"
 #include "jvm.h"
+#include "logging/log.hpp"
+#include "logging/logStream.hpp"
 #include "memory/metaspaceUtils.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/globals.hpp"
 #include "runtime/orderAccess.hpp"
 #include "runtime/vmThread.hpp"
 #include "runtime/vmOperations.hpp"
 #include "services/memBaseline.hpp"
 #include "services/memReporter.hpp"
 #include "services/mallocTracker.inline.hpp"
 #include "services/memTracker.hpp"
+#include "services/nmtCommon.hpp"
+#include "services/nmtPreInit.hpp"
 #include "services/threadStackTracker.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/defaultStream.hpp"
@@ -45,79 +50,44 @@ volatile NMT_TrackingLevel MemTracker::_tracking_level = NMT_unknown;
 NMT_TrackingLevel MemTracker::_cmdline_tracking_level = NMT_unknown;
 
 MemBaseline MemTracker::_baseline;
-bool MemTracker::_is_nmt_env_valid = true;
 
-static const size_t buffer_size = 64;
+void MemTracker::initialize() {
+  bool rc = true;
+  assert(_tracking_level == NMT_unknown, "only call once");
+
+  NMT_TrackingLevel level = NMTUtil::parse_tracking_level(NativeMemoryTracking);
+  // Should have been validated before in arguments.cpp
+  assert(level == NMT_off || level == NMT_summary || level == NMT_detail,
+         "Invalid setting for NativeMemoryTracking (%s)", NativeMemoryTracking);
 
-NMT_TrackingLevel MemTracker::init_tracking_level() {
   // Memory type is encoded into tracking header as a byte field,
   // make sure that we don't overflow it.
   STATIC_ASSERT(mt_number_of_types <= max_jubyte);
 
-  char nmt_env_variable[buffer_size];
-  jio_snprintf(nmt_env_variable, sizeof(nmt_env_variable), "NMT_LEVEL_%d", os::current_process_id());
-  const char* nmt_env_value;
-#ifdef _WINDOWS
-  // Read the NMT environment variable from the PEB instead of the CRT
-  char value[buffer_size];
-  nmt_env_value = GetEnvironmentVariable(nmt_env_variable, value, (DWORD)sizeof(value)) != 0 ? value : NULL;
-#else
-  nmt_env_value = ::getenv(nmt_env_variable);
-#endif
-  NMT_TrackingLevel level = NMT_off;
-  if (nmt_env_value != NULL) {
-    if (strcmp(nmt_env_value, "summary") == 0) {
-      level = NMT_summary;
-    } else if (strcmp(nmt_env_value, "detail") == 0) {
-      level = NMT_detail;
-    } else if (strcmp(nmt_env_value, "off") != 0) {
-      // The value of the environment variable is invalid
-      _is_nmt_env_valid = false;
-    }
-    // Remove the environment variable to avoid leaking to child processes
-    os::unsetenv(nmt_env_variable);
-  }
-
-  if (!MallocTracker::initialize(level) ||
-      !VirtualMemoryTracker::initialize(level)) {
-    level = NMT_off;
-  }
-  return level;
-}
-
-void MemTracker::init() {
-  NMT_TrackingLevel level = tracking_level();
-  if (level >= NMT_summary) {
-    if (!VirtualMemoryTracker::late_initialize(level) ||
-        !ThreadStackTracker::late_initialize(level)) {
-      shutdown();
+  if (level > NMT_off) {
+    if (!MallocTracker::initialize(level) ||
+        !VirtualMemoryTracker::initialize(level) ||
+        !ThreadStackTracker::initialize(level)) {
+      assert(false, "NMT initialization failed");
+      level = NMT_off;
+      log_warning(nmt)("NMT initialization failed. NMT disabled.");
       return;
     }
   }
-}
 
-bool MemTracker::check_launcher_nmt_support(const char* value) {
-  if (strcmp(value, "=detail") == 0) {
-    if (MemTracker::tracking_level() != NMT_detail) {
-      return false;
-    }
-  } else if (strcmp(value, "=summary") == 0) {
-    if (MemTracker::tracking_level() != NMT_summary) {
-      return false;
-    }
-  } else if (strcmp(value, "=off") == 0) {
-    if (MemTracker::tracking_level() != NMT_off) {
-      return false;
-    }
-  } else {
-    _is_nmt_env_valid = false;
-  }
+  NMTPreInit::pre_to_post();
 
-  return true;
-}
+  _tracking_level = _cmdline_tracking_level = level;
 
-bool MemTracker::verify_nmt_option() {
-  return _is_nmt_env_valid;
+  // Log state right after NMT initialization
+  if (log_is_enabled(Info, nmt)) {
+    LogTarget(Info, nmt) lt;
+    LogStream ls(lt);
+    ls.print_cr("NMT initialized: %s", NMTUtil::tracking_level_to_string(_tracking_level));
+    ls.print_cr("Preinit state: ");
+    NMTPreInit::print_state(&ls);
+    ls.cr();
+  }
 }
 
 void* MemTracker::malloc_base(void* memblock) {
@@ -174,6 +144,8 @@ bool MemTracker::transition_to(NMT_TrackingLevel level) {
 void MemTracker::error_report(outputStream* output) {
   if (tracking_level() >= NMT_summary) {
     report(true, output, MemReporterBase::default_scale); // just print summary for error case.
+    output->print("Preinit state:");
+    NMTPreInit::print_state(output);
   }
 }
 
@@ -214,9 +186,14 @@ void MemTracker::report(bool summary_only, outputStream* output, size_t scale) {
 void MemTracker::tuning_statistics(outputStream* out) {
   // NMT statistics
   out->print_cr("Native Memory Tracking Statistics:");
+  out->print_cr("State: %s", NMTUtil::tracking_level_to_string(_tracking_level));
   out->print_cr("Malloc allocation site table size: %d", MallocSiteTable::hash_buckets());
   out->print_cr("             Tracking stack depth: %d", NMT_TrackingStackDepth);
   NOT_PRODUCT(out->print_cr("Peak concurrent access: %d", MallocSiteTable::access_peak_count());)
   out->cr();
   MallocSiteTable::print_tuning_statistics(out);
+  out->cr();
+  out->print_cr("Preinit state:");
+  NMTPreInit::print_state(out);
+  out->cr();
 }