8296796: Provide clean, platform-agnostic interface to C-heap trimming

Reviewed-by: stuefe
Backport-of: 0845b39caf6f04dca9cb7a5852f05b4b5ffbc034

Author: shipilev

src/hotspot/os/aix/os_aix.inline.hpp

@@ -47,4 +47,8 @@ inline bool os::must_commit_stack_guard_pages() {
 inline void os::map_stack_shadow_pages(address sp) {
 }
 
+// stubbed-out trim-native support
+inline bool os::can_trim_native_heap() { return false; }
+inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; }
+
 #endif // OS_AIX_OS_AIX_INLINE_HPP

src/hotspot/os/bsd/os_bsd.inline.hpp

@@ -51,4 +51,8 @@ inline bool os::must_commit_stack_guard_pages() {
 inline void os::map_stack_shadow_pages(address sp) {
 }
 
+// stubbed-out trim-native support
+inline bool os::can_trim_native_heap() { return false; }
+inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; }
+
 #endif // OS_BSD_OS_BSD_INLINE_HPP

src/hotspot/os/linux/os_linux.cpp

@@ -5509,3 +5509,32 @@ void os::print_memory_mappings(char* addr, size_t bytes, outputStream* st) {
     st->cr();
   }
 }
+
+bool os::trim_native_heap(os::size_change_t* rss_change) {
+#ifdef __GLIBC__
+  os::Linux::meminfo_t info1;
+  os::Linux::meminfo_t info2;
+
+  bool have_info1 = rss_change != nullptr &&
+                    os::Linux::query_process_memory_info(&info1);
+  ::malloc_trim(0);
+  bool have_info2 = rss_change != nullptr && have_info1 &&
+                    os::Linux::query_process_memory_info(&info2);
+  ssize_t delta = (ssize_t) -1;
+  if (rss_change != nullptr) {
+    if (have_info1 && have_info2 &&
+        info1.vmrss != -1 && info2.vmrss != -1 &&
+        info1.vmswap != -1 && info2.vmswap != -1) {
+      // Note: query_process_memory_info returns values in K
+      rss_change->before = (info1.vmrss + info1.vmswap) * K;
+      rss_change->after = (info2.vmrss + info2.vmswap) * K;
+    } else {
+      rss_change->after = rss_change->before = SIZE_MAX;
+    }
+  }
+
+  return true;
+#else
+  return false; // musl
+#endif
+}

src/hotspot/os/linux/os_linux.inline.hpp

@@ -43,4 +43,13 @@ inline bool os::must_commit_stack_guard_pages() {
 inline void os::map_stack_shadow_pages(address sp) {
 }
 
+// Trim-native support
+inline bool os::can_trim_native_heap() {
+#ifdef __GLIBC__
+  return true;
+#else
+  return false; // musl
+#endif
+}
+
 #endif // OS_LINUX_OS_LINUX_INLINE_HPP

src/hotspot/os/linux/trimCHeapDCmd.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2021 SAP SE. All rights reserved.
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022 SAP SE. All rights reserved.
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,56 +24,29 @@
  */
 
 #include "precompiled.hpp"
-#include "logging/log.hpp"
-#include "runtime/os.hpp"
+#include "runtime/os.inline.hpp"
+#include "trimCHeapDCmd.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
 #include "utilities/ostream.hpp"
-#include "trimCHeapDCmd.hpp"
 
 #include <malloc.h>
 
 void TrimCLibcHeapDCmd::execute(DCmdSource source, TRAPS) {
-#ifdef __GLIBC__
-  stringStream ss_report(1024); // Note: before calling trim
-
-  os::Linux::meminfo_t info1;
-  os::Linux::meminfo_t info2;
-  // Query memory before...
-  bool have_info1 = os::Linux::query_process_memory_info(&info1);
-
-  _output->print_cr("Attempting trim...");
-  ::malloc_trim(0);
-  _output->print_cr("Done.");
-
-  // ...and after trim.
-  bool have_info2 = os::Linux::query_process_memory_info(&info2);
-
-  // Print report both to output stream as well to UL
-  bool wrote_something = false;
-  if (have_info1 && have_info2) {
-    if (info1.vmsize != -1 && info2.vmsize != -1) {
-      ss_report.print_cr("Virtual size before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)",
-                         info1.vmsize, info2.vmsize, (info2.vmsize - info1.vmsize));
-      wrote_something = true;
-    }
-    if (info1.vmrss != -1 && info2.vmrss != -1) {
-      ss_report.print_cr("RSS before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)",
-                         info1.vmrss, info2.vmrss, (info2.vmrss - info1.vmrss));
-      wrote_something = true;
-    }
-    if (info1.vmswap != -1 && info2.vmswap != -1) {
-      ss_report.print_cr("Swap before: " SSIZE_FORMAT "k, after: " SSIZE_FORMAT "k, (" SSIZE_FORMAT "k)",
-                         info1.vmswap, info2.vmswap, (info2.vmswap - info1.vmswap));
-      wrote_something = true;
+  if (os::can_trim_native_heap()) {
+    os::size_change_t sc;
+    if (os::trim_native_heap(&sc)) {
+      _output->print("Trim native heap: ");
+      if (sc.after != SIZE_MAX) {
+        const size_t delta = sc.after < sc.before ? (sc.before - sc.after) : (sc.after - sc.before);
+        const char sign = sc.after < sc.before ? '-' : '+';
+        _output->print_cr("RSS+Swap: " PROPERFMT "->" PROPERFMT " (%c" PROPERFMT ")",
+                          PROPERFMTARGS(sc.before), PROPERFMTARGS(sc.after), sign, PROPERFMTARGS(delta));
+      } else {
+        _output->print_cr("(no details available).");
+      }
     }
+  } else {
+    _output->print_cr("Not available.");
   }
-  if (!wrote_something) {
-    ss_report.print_raw("No details available.");
-  }
-
-  _output->print_raw(ss_report.base());
-  log_info(os)("malloc_trim:\n%s", ss_report.base());
-#else
-  _output->print_cr("Not available.");
-#endif
 }

src/hotspot/os/windows/os_windows.inline.hpp

@@ -93,4 +93,8 @@ inline void os::PlatformMonitor::notify_all() {
   WakeAllConditionVariable(&_cond);
 }
 
+// stubbed-out trim-native support
+inline bool os::can_trim_native_heap() { return false; }
+inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; }
+
 #endif // OS_WINDOWS_OS_WINDOWS_INLINE_HPP

src/hotspot/share/runtime/os.hpp

@@ -366,6 +366,15 @@ class os: AllStatic {
   static bool   uncommit_memory(char* addr, size_t bytes, bool executable = false);
   static bool   release_memory(char* addr, size_t bytes);
 
+  // Does the platform support trimming the native heap?
+  static bool can_trim_native_heap();
+
+  // Trim the C-heap. Optionally returns working set size change (RSS+Swap) in *rss_change.
+  // Note: If trimming succeeded but no size change information could be obtained,
+  // rss_change.after will contain SIZE_MAX upon return.
+  struct size_change_t { size_t before; size_t after; };
+  static bool trim_native_heap(size_change_t* rss_change = nullptr);
+
   // A diagnostic function to print memory mappings in the given range.
   static void print_memory_mappings(char* addr, size_t bytes, outputStream* st);
   // Prints all mappings

src/hotspot/share/utilities/globalDefinitions.hpp

@@ -324,6 +324,9 @@ inline T byte_size_in_proper_unit(T s) {
   }
 }
 
+#define PROPERFMT             SIZE_FORMAT "%s"
+#define PROPERFMTARGS(s)      byte_size_in_proper_unit(s), proper_unit_for_byte_size(s)
+
 inline const char* exact_unit_for_byte_size(size_t s) {
 #ifdef _LP64
   if (s >= G && (s % G) == 0) {

test/hotspot/gtest/runtime/test_os.cpp

@@ -24,7 +24,7 @@
 #include "precompiled.hpp"
 #include "memory/allocation.hpp"
 #include "memory/resourceArea.hpp"
-#include "runtime/os.hpp"
+#include "runtime/os.inline.hpp"
 #include "runtime/thread.hpp"
 #include "services/memTracker.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -876,3 +876,27 @@ TEST_VM(os, is_first_C_frame) {
   EXPECT_FALSE(os::is_first_C_frame(&cur_frame));
 #endif // _WIN32
 }
+
+#ifdef __GLIBC__
+TEST_VM(os, trim_native_heap) {
+  EXPECT_TRUE(os::can_trim_native_heap());
+  os::size_change_t sc;
+  sc.before = sc.after = (size_t)-1;
+  EXPECT_TRUE(os::trim_native_heap(&sc));
+  tty->print_cr(SIZE_FORMAT "->" SIZE_FORMAT, sc.before, sc.after);
+  // Regardless of whether we freed memory, both before and after
+  // should be somewhat believable numbers (RSS).
+  const size_t min = 5 * M;
+  const size_t max = LP64_ONLY(20 * G) NOT_LP64(3 * G);
+  ASSERT_LE(min, sc.before);
+  ASSERT_GT(max, sc.before);
+  ASSERT_LE(min, sc.after);
+  ASSERT_GT(max, sc.after);
+  // Should also work
+  EXPECT_TRUE(os::trim_native_heap());
+}
+#else
+TEST_VM(os, trim_native_heap) {
+  EXPECT_FALSE(os::can_trim_native_heap());
+}
+#endif // __GLIBC__

test/hotspot/jtreg/serviceability/dcmd/vm/TrimLibcHeapTest.java

@@ -31,7 +31,7 @@
  * @test
  * @summary Test of diagnostic command VM.trim_libc_heap
  * @library /test/lib
- * @requires os.family == "linux"
+ * @requires (os.family=="linux") & !vm.musl
  * @modules java.base/jdk.internal.misc
  *          java.compiler
  *          java.management
@@ -42,10 +42,7 @@ public class TrimLibcHeapTest {
     public void run(CommandExecutor executor) {
         OutputAnalyzer output = executor.execute("System.trim_native_heap");
         output.reportDiagnosticSummary();
-        output.shouldMatch("(Done|Not available)"); // Not available could happen on Linux + non-glibc (eg. muslc)
-        if (output.firstMatch("Done") != null) {
-            output.shouldMatch("(Virtual size before|RSS before|Swap before|No details available)");
-        }
+        output.shouldMatch(".*Trim native heap: RSS\\+Swap: \\d+[BKM]->\\d+[BKM].*");
     }
 
     @Test