8057586: Explicit GC ignored if GCLocker is active

Reviewed-by: tschatzl, ayang

Author: Ivan Walulya

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1928,6 +1928,35 @@ bool G1CollectedHeap::try_collect_concurrently(GCCause::Cause cause,
   }
 }
 
+bool G1CollectedHeap::try_collect_fullgc(GCCause::Cause cause,
+                                         const G1GCCounters& counters_before) {
+  assert_heap_not_locked();
+
+  while(true) {
+    VM_G1CollectFull op(counters_before.total_collections(),
+                        counters_before.total_full_collections(),
+                        cause);
+    VMThread::execute(&op);
+
+    // Request is trivially finished.
+    if (!GCCause::is_explicit_full_gc(cause) || op.gc_succeeded()) {
+      return op.gc_succeeded();
+    }
+
+    {
+      MutexLocker ml(Heap_lock);
+      if (counters_before.total_full_collections() != total_full_collections()) {
+        return true;
+      }
+    }
+
+    if (GCLocker::is_active_and_needs_gc()) {
+      // If GCLocker is active, wait until clear before retrying.
+      GCLocker::stall_until_clear();
+    }
+  }
+}
+
 bool G1CollectedHeap::try_collect(GCCause::Cause cause,
                                   const G1GCCounters& counters_before) {
   if (should_do_concurrent_full_gc(cause)) {
@@ -1951,11 +1980,7 @@ bool G1CollectedHeap::try_collect(GCCause::Cause cause,
     return op.gc_succeeded();
   } else {
     // Schedule a Full GC.
-    VM_G1CollectFull op(counters_before.total_collections(),
-                        counters_before.total_full_collections(),
-                        cause);
-    VMThread::execute(&op);
-    return op.gc_succeeded();
+    return try_collect_fullgc(cause, counters_before);
   }
 }
 

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -283,6 +283,9 @@ class G1CollectedHeap : public CollectedHeap {
                                 uint gc_counter,
                                 uint old_marking_started_before);
 
+  bool try_collect_fullgc(GCCause::Cause cause,
+                          const G1GCCounters& counters_before);
+
   // indicates whether we are in young or mixed GC mode
   G1CollectorState _collector_state;
 

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -549,8 +549,26 @@ void ParallelScavengeHeap::collect(GCCause::Cause cause) {
     return;
   }
 
-  VM_ParallelGCSystemGC op(gc_count, full_gc_count, cause);
-  VMThread::execute(&op);
+  while (true) {
+    VM_ParallelGCSystemGC op(gc_count, full_gc_count, cause);
+    VMThread::execute(&op);
+
+    if (!GCCause::is_explicit_full_gc(cause) || op.full_gc_succeeded()) {
+      return;
+    }
+
+    {
+      MutexLocker ml(Heap_lock);
+      if (full_gc_count != total_full_collections()) {
+        return;
+      }
+    }
+
+    if (GCLocker::is_active_and_needs_gc()) {
+      // If GCLocker is active, wait until clear before retrying.
+      GCLocker::stall_until_clear();
+    }
+  }
 }
 
 void ParallelScavengeHeap::object_iterate(ObjectClosure* cl) {

src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp

@@ -211,7 +211,7 @@ class ParallelScavengeHeap : public CollectedHeap {
   // will then attempt a full gc.  The second collects the entire heap; if
   // maximum_compaction is true, it will compact everything and clear all soft
   // references.
-  inline void invoke_scavenge();
+  inline bool invoke_scavenge();
 
   // Perform a full collection
   void do_full_collection(bool clear_all_soft_refs) override;

src/hotspot/share/gc/parallel/parallelScavengeHeap.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,8 +35,8 @@ inline bool ParallelScavengeHeap::should_alloc_in_eden(const size_t size) const
   return size < eden_size / 2;
 }
 
-inline void ParallelScavengeHeap::invoke_scavenge() {
-  PSScavenge::invoke();
+inline bool ParallelScavengeHeap::invoke_scavenge() {
+  return PSScavenge::invoke();
 }
 
 inline bool ParallelScavengeHeap::is_in_young(const void* p) const {

src/hotspot/share/gc/parallel/psParallelCompact.cpp

@@ -1678,7 +1678,7 @@ void PSParallelCompact::summary_phase(bool maximum_compaction)
 // may be true because this method can be called without intervening
 // activity.  For example when the heap space is tight and full measure
 // are being taken to free space.
-void PSParallelCompact::invoke(bool maximum_heap_compaction) {
+bool PSParallelCompact::invoke(bool maximum_heap_compaction) {
   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
   assert(Thread::current() == (Thread*)VMThread::vm_thread(),
          "should be in vm thread");
@@ -1695,8 +1695,8 @@ void PSParallelCompact::invoke(bool maximum_heap_compaction) {
   const bool clear_all_soft_refs =
     heap->soft_ref_policy()->should_clear_all_soft_refs();
 
-  PSParallelCompact::invoke_no_policy(clear_all_soft_refs ||
-                                      maximum_heap_compaction);
+  return PSParallelCompact::invoke_no_policy(clear_all_soft_refs ||
+                                             maximum_heap_compaction);
 }
 
 // This method contains no policy. You should probably

src/hotspot/share/gc/parallel/psParallelCompact.hpp

@@ -1141,7 +1141,7 @@ class PSParallelCompact : AllStatic {
 
   PSParallelCompact();
 
-  static void invoke(bool maximum_heap_compaction);
+  static bool invoke(bool maximum_heap_compaction);
   static bool invoke_no_policy(bool maximum_heap_compaction);
 
   static void post_initialize();

src/hotspot/share/gc/parallel/psVMOperations.cpp

@@ -58,7 +58,8 @@ static bool is_cause_full(GCCause::Cause cause) {
 VM_ParallelGCSystemGC::VM_ParallelGCSystemGC(uint gc_count,
                                              uint full_gc_count,
                                              GCCause::Cause gc_cause) :
-  VM_GC_Operation(gc_count, gc_cause, full_gc_count, is_cause_full(gc_cause))
+  VM_GC_Operation(gc_count, gc_cause, full_gc_count, is_cause_full(gc_cause)),
+  _full_gc_succeeded(false)
 {
 }
 
@@ -70,8 +71,8 @@ void VM_ParallelGCSystemGC::doit() {
   GCCauseSetter gccs(heap, _gc_cause);
   if (!_full) {
     // If (and only if) the scavenge fails, this will invoke a full gc.
-    heap->invoke_scavenge();
+    _full_gc_succeeded = heap->invoke_scavenge();
   } else {
-    heap->do_full_collection(false);
+    _full_gc_succeeded = PSParallelCompact::invoke(false);
   }
 }

src/hotspot/share/gc/parallel/psVMOperations.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,10 +40,12 @@ class VM_ParallelGCFailedAllocation : public VM_CollectForAllocation {
 };
 
 class VM_ParallelGCSystemGC: public VM_GC_Operation {
+  bool _full_gc_succeeded;
  public:
   VM_ParallelGCSystemGC(uint gc_count, uint full_gc_count, GCCause::Cause gc_cause);
   virtual VMOp_Type type() const { return VMOp_ParallelGCSystemGC; }
   virtual void doit();
+  bool full_gc_succeeded() const { return _full_gc_succeeded; }
 };
 
 #endif // SHARE_GC_PARALLEL_PSVMOPERATIONS_HPP

src/hotspot/share/gc/shared/gcCause.hpp

@@ -95,6 +95,12 @@ class GCCause : public AllStatic {
             cause == GCCause::_dcmd_gc_run);
   }
 
+  inline static bool is_explicit_full_gc(GCCause::Cause cause) {
+    return (is_user_requested_gc(cause) ||
+            is_serviceability_requested_gc(cause) ||
+            cause == GCCause::_wb_full_gc);
+  }
+
   inline static bool is_serviceability_requested_gc(GCCause::Cause
                                                              cause) {
     return (cause == GCCause::_jvmti_force_gc ||