8351464: Shenandoah: Hang on ShenandoahController::handle_alloc_failure when run test TestAllocHumongousFragment#generational

Reviewed-by: xpeng, ysr

Author: William Kemper

src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp

@@ -167,8 +167,8 @@ void ShenandoahControlThread::run_service() {
         notify_gc_waiters();
       }
 
-      // If this was the allocation failure GC cycle, notify waiters about it
-      if (alloc_failure_pending) {
+      // If this cycle completed without being cancelled, notify waiters about it
+      if (!heap->cancelled_gc()) {
         notify_alloc_failure_waiters();
       }
 

src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp

@@ -277,8 +277,8 @@ void ShenandoahGenerationalControlThread::run_gc_cycle(const ShenandoahGCRequest
     notify_gc_waiters();
   }
 
-  // If this was an allocation failure GC cycle, notify waiters about it
-  if (ShenandoahCollectorPolicy::is_allocation_failure(request.cause)) {
+  // If this cycle completed successfully, notify threads waiting to retry allocation
+  if (!_heap->cancelled_gc()) {
     notify_alloc_failure_waiters();
   }
 

test/hotspot/jtreg/ProblemList.txt

@@ -100,7 +100,6 @@ gc/TestAlwaysPreTouchBehavior.java#Z 8334513 generic-all
 gc/TestAlwaysPreTouchBehavior.java#Epsilon 8334513 generic-all
 gc/shenandoah/oom/TestAllocOutOfMemory.java#large 8344312 linux-ppc64le
 gc/shenandoah/TestEvilSyncBug.java#generational 8345501 generic-all
-gc/TestAllocHumongousFragment.java#generational 8351464 generic-all
 
 #############################################################################