8274191: Improve g1 evacuation failure injector performance

Reviewed-by: kbarrett, ayang

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -85,6 +85,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
     _num_optional_regions(optional_cset_length),
     _numa(g1h->numa()),
     _obj_alloc_stat(NULL),
+    NOT_PRODUCT(_evac_failure_inject_counter(0) COMMA)
     _preserved_marks(preserved_marks),
     _evacuation_failed_info(),
     _evac_failure_regions(evac_failure_regions)
@@ -414,6 +415,12 @@ HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr,
   return obj_ptr;
 }
 
+#ifndef PRODUCT
+bool G1ParScanThreadState::inject_evacuation_failure() {
+  return _g1h->evac_failure_injector()->evacuation_should_fail(_evac_failure_inject_counter);
+}
+#endif
+
 NOINLINE
 void G1ParScanThreadState::undo_allocation(G1HeapRegionAttr dest_attr,
                                            HeapWord* obj_ptr,
@@ -458,7 +465,7 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
   assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap");
 
   // Should this evacuation fail?
-  if (_g1h->evac_failure_injector()->evacuation_should_fail()) {
+  if (inject_evacuation_failure()) {
     // Doing this after all the allocation attempts also tests the
     // undo_allocation() method too.
     undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
@@ -515,7 +522,6 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
     G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_young());
     obj->oop_iterate_backwards(&_scanner, klass);
     return obj;
-
   } else {
     _plab_allocator->undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
     return forward_ptr;

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -98,10 +98,15 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   size_t* _obj_alloc_stat;
 
   // Per-thread evacuation failure data structures.
+#ifndef PRODUCT
+  size_t _evac_failure_inject_counter;
+#endif
   PreservedMarks* _preserved_marks;
   EvacuationFailedInfo _evacuation_failed_info;
   G1EvacFailureRegions* _evac_failure_regions;
 
+  bool inject_evacuation_failure() PRODUCT_RETURN_( return false; );
+
 public:
   G1ParScanThreadState(G1CollectedHeap* g1h,
                        G1RedirtyCardsQueueSet* rdcqs,

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.cpp

@@ -72,11 +72,8 @@ void G1YoungGCEvacFailureInjector::arm_if_needed() {
 }
 
 void G1YoungGCEvacFailureInjector::reset() {
-  if (G1EvacuationFailureALot) {
-    _last_collection_with_evacuation_failure = G1CollectedHeap::heap()->total_collections();
-    _evacuation_failure_object_count = 0;
-    _inject_evacuation_failure_for_current_gc = false;
-  }
+  _last_collection_with_evacuation_failure = G1CollectedHeap::heap()->total_collections();
+  _inject_evacuation_failure_for_current_gc = false;
 }
 
 #endif // #ifndef PRODUCT

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.hpp

@@ -45,9 +45,6 @@ class G1YoungGCEvacFailureInjector {
   // Used to determine whether evacuation failure injection should be in effect
   // for the current GC.
   size_t _last_collection_with_evacuation_failure;
-
-  // The number of evacuations between induced failures.
-  volatile size_t _evacuation_failure_object_count;
 #endif
 
   bool arm_if_needed_for_gc_type(bool for_young_gc,
@@ -59,8 +56,9 @@ class G1YoungGCEvacFailureInjector {
   // GC (based upon the type of GC and which command line flags are set);
   void arm_if_needed() PRODUCT_RETURN;
 
-  // Return true if it's time to cause an evacuation failure.
-  bool evacuation_should_fail() PRODUCT_RETURN_( return false; );
+  // Return true if it's time to cause an evacuation failure; the caller
+  // provides the (preferably thread-local) counter to minimize performance impact.
+  bool evacuation_should_fail(size_t& counter) PRODUCT_RETURN_( return false; );
 
   // Reset the evacuation failure injection counters. Should be called at
   // the end of an evacuation pause in which an evacuation failure occurred.

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.inline.hpp

@@ -25,23 +25,21 @@
 #ifndef SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_INLINE_HPP
 #define SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_INLINE_HPP
 
+#include "gc/g1/g1YoungGCEvacFailureInjector.hpp"
+
 #include "gc/g1/g1_globals.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1YoungGCEvacFailureInjector.hpp"
 
 #ifndef PRODUCT
 
-inline bool G1YoungGCEvacFailureInjector::evacuation_should_fail() {
-  if (!G1EvacuationFailureALot || !_inject_evacuation_failure_for_current_gc) {
+inline bool G1YoungGCEvacFailureInjector::evacuation_should_fail(size_t& counter) {
+  if (!_inject_evacuation_failure_for_current_gc) {
     return false;
   }
-  // Injecting evacuation failures is in effect for current GC
-  // Access to _evacuation_failure_alot_count is not atomic;
-  // the value does not have to be exact.
-  if (++_evacuation_failure_object_count < G1EvacuationFailureALotCount) {
+  if (++counter < G1EvacuationFailureALotCount) {
     return false;
   }
-  _evacuation_failure_object_count = 0;
+  counter = 0;
   return true;
 }
 

src/hotspot/share/gc/g1/g1_globals.hpp

@@ -275,7 +275,7 @@
                                                                             \
   develop(uintx, G1EvacuationFailureALotCount, 1000,                        \
           "Number of successful evacuations between evacuation failures "   \
-          "occurring at object copying")                                    \
+          "occurring at object copying per thread")                         \
                                                                             \
   develop(uintx, G1EvacuationFailureALotInterval, 5,                        \
           "Total collections between forced triggering of evacuation "      \