8273492: Move evacuation failure handling into G1YoungCollector

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1752,8 +1752,6 @@ jint G1CollectedHeap::initialize() {
 
   _collection_set.initialize(max_reserved_regions());
 
-  _evac_failure_regions.initialize(max_reserved_regions());
-
   evac_failure_injector()->reset();
 
   G1InitLogger::print();
@@ -2772,10 +2770,6 @@ void G1CollectedHeap::verify_after_young_collection(G1HeapVerifier::G1VerifyType
     return;
   }
   Ticks start = Ticks::now();
-  // Inject evacuation failure tag into type if needed.
-  if (evacuation_failed()) {
-    type = (G1HeapVerifier::G1VerifyType)(type | G1HeapVerifier::G1VerifyYoungEvacFail);
-  }
   if (VerifyRememberedSets) {
     log_info(gc, verify)("[Verifying RemSets after GC]");
     VerifyRegionRemSetClosure v_cl;
@@ -2884,7 +2878,7 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
   bool should_start_concurrent_mark_operation = collector_state()->in_concurrent_start_gc();
 
   // Perform the collection.
-  G1YoungCollector collector(gc_cause(), target_pause_time_ms, &_evac_failure_regions);
+  G1YoungCollector collector(gc_cause(), target_pause_time_ms);
   collector.collect();
 
   // It should now be safe to tell the concurrent mark thread to start
@@ -3392,8 +3386,8 @@ void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
   nm->oops_do(&reg_cl, true);
 }
 
-void G1CollectedHeap::update_used_after_gc() {
-  if (evacuation_failed()) {
+void G1CollectedHeap::update_used_after_gc(bool evacuation_failed) {
+  if (evacuation_failed) {
     // Reset the G1EvacuationFailureALot counters and flags
     evac_failure_injector()->reset();
 

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -34,7 +34,6 @@
 #include "gc/g1/g1ConcurrentMark.hpp"
 #include "gc/g1/g1EdenRegions.hpp"
 #include "gc/g1/g1EvacStats.hpp"
-#include "gc/g1/g1EvacFailureRegions.hpp"
 #include "gc/g1/g1GCPauseType.hpp"
 #include "gc/g1/g1HeapTransition.hpp"
 #include "gc/g1/g1HeapVerifier.hpp"
@@ -816,8 +815,6 @@ class G1CollectedHeap : public CollectedHeap {
   // The parallel task queues
   G1ScannerTasksQueueSet *_task_queues;
 
-  G1EvacFailureRegions _evac_failure_regions;
-
   // ("Weak") Reference processing support.
   //
   // G1 has 2 instances of the reference processor class.
@@ -1035,9 +1032,6 @@ class G1CollectedHeap : public CollectedHeap {
 
   void start_concurrent_gc_for_metadata_allocation(GCCause::Cause gc_cause);
 
-  // True iff an evacuation has failed in the most-recent collection.
-  inline bool evacuation_failed() const;
-
   void remove_from_old_gen_sets(const uint old_regions_removed,
                                 const uint archive_regions_removed,
                                 const uint humongous_regions_removed);
@@ -1283,7 +1277,7 @@ class G1CollectedHeap : public CollectedHeap {
 
   // Recalculate amount of used memory after GC. Must be called after all allocation
   // has finished.
-  void update_used_after_gc();
+  void update_used_after_gc(bool evacuation_failed);
   // Reset and re-enable the hot card cache.
   // Note the counts for the cards in the regions in the
   // collection set are reset when the collection set is freed.

src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp

@@ -201,10 +201,6 @@ void G1CollectedHeap::register_optional_region_with_region_attr(HeapRegion* r) {
   _region_attr.set_optional(r->hrm_index(), r->rem_set()->is_tracked());
 }
 
-bool G1CollectedHeap::evacuation_failed() const {
-  return _evac_failure_regions.num_regions_failed_evacuation() > 0;
-}
-
 inline bool G1CollectedHeap::is_in_young(const oop obj) {
   if (obj == NULL) {
     return false;

src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp

@@ -25,27 +25,35 @@
 #include "precompiled.hpp"
 
 #include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/g1EvacFailureRegions.hpp"
+#include "gc/g1/g1EvacFailureRegions.inline.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "memory/allocation.hpp"
 #include "runtime/atomic.hpp"
 
-
 G1EvacFailureRegions::G1EvacFailureRegions() :
-  _regions_failed_evacuation(mtGC) {
-}
+  _regions_failed_evacuation(mtGC),
+  _evac_failure_regions(nullptr),
+  _evac_failure_regions_cur_length(0),
+  _max_regions(0) { }
 
 G1EvacFailureRegions::~G1EvacFailureRegions() {
-  FREE_C_HEAP_ARRAY(uint, _evac_failure_regions);
+  assert(_evac_failure_regions == nullptr, "not cleaned up");
 }
 
-void G1EvacFailureRegions::initialize(uint max_regions) {
+void G1EvacFailureRegions::pre_collection(uint max_regions) {
   Atomic::store(&_evac_failure_regions_cur_length, 0u);
   _max_regions = max_regions;
   _regions_failed_evacuation.resize(_max_regions);
   _evac_failure_regions = NEW_C_HEAP_ARRAY(uint, _max_regions, mtGC);
 }
 
+void G1EvacFailureRegions::post_collection() {
+  _regions_failed_evacuation.resize(0);
+  FREE_C_HEAP_ARRAY(uint, _evac_failure_regions);
+  _evac_failure_regions = nullptr;
+  _max_regions = 0; // To have any record() attempt fail in the future.
+}
+
 void G1EvacFailureRegions::par_iterate(HeapRegionClosure* closure,
                                        HeapRegionClaimer* _hrclaimer,
                                        uint worker_id) {
@@ -56,11 +64,6 @@ void G1EvacFailureRegions::par_iterate(HeapRegionClosure* closure,
                                                      worker_id);
 }
 
-void G1EvacFailureRegions::reset() {
-  Atomic::store(&_evac_failure_regions_cur_length, 0u);
-  _regions_failed_evacuation.clear();
-}
-
 bool G1EvacFailureRegions::contains(uint region_idx) const {
   assert(region_idx < _max_regions, "must be");
   return _regions_failed_evacuation.par_at(region_idx, memory_order_relaxed);

src/hotspot/share/gc/g1/g1EvacFailureRegions.hpp

@@ -47,9 +47,11 @@ class G1EvacFailureRegions {
 public:
   G1EvacFailureRegions();
   ~G1EvacFailureRegions();
-  void initialize(uint max_regions);
 
-  void reset();
+  // Sets up the bitmap and failed regions array for addition.
+  void pre_collection(uint max_regions);
+  // Drops memory for internal data structures, but keep counts.
+  void post_collection();
 
   bool contains(uint region_idx) const;
   void par_iterate(HeapRegionClosure* closure,
@@ -60,6 +62,10 @@ class G1EvacFailureRegions {
     return Atomic::load(&_evac_failure_regions_cur_length);
   }
 
+  bool evacuation_failed() const {
+    return num_regions_failed_evacuation() > 0;
+  }
+
   // Record that the garbage collection encountered an evacuation failure in the
   // given region. Returns whether this has been the first occurrence of an evacuation
   // failure in that region.

src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp

@@ -457,7 +457,7 @@ double G1GCPhaseTimes::print_evacuate_initial_collection_set() const {
   return _cur_collection_initial_evac_time_ms + _cur_merge_heap_roots_time_ms;
 }
 
-double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
+double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed) const {
   const double sum_ms = _cur_collection_nmethod_list_cleanup_time_ms +
                         _recorded_preserve_cm_referents_time_ms +
                         _cur_ref_proc_time_ms +
@@ -481,13 +481,13 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
   debug_phase(_gc_par_phases[MergePSS], 1);
   debug_phase(_gc_par_phases[ClearCardTable], 1);
   debug_phase(_gc_par_phases[RecalculateUsed], 1);
-  if (G1CollectedHeap::heap()->evacuation_failed()) {
+  if (evacuation_failed) {
     debug_phase(_gc_par_phases[RemoveSelfForwardingPtr], 1);
   }
 
   trace_phase(_gc_par_phases[RedirtyCards]);
   debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
-  if (G1CollectedHeap::heap()->evacuation_failed()) {
+  if (evacuation_failed) {
     debug_phase(_gc_par_phases[RecalculateUsed], 1);
     debug_phase(_gc_par_phases[RestorePreservedMarks], 1);
   }
@@ -526,7 +526,7 @@ void G1GCPhaseTimes::print_other(double accounted_ms) const {
   info_time("Other", _gc_pause_time_ms - accounted_ms);
 }
 
-void G1GCPhaseTimes::print() {
+void G1GCPhaseTimes::print(bool evacuation_failed) {
   // Check if some time has been recorded for verification and only then print
   // the message. We do not use Verify*GC here to print because VerifyGCType
   // further limits actual verification.
@@ -538,7 +538,7 @@ void G1GCPhaseTimes::print() {
   accounted_ms += print_pre_evacuate_collection_set();
   accounted_ms += print_evacuate_initial_collection_set();
   accounted_ms += print_evacuate_optional_collection_set();
-  accounted_ms += print_post_evacuate_collection_set();
+  accounted_ms += print_post_evacuate_collection_set(evacuation_failed);
   print_other(accounted_ms);
 
   // See above comment on the _cur_verify_before_time_ms check.

src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp

@@ -227,14 +227,14 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
   double print_merge_heap_roots_time() const;
   double print_evacuate_initial_collection_set() const;
   double print_evacuate_optional_collection_set() const;
-  double print_post_evacuate_collection_set() const;
+  double print_post_evacuate_collection_set(bool evacuation_failed) const;
   void print_other(double accounted_ms) const;
 
  public:
   G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads);
   void record_gc_pause_start();
   void record_gc_pause_end();
-  void print();
+  void print(bool evacuation_failed);
   static const char* phase_name(GCParPhases phase);
 
   // record the time a phase took in seconds

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -38,6 +38,7 @@
 #include "utilities/ticks.hpp"
 
 class G1CardTable;
+class G1EvacFailureRegions;
 class G1EvacuationRootClosures;
 class G1OopStarChunkedList;
 class G1PLABAllocator;

src/hotspot/share/gc/g1/g1Policy.cpp

@@ -629,7 +629,7 @@ double G1Policy::logged_cards_processing_time() const {
 // Anything below that is considered to be zero
 #define MIN_TIMER_GRANULARITY 0.0000001
 
-void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mark) {
+void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mark, bool evacuation_failure) {
   G1GCPhaseTimes* p = phase_times();
 
   double start_time_sec = phase_times()->cur_collection_start_sec();
@@ -638,8 +638,6 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
 
   G1GCPauseType this_pause = collector_state()->young_gc_pause_type(concurrent_operation_is_full_mark);
 
-  bool update_stats = should_update_gc_stats();
-
   if (G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause)) {
     record_concurrent_mark_init_end();
   } else {
@@ -654,6 +652,10 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
     app_time_ms = 1.0;
   }
 
+  // Evacuation failures skew the timing too much to be considered for some statistics updates.
+  // We make the assumption that these are rare.
+  bool update_stats = !evacuation_failure;
+
   if (update_stats) {
     // We maintain the invariant that all objects allocated by mutator
     // threads will be allocated out of eden regions. So, we can use
@@ -668,7 +670,7 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
     _analytics->report_alloc_rate_ms(alloc_rate_ms);
   }
 
-  record_pause(this_pause, start_time_sec, end_time_sec);
+  record_pause(this_pause, start_time_sec, end_time_sec, evacuation_failure);
 
   if (G1GCPauseTypeHelper::is_last_young_pause(this_pause)) {
     assert(!G1GCPauseTypeHelper::is_concurrent_start_pause(this_pause),
@@ -891,9 +893,9 @@ void G1Policy::report_ihop_statistics() {
   _ihop_control->print();
 }
 
-void G1Policy::record_young_gc_pause_end() {
+void G1Policy::record_young_gc_pause_end(bool evacuation_failed) {
   phase_times()->record_gc_pause_end();
-  phase_times()->print();
+  phase_times()->print(evacuation_failed);
 }
 
 double G1Policy::predict_base_elapsed_time_ms(size_t pending_cards,
@@ -1176,12 +1178,6 @@ void G1Policy::maybe_start_marking() {
   }
 }
 
-bool G1Policy::should_update_gc_stats() {
-  // Evacuation failures skew the timing too much to be considered for statistics updates.
-  // We make the assumption that these are rare.
-  return !_g1h->evacuation_failed();
-}
-
 void G1Policy::update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_time_sec, double end_time_sec) {
 
   double pause_time_sec = end_time_sec - start_time_sec;
@@ -1199,13 +1195,14 @@ void G1Policy::update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_t
 
 void G1Policy::record_pause(G1GCPauseType gc_type,
                             double start,
-                            double end) {
+                            double end,
+                            bool evacuation_failure) {
   // Manage the MMU tracker. For some reason it ignores Full GCs.
   if (gc_type != G1GCPauseType::FullGC) {
     _mmu_tracker->add_pause(start, end);
   }
 
-  if (should_update_gc_stats()) {
+  if (!evacuation_failure) {
     update_gc_pause_time_ratios(gc_type, start, end);
   }
 

src/hotspot/share/gc/g1/g1Policy.hpp

@@ -269,9 +269,10 @@ class G1Policy: public CHeapObj<mtGC> {
   // Manage time-to-mixed tracking.
   void update_time_to_mixed_tracking(G1GCPauseType gc_type, double start, double end);
   // Record the given STW pause with the given start and end times (in s).
-  void record_pause(G1GCPauseType gc_type, double start, double end);
-
-  bool should_update_gc_stats();
+  void record_pause(G1GCPauseType gc_type,
+                    double start,
+                    double end,
+                    bool evacuation_failure = false);
 
   void update_gc_pause_time_ratios(G1GCPauseType gc_type, double start_sec, double end_sec);
 
@@ -303,7 +304,7 @@ class G1Policy: public CHeapObj<mtGC> {
 
   // Record the start and end of the young gc pause.
   void record_young_gc_pause_start();
-  void record_young_gc_pause_end();
+  void record_young_gc_pause_end(bool evacuation_failed);
 
   bool need_to_start_conc_mark(const char* source, size_t alloc_word_size = 0);
 
@@ -313,7 +314,7 @@ class G1Policy: public CHeapObj<mtGC> {
 
   // Record the start and end of the actual collection part of the evacuation pause.
   void record_young_collection_start();
-  void record_young_collection_end(bool concurrent_operation_is_full_mark);
+  void record_young_collection_end(bool concurrent_operation_is_full_mark, bool evacuation_failure);
 
   // Record the start and end of a full collection.
   void record_full_collection_start();