8341485: GenShen: Make evac tracker a non-product feature and confine…

… it to generational mode

Reviewed-by: ysr
Backport-of: bd7d34d8e4c89a855a2398965f776803ded557ea

src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp

@@ -27,14 +27,13 @@
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
-#include "gc/shenandoah/shenandoahRootProcessor.hpp"
 #include "runtime/threadSMR.inline.hpp"
 #include "runtime/thread.hpp"
 
-ShenandoahEvacuationStats::ShenandoahEvacuationStats(bool generational)
+ShenandoahEvacuationStats::ShenandoahEvacuationStats()
   : _evacuations_completed(0), _bytes_completed(0),
     _evacuations_attempted(0), _bytes_attempted(0),
-    _use_age_table(generational && (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring)) {
+    _use_age_table(ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
   if (_use_age_table) {
     _age_table = new AgeTable(false);
   }
@@ -81,6 +80,7 @@ void ShenandoahEvacuationStats::reset() {
 }
 
 void ShenandoahEvacuationStats::print_on(outputStream* st) {
+#ifndef PRODUCT
   size_t abandoned_size = _bytes_attempted - _bytes_completed;
   size_t abandoned_count = _evacuations_attempted - _evacuations_completed;
   st->print_cr("Evacuated " SIZE_FORMAT "%s across " SIZE_FORMAT " objects, "
@@ -89,6 +89,7 @@ void ShenandoahEvacuationStats::print_on(outputStream* st) {
             _evacuations_completed,
             byte_size_in_proper_unit(abandoned_size),   proper_unit_for_byte_size(abandoned_size),
             abandoned_count);
+#endif
   if (_use_age_table) {
     shenandoah_assert_generational();
     _age_table->print_on(st, ShenandoahGenerationalHeap::heap()->age_census()->tenuring_threshold());
@@ -109,35 +110,33 @@ void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st,
   mutators->print_on(st);
   st->cr();
 
-  if (_generational) {
-    ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
-    AgeTable young_region_ages(false);
-    for (uint i = 0; i < heap->num_regions(); ++i) {
-      ShenandoahHeapRegion* r = heap->get_region(i);
-      if (r->is_young()) {
-        young_region_ages.add(r->age(), r->get_live_data_words());
-      }
+  ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap();
+  AgeTable young_region_ages(false);
+  for (uint i = 0; i < heap->num_regions(); ++i) {
+    ShenandoahHeapRegion* r = heap->get_region(i);
+    if (r->is_young()) {
+      young_region_ages.add(r->age(), r->get_live_data_words());
     }
-    uint tenuring_threshold = heap->age_census()->tenuring_threshold();
-    st->print("Young regions: ");
-    young_region_ages.print_on(st, tenuring_threshold);
-    st->cr();
   }
+  uint tenuring_threshold = heap->age_census()->tenuring_threshold();
+  st->print("Young regions: ");
+  young_region_ages.print_on(st, tenuring_threshold);
+  st->cr();
 }
 
 class ShenandoahStatAggregator : public ThreadClosure {
 public:
   ShenandoahEvacuationStats* _target;
   explicit ShenandoahStatAggregator(ShenandoahEvacuationStats* target) : _target(target) {}
-  virtual void do_thread(Thread* thread) override {
+  void do_thread(Thread* thread) override {
     ShenandoahEvacuationStats* local = ShenandoahThreadLocalData::evacuation_stats(thread);
     _target->accumulate(local);
     local->reset();
   }
 };
 
 ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() {
-  ShenandoahEvacuationStats mutators(_generational), workers(_generational);
+  ShenandoahEvacuationStats mutators, workers;
 
   ThreadsListHandle java_threads_iterator;
   ShenandoahStatAggregator aggregate_mutators(&mutators);
@@ -149,7 +148,7 @@ ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() {
   _mutators_global.accumulate(&mutators);
   _workers_global.accumulate(&workers);
 
-  if (_generational && (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring)) {
+  if (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring) {
     // Ingest mutator & worker collected population vectors into the heap's
     // global census data, and use it to compute an appropriate tenuring threshold
     // for use in the next cycle.

src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp

@@ -39,7 +39,7 @@ class ShenandoahEvacuationStats : public CHeapObj<mtGC> {
   AgeTable* _age_table;
 
  public:
-  ShenandoahEvacuationStats(bool generational);
+  ShenandoahEvacuationStats();
 
   AgeTable* age_table() const;
 
@@ -59,25 +59,21 @@ struct ShenandoahCycleStats {
 
 class ShenandoahEvacuationTracker : public CHeapObj<mtGC> {
 private:
-  bool _generational;
 
   ShenandoahEvacuationStats _workers_global;
   ShenandoahEvacuationStats _mutators_global;
 
 public:
-  ShenandoahEvacuationTracker(bool generational) :
-   _generational(generational),
-   _workers_global(generational),
-   _mutators_global(generational) {}
+  ShenandoahEvacuationTracker() = default;
 
   void begin_evacuation(Thread* thread, size_t bytes);
   void end_evacuation(Thread* thread, size_t bytes);
   void record_age(Thread* thread, size_t bytes, uint age);
 
   void print_global_on(outputStream* st);
   void print_evacuations_on(outputStream* st,
-                                   ShenandoahEvacuationStats* workers,
-                                   ShenandoahEvacuationStats* mutators);
+                            ShenandoahEvacuationStats* workers,
+                            ShenandoahEvacuationStats* mutators);
 
   ShenandoahCycleStats flush_cycle_to_global();
 };

src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp

@@ -34,6 +34,7 @@
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
 #include "gc/shenandoah/shenandoahFullGC.hpp"
 #include "gc/shenandoah/shenandoahGeneration.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 #include "gc/shenandoah/shenandoahOldGC.hpp"
 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
@@ -340,7 +341,7 @@ void ShenandoahGenerationalControlThread::run_service() {
   }
 }
 
-void ShenandoahGenerationalControlThread::process_phase_timings(const ShenandoahHeap* heap) {
+void ShenandoahGenerationalControlThread::process_phase_timings(const ShenandoahGenerationalHeap* heap) {
   // Commit worker statistics to cycle data
   heap->phase_timings()->flush_par_workers_to_cycle();
   if (ShenandoahPacing) {
@@ -394,9 +395,9 @@ void ShenandoahGenerationalControlThread::process_phase_timings(const Shenandoah
 //      |        v                                   v       |
 //      +--->  Global Degen +--------------------> Full <----+
 //
-void ShenandoahGenerationalControlThread::service_concurrent_normal_cycle(ShenandoahHeap* heap,
-                                                              const ShenandoahGenerationType generation,
-                                                              GCCause::Cause cause) {
+void ShenandoahGenerationalControlThread::service_concurrent_normal_cycle(ShenandoahGenerationalHeap* heap,
+                                                                          const ShenandoahGenerationType generation,
+                                                                          GCCause::Cause cause) {
   GCIdMark gc_id_mark;
   switch (generation) {
     case YOUNG: {
@@ -424,7 +425,7 @@ void ShenandoahGenerationalControlThread::service_concurrent_normal_cycle(Shenan
   }
 }
 
-void ShenandoahGenerationalControlThread::service_concurrent_old_cycle(ShenandoahHeap* heap, GCCause::Cause &cause) {
+void ShenandoahGenerationalControlThread::service_concurrent_old_cycle(ShenandoahGenerationalHeap* heap, GCCause::Cause &cause) {
   ShenandoahOldGeneration* old_generation = heap->old_generation();
   ShenandoahYoungGeneration* young_generation = heap->young_generation();
   ShenandoahOldGeneration::State original_state = old_generation->state();

src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.hpp

@@ -28,12 +28,15 @@
 
 #include "gc/shared/gcCause.hpp"
 #include "gc/shenandoah/shenandoahController.hpp"
+#include "gc/shenandoah/shenandoahGenerationType.hpp"
 #include "gc/shenandoah/shenandoahGC.hpp"
-#include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahPadding.hpp"
 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
 
 class ShenandoahOldGeneration;
+class ShenandoahGeneration;
+class ShenandoahGenerationalHeap;
+class ShenandoahHeap;
 
 class ShenandoahGenerationalControlThread: public ShenandoahController {
   friend class VMStructs;
@@ -101,13 +104,13 @@ class ShenandoahGenerationalControlThread: public ShenandoahController {
   // Returns true if the old generation marking was interrupted to allow a young cycle.
   bool preempt_old_marking(ShenandoahGenerationType generation);
 
-  void process_phase_timings(const ShenandoahHeap* heap);
+  void process_phase_timings(const ShenandoahGenerationalHeap* heap);
 
-  void service_concurrent_normal_cycle(ShenandoahHeap* heap,
+  void service_concurrent_normal_cycle(ShenandoahGenerationalHeap* heap,
                                        ShenandoahGenerationType generation,
                                        GCCause::Cause cause);
 
-  void service_concurrent_old_cycle(ShenandoahHeap* heap,
+  void service_concurrent_old_cycle(ShenandoahGenerationalHeap* heap,
                                     GCCause::Cause &cause);
 
   void set_gc_mode(GCMode new_mode);

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp

@@ -95,6 +95,7 @@ size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc(Thread *thread) const {
 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
   ShenandoahHeap(policy),
   _age_census(nullptr),
+  _evac_tracker(new ShenandoahEvacuationTracker()),
   _min_plab_size(calculate_min_plab()),
   _max_plab_size(calculate_max_plab()),
   _regulator_thread(nullptr),
@@ -114,6 +115,18 @@ void ShenandoahGenerationalHeap::print_init_logger() const {
   logger.print_all();
 }
 
+void ShenandoahGenerationalHeap::print_tracing_info() const {
+  ShenandoahHeap::print_tracing_info();
+
+  LogTarget(Info, gc, stats) lt;
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
+    ls.cr();
+    ls.cr();
+    evac_tracker()->print_global_on(&ls);
+  }
+}
+
 void ShenandoahGenerationalHeap::initialize_heuristics() {
   // Initialize global generation and heuristics even in generational mode.
   ShenandoahHeap::initialize_heuristics();
@@ -317,7 +330,7 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, Shena
   }
 
   // Copy the object:
-  evac_tracker()->begin_evacuation(thread, size * HeapWordSize);
+  NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
   oop copy_val = cast_to_oop(copy);
 
@@ -339,7 +352,7 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, Shena
     ContinuationGCSupport::relativize_stack_chunk(copy_val);
 
     // Record that the evacuation succeeded
-    evac_tracker()->end_evacuation(thread, size * HeapWordSize);
+    NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
 
     if (target_gen == OLD_GENERATION) {
       old_generation()->handle_evacuation(copy, size, from_region->is_young());

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp

@@ -52,6 +52,8 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap {
   }
 
   void print_init_logger() const override;
+  void print_tracing_info() const override;
+
   size_t unsafe_max_tlab_alloc(Thread *thread) const override;
 
 private:
@@ -61,6 +63,8 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap {
   ShenandoahSharedFlag  _is_aging_cycle;
   // Age census used for adapting tenuring threshold
   ShenandoahAgeCensus* _age_census;
+  // Used primarily to look for failed evacuation attempts.
+  ShenandoahEvacuationTracker*  _evac_tracker;
 
 public:
   void set_aging_cycle(bool cond) {
@@ -76,6 +80,10 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap {
     return _age_census;
   }
 
+  ShenandoahEvacuationTracker* evac_tracker() const {
+    return _evac_tracker;
+  }
+
   // Ages regions that haven't been used for allocations in the current cycle.
   // Resets ages for regions that have been used for allocations.
   void update_region_ages(ShenandoahMarkingContext* ctx);

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -516,7 +516,6 @@ void ShenandoahHeap::initialize_mode() {
 void ShenandoahHeap::initialize_heuristics() {
   _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity(), max_capacity());
   _global_generation->initialize_heuristics(mode());
-  _evac_tracker = new ShenandoahEvacuationTracker(mode()->is_generational());
 }
 
 #ifdef _MSC_VER
@@ -551,7 +550,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
   _pacer(nullptr),
   _verifier(nullptr),
   _phase_timings(nullptr),
-  _evac_tracker(nullptr),
   _mmu_tracker(),
   _monitoring_support(nullptr),
   _memory_pool(nullptr),
@@ -1248,15 +1246,7 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
 #endif
     if (UseTLAB) {
       copy = allocate_from_gclab(thread, size);
-      if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) {
-        // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve.  Try resetting
-        // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations.
-        ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size());
-        copy = allocate_from_gclab(thread, size);
-        // If we still get nullptr, we'll try a shared allocation below.
-      }
     }
-
     if (copy == nullptr) {
       // If we failed to allocate in LAB, we'll try a shared allocation.
       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
@@ -1276,18 +1266,14 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
   }
 
   // Copy the object:
-  _evac_tracker->begin_evacuation(thread, size * HeapWordSize);
   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
 
-  oop copy_val = cast_to_oop(copy);
-
   // Try to install the new forwarding pointer.
-  ContinuationGCSupport::relativize_stack_chunk(copy_val);
-
+  oop copy_val = cast_to_oop(copy);
   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
   if (result == copy_val) {
     // Successfully evacuated. Our copy is now the public one!
-    _evac_tracker->end_evacuation(thread, size * HeapWordSize);
+    ContinuationGCSupport::relativize_stack_chunk(copy_val);
     shenandoah_assert_correct(nullptr, copy_val);
     return copy_val;
   }  else {
@@ -1534,10 +1520,6 @@ void ShenandoahHeap::print_tracing_info() const {
 
     shenandoah_policy()->print_gc_stats(&ls);
 
-    ls.cr();
-
-    evac_tracker()->print_global_on(&ls);
-
     ls.cr();
     ls.cr();
   }

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -509,7 +509,6 @@ class ShenandoahHeap : public CollectedHeap {
   ShenandoahVerifier*        _verifier;
 
   ShenandoahPhaseTimings*       _phase_timings;
-  ShenandoahEvacuationTracker*  _evac_tracker;
   ShenandoahMmuTracker          _mmu_tracker;
 
 public:
@@ -534,7 +533,6 @@ class ShenandoahHeap : public CollectedHeap {
   ShenandoahPacer*           pacer()             const { return _pacer;             }
 
   ShenandoahPhaseTimings*      phase_timings()   const { return _phase_timings;     }
-  ShenandoahEvacuationTracker* evac_tracker()    const { return _evac_tracker;      }
 
   ShenandoahEvacOOMHandler* oom_evac_handler() { return &_oom_evac_handler; }
 

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp

@@ -46,8 +46,9 @@ ShenandoahThreadLocalData::ShenandoahThreadLocalData() :
   _plab_allows_promotion(true),
   _plab_retries_enabled(true),
   _evacuation_stats(nullptr) {
-  bool gen_mode = ShenandoahHeap::heap()->mode()->is_generational();
-  _evacuation_stats = new ShenandoahEvacuationStats(gen_mode);
+  if (ShenandoahHeap::heap()->mode()->is_generational()) {
+    _evacuation_stats = new ShenandoahEvacuationStats();
+  }
 }
 
 ShenandoahThreadLocalData::~ShenandoahThreadLocalData() {

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp

@@ -152,6 +152,7 @@ class ShenandoahThreadLocalData {
   }
 
   static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) {
+    shenandoah_assert_generational();
     return data(thread)->_evacuation_stats;
   }