8315479: GenShen: Expand old-gen while selecting collection set during GLOBAL GC

Reviewed-by: wkemper

Author: kdnilsen

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp

@@ -80,15 +80,39 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti
                                                               size_t size, size_t actual_free,
                                                               size_t cur_young_garbage) const {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
+  size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
   size_t capacity = heap->young_generation()->max_capacity();
-  size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100;
-  size_t ignore_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahIgnoreGarbageThreshold / 100;
+  size_t garbage_threshold = region_size_bytes * ShenandoahGarbageThreshold / 100;
+  size_t ignore_threshold = region_size_bytes * ShenandoahIgnoreGarbageThreshold / 100;
   const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
 
   size_t max_young_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste);
   size_t young_cur_cset = 0;
   size_t max_old_cset = (size_t) (heap->get_old_evac_reserve() / ShenandoahOldEvacWaste);
   size_t old_cur_cset = 0;
+
+  // Figure out how many unaffiliated young regions are dedicated to mutator and to evacuator.  Allow the young
+  // collector's unaffiliated regions to be transferred to old-gen if old-gen has more easily reclaimed garbage
+  // than young-gen.  At the end of this cycle, any excess regions remaining in old-gen will be transferred back
+  // to young.  Do not transfer the mutator's unaffiliated regions to old-gen.  Those must remain available
+  // to the mutator as it needs to be able to consume this memory during concurrent GC.
+
+  size_t unaffiliated_young_regions = heap->young_generation()->free_unaffiliated_regions();
+  size_t unaffiliated_young_memory = unaffiliated_young_regions * region_size_bytes;
+
+  if (unaffiliated_young_memory > max_young_cset) {
+    size_t unaffiliated_mutator_memory = unaffiliated_young_memory - max_young_cset;
+    unaffiliated_young_memory -= unaffiliated_mutator_memory;
+    unaffiliated_young_regions = unaffiliated_young_memory / region_size_bytes; // round down
+    unaffiliated_young_memory = unaffiliated_young_regions * region_size_bytes;
+  }
+
+  // We'll affiliate these unaffiliated regions with either old or young, depending on need.
+  max_young_cset -= unaffiliated_young_memory;
+
+  // Keep track of how many regions we plan to transfer from young to old.
+  size_t regions_transferred_to_old = 0;
+
   size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_young_cset;
   size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0;
 
@@ -101,31 +125,50 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti
   for (size_t idx = 0; idx < size; idx++) {
     ShenandoahHeapRegion* r = data[idx]._region;
     if (cset->is_preselected(r->index())) {
+      fatal("There should be no preselected regions during GLOBAL GC");
       continue;
     }
     bool add_region = false;
-    if (r->is_old()) {
+    if (r->is_old() || (r->age() >= tenuring_threshold)) {
       size_t new_cset = old_cur_cset + r->get_live_data_bytes();
+      if ((r->garbage() > garbage_threshold)) {
+        while ((new_cset > max_old_cset) && (unaffiliated_young_regions > 0)) {
+          unaffiliated_young_regions--;
+          regions_transferred_to_old++;
+          max_old_cset += region_size_bytes / ShenandoahOldEvacWaste;
+        }
+      }
       if ((new_cset <= max_old_cset) && (r->garbage() > garbage_threshold)) {
         add_region = true;
         old_cur_cset = new_cset;
       }
-    } else if (r->age() < tenuring_threshold) {
+    } else {
+      assert(r->is_young() && (r->age() < tenuring_threshold), "DeMorgan's law (assuming r->is_affiliated)");
       size_t new_cset = young_cur_cset + r->get_live_data_bytes();
       size_t region_garbage = r->garbage();
       size_t new_garbage = cur_young_garbage + region_garbage;
       bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage);
+
+      if (add_regardless || (r->garbage() > garbage_threshold)) {
+        while ((new_cset > max_young_cset) && (unaffiliated_young_regions > 0)) {
+          unaffiliated_young_regions--;
+          max_young_cset += region_size_bytes / ShenandoahEvacWaste;
+        }
+      }
       if ((new_cset <= max_young_cset) && (add_regardless || (region_garbage > garbage_threshold))) {
         add_region = true;
         young_cur_cset = new_cset;
         cur_young_garbage = new_garbage;
       }
     }
-    // Note that we do not add aged regions if they were not pre-selected.  The reason they were not preselected
-    // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects.
-
     if (add_region) {
       cset->add_region(r);
     }
   }
+
+  if (regions_transferred_to_old > 0) {
+    heap->generation_sizer()->force_transfer_to_old(regions_transferred_to_old);
+    heap->set_young_evac_reserve(heap->get_young_evac_reserve() - regions_transferred_to_old * region_size_bytes);
+    heap->set_old_evac_reserve(heap->get_old_evac_reserve() + regions_transferred_to_old * region_size_bytes);
+  }
 }

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -298,7 +298,23 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool
   // do not add to the update-refs burden of GC.
 
   size_t old_promo_reserve;
-  if (old_heuristics->unprocessed_old_collection_candidates() > 0) {
+  if (is_global()) {
+    // Global GC is typically triggered by user invocation of System.gc(), and typically indicates that there is lots
+    // of garbage to be reclaimed because we are starting a new phase of execution.  Marking for global GC may take
+    // significantly longer than typical young marking because we must mark through all old objects.  To expedite
+    // evacuation and update-refs, we give emphasis to reclaiming garbage first, wherever that garbage is found.
+    // Global GC will adjust generation sizes to accommodate the collection set it chooses.
+
+    // Set old_promo_reserve to enforce that no regions are preselected for promotion.  Such regions typically
+    // have relatively high memory utilization.  We still call select_aged_regions() because this will prepare for
+    // promotions in place, if relevant.
+    old_promo_reserve = 0;
+
+    // Dedicate all available old memory to old_evacuation reserve.  This may be small, because old-gen is only
+    // expanded based on an existing mixed evacuation workload at the end of the previous GC cycle.  We'll expand
+    // the budget for evacuation of old during GLOBAL cset selection.
+    old_evacuation_reserve = maximum_old_evacuation_reserve;
+  } else if (old_heuristics->unprocessed_old_collection_candidates() > 0) {
     // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen.  If this is
     // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote.  Prioritize compaction
     // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory.
@@ -319,21 +335,25 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool
     size_t delta = old_evacuation_reserve - old_free_unfragmented;
     old_evacuation_reserve -= delta;
 
-    // Let promo consume fragments of old-gen memory.
-    old_promo_reserve += delta;
+    // Let promo consume fragments of old-gen memory if not global
+    if (!is_global()) {
+      old_promo_reserve += delta;
+    }
   }
   collection_set->establish_preselected(preselected_regions);
   consumed_by_advance_promotion = select_aged_regions(old_promo_reserve, num_regions, preselected_regions);
   assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory");
-  if (consumed_by_advance_promotion < old_promo_reserve) {
-    // If we're in a global collection, this memory can be used for old evacuations
-    old_evacuation_reserve += old_promo_reserve - consumed_by_advance_promotion;
-  }
+
+  // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion.  Do not transfer this
+  // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood
+  // of old evacuatino failure.
+
   heap->set_young_evac_reserve(young_evacuation_reserve);
   heap->set_old_evac_reserve(old_evacuation_reserve);
   heap->set_promoted_reserve(consumed_by_advance_promotion);
 
-  // There is no need to expand OLD because all memory used here was set aside at end of previous GC
+  // There is no need to expand OLD because all memory used here was set aside at end of previous GC, except in the
+  // case of a GLOBAL gc.  During choose_collection_set() of GLOBAL, old will be expanded on demand.
 }
 
 // Having chosen the collection set, adjust the budgets for generational mode based on its composition.  Note