8241139: Shenandoah: distribute mark-compact work exactly to minimize…

… fragmentation

Reviewed-by: rkennke

src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp

@@ -332,57 +332,46 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
   PreservedMarksSet*        const _preserved_marks;
   ShenandoahHeap*           const _heap;
   ShenandoahHeapRegionSet** const _worker_slices;
-  ShenandoahRegionIterator        _heap_regions;
-
-  ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
-    ShenandoahHeapRegion* from_region = _heap_regions.next();
-
-    // Look for next candidate for this slice:
-    while (from_region != NULL) {
-      // Empty region: get it into the slice to defragment the slice itself.
-      // We could have skipped this without violating correctness, but we really
-      // want to compact all live regions to the start of the heap, which sometimes
-      // means moving them into the fully empty regions.
-      if (from_region->is_empty()) break;
-
-      // Can move the region, and this is not the humongous region. Humongous
-      // moves are special cased here, because their moves are handled separately.
-      if (from_region->is_stw_move_allowed() && !from_region->is_humongous()) break;
-
-      from_region = _heap_regions.next();
-    }
-
-    if (from_region != NULL) {
-      assert(slice != NULL, "sanity");
-      assert(!from_region->is_humongous(), "this path cannot handle humongous regions");
-      assert(from_region->is_empty() || from_region->is_stw_move_allowed(), "only regions that can be moved in mark-compact");
-      slice->add_region(from_region);
-    }
-
-    return from_region;
-  }
 
 public:
-  ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
+  ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
     AbstractGangTask("Shenandoah Prepare For Compaction Task"),
     _preserved_marks(preserved_marks),
     _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
   }
 
+  static bool is_candidate_region(ShenandoahHeapRegion* r) {
+    // Empty region: get it into the slice to defragment the slice itself.
+    // We could have skipped this without violating correctness, but we really
+    // want to compact all live regions to the start of the heap, which sometimes
+    // means moving them into the fully empty regions.
+    if (r->is_empty()) return true;
+
+    // Can move the region, and this is not the humongous region. Humongous
+    // moves are special cased here, because their moves are handled separately.
+    return r->is_stw_move_allowed() && !r->is_humongous();
+  }
+
   void work(uint worker_id) {
     ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
-    ShenandoahHeapRegion* from_region = next_from_region(slice);
+    ShenandoahHeapRegionSetIterator it(slice);
+    ShenandoahHeapRegion* from_region = it.next();
     // No work?
     if (from_region == NULL) {
-      return;
+       return;
     }
 
     // Sliding compaction. Walk all regions in the slice, and compact them.
     // Remember empty regions and reuse them as needed.
     ResourceMark rm;
+
     GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
+
     ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+
     while (from_region != NULL) {
+      assert(is_candidate_region(from_region), "Sanity");
+
       cl.set_from_region(from_region);
       if (from_region->has_live()) {
         _heap->marked_object_iterate(from_region, &cl);
@@ -392,7 +381,7 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
       if (!cl.is_compact_same_region()) {
         empty_regions.append(from_region);
       }
-      from_region = next_from_region(slice);
+      from_region = it.next();
     }
     cl.finish_region();
 
@@ -509,6 +498,148 @@ class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure
   }
 };
 
+void ShenandoahMarkCompact::distribute_slices(ShenandoahHeapRegionSet** worker_slices) {
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+
+  uint n_workers = heap->workers()->active_workers();
+  size_t n_regions = heap->num_regions();
+
+  // What we want to accomplish: have the dense prefix of data, while still balancing
+  // out the parallel work.
+  //
+  // Assuming the amount of work is driven by the live data that needs moving, we can slice
+  // the entire heap into equal-live-sized prefix slices, and compact into them. So, each
+  // thread takes all regions in its prefix subset, and then it takes some regions from
+  // the tail.
+  //
+  // Tail region selection becomes interesting.
+  //
+  // First, we want to distribute the regions fairly between the workers, and those regions
+  // might have different amount of live data. So, until we sure no workers need live data,
+  // we need to only take what the worker needs.
+  //
+  // Second, since we slide everything to the left in each slice, the most busy regions
+  // would be the ones on the left. Which means we want to have all workers have their after-tail
+  // regions as close to the left as possible.
+  //
+  // The easiest way to do this is to distribute after-tail regions in round-robin between
+  // workers that still need live data.
+  //
+  // Consider parallel workers A, B, C, then the target slice layout would be:
+  //
+  //  AAAAAAAABBBBBBBBCCCCCCCC|ABCABCABCABCABCABCABCABABABABABABABABABABAAAAA
+  //
+  //  (.....dense-prefix.....) (.....................tail...................)
+  //  [all regions fully live] [left-most regions are fuller that right-most]
+  //
+
+  // Compute how much live data is there. This would approximate the size of dense prefix
+  // we target to create.
+  size_t total_live = 0;
+  for (size_t idx = 0; idx < n_regions; idx++) {
+    ShenandoahHeapRegion *r = heap->get_region(idx);
+    if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
+      total_live += r->get_live_data_words();
+    }
+  }
+
+  // Estimate the size for the dense prefix. Note that we specifically count only the
+  // "full" regions, so there would be some non-full regions in the slice tail.
+  size_t live_per_worker = total_live / n_workers;
+  size_t prefix_regions_per_worker = live_per_worker / ShenandoahHeapRegion::region_size_words();
+  size_t prefix_regions_total = prefix_regions_per_worker * n_workers;
+  prefix_regions_total = MIN2(prefix_regions_total, n_regions);
+  assert(prefix_regions_total <= n_regions, "Sanity");
+
+  // There might be non-candidate regions in the prefix. To compute where the tail actually
+  // ends up being, we need to account those as well.
+  size_t prefix_end = prefix_regions_total;
+  for (size_t idx = 0; idx < prefix_regions_total; idx++) {
+    ShenandoahHeapRegion *r = heap->get_region(idx);
+    if (!ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
+      prefix_end++;
+    }
+  }
+  prefix_end = MIN2(prefix_end, n_regions);
+  assert(prefix_end <= n_regions, "Sanity");
+
+  // Distribute prefix regions per worker: each thread definitely gets its own same-sized
+  // subset of dense prefix.
+  size_t prefix_idx = 0;
+
+  size_t* live = NEW_C_HEAP_ARRAY(size_t, n_workers, mtGC);
+
+  for (size_t wid = 0; wid < n_workers; wid++) {
+    ShenandoahHeapRegionSet* slice = worker_slices[wid];
+
+    live[wid] = 0;
+    size_t regs = 0;
+
+    // Add all prefix regions for this worker
+    while (prefix_idx < prefix_end && regs < prefix_regions_per_worker) {
+      ShenandoahHeapRegion *r = heap->get_region(prefix_idx);
+      if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
+        slice->add_region(r);
+        live[wid] += r->get_live_data_words();
+        regs++;
+      }
+      prefix_idx++;
+    }
+  }
+
+  // Distribute the tail among workers in round-robin fashion.
+  size_t wid = n_workers - 1;
+
+  for (size_t tail_idx = prefix_end; tail_idx < n_regions; tail_idx++) {
+    ShenandoahHeapRegion *r = heap->get_region(tail_idx);
+    if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
+      assert(wid < n_workers, "Sanity");
+
+      size_t live_region = r->get_live_data_words();
+
+      // Select next worker that still needs live data.
+      size_t old_wid = wid;
+      do {
+        wid++;
+        if (wid == n_workers) wid = 0;
+      } while (live[wid] + live_region >= live_per_worker && old_wid != wid);
+
+      if (old_wid == wid) {
+        // Circled back to the same worker? This means liveness data was
+        // miscalculated. Bump the live_per_worker limit so that
+        // everyone gets a piece of the leftover work.
+        live_per_worker += ShenandoahHeapRegion::region_size_words();
+      }
+
+      worker_slices[wid]->add_region(r);
+      live[wid] += live_region;
+    }
+  }
+
+  FREE_C_HEAP_ARRAY(size_t, live);
+
+#ifdef ASSERT
+  ResourceBitMap map(n_regions);
+  for (size_t wid = 0; wid < n_workers; wid++) {
+    ShenandoahHeapRegionSetIterator it(worker_slices[wid]);
+    ShenandoahHeapRegion* r = it.next();
+    while (r != NULL) {
+      size_t num = r->region_number();
+      assert(ShenandoahPrepareForCompactionTask::is_candidate_region(r), "Sanity: " SIZE_FORMAT, num);
+      assert(!map.at(num), "No region distributed twice: " SIZE_FORMAT, num);
+      map.at_put(num, true);
+      r = it.next();
+    }
+  }
+
+  for (size_t rid = 0; rid < n_regions; rid++) {
+    bool is_candidate = ShenandoahPrepareForCompactionTask::is_candidate_region(heap->get_region(rid));
+    bool is_distributed = map.at(rid);
+    assert(is_distributed || !is_candidate, "All candidates are distributed: " SIZE_FORMAT, rid);
+  }
+#endif
+}
+
 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
   GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
   ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
@@ -533,8 +664,11 @@ void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegi
   // Compute the new addresses for regular objects
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
-    ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
-    heap->workers()->run_task(&prepare_task);
+
+    distribute_slices(worker_slices);
+
+    ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices);
+    heap->workers()->run_task(&task);
   }
 
   // Compute the new addresses for humongous objects

src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.hpp

@@ -71,6 +71,7 @@ class ShenandoahMarkCompact : public CHeapObj<mtGC> {
   void phase3_update_references();
   void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices);
 
+  void distribute_slices(ShenandoahHeapRegionSet** worker_slices);
   void calculate_target_humongous_objects();
   void compact_humongous_objects();
 };