8256265 G1: Improve parallelism in regions that failed evacuation

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -1870,6 +1870,10 @@ void G1ConcurrentMark::clear_range_in_prev_bitmap(MemRegion mr) {
   _prev_mark_bitmap->clear_range(mr);
 }
 
+void G1ConcurrentMark::par_clear_range_in_prev_bitmap(MemRegion mr) {
+  _prev_mark_bitmap->par_clear_range(mr);
+}
+
 HeapRegion*
 G1ConcurrentMark::claim_region(uint worker_id) {
   // "checkpoint" the finger

src/hotspot/share/gc/g1/g1ConcurrentMark.hpp

@@ -564,11 +564,13 @@ class G1ConcurrentMark : public CHeapObj<mtGC> {
   // Mark in the previous bitmap. Caution: the prev bitmap is usually read-only, so use
   // this carefully.
   inline void mark_in_prev_bitmap(oop p);
+  inline void par_mark_in_prev_bitmap(oop p);
 
   // Clears marks for all objects in the given range, for the prev or
   // next bitmaps.  Caution: the previous bitmap is usually
   // read-only, so use this carefully!
   void clear_range_in_prev_bitmap(MemRegion mr);
+  void par_clear_range_in_prev_bitmap(MemRegion mr);
 
   inline bool is_marked_in_prev_bitmap(oop p) const;
 

src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp

@@ -273,6 +273,11 @@ inline void G1ConcurrentMark::mark_in_prev_bitmap(oop p) {
  _prev_mark_bitmap->mark(p);
 }
 
+inline void G1ConcurrentMark::par_mark_in_prev_bitmap(oop p) {
+  assert(!_prev_mark_bitmap->is_marked(p), "sanity");
+  _prev_mark_bitmap->par_mark(p);
+}
+
 bool G1ConcurrentMark::is_marked_in_prev_bitmap(oop p) const {
   assert(p != NULL && oopDesc::is_oop(p), "expected an oop");
   return _prev_mark_bitmap->is_marked(cast_from_oop<HeapWord*>(p));

src/hotspot/share/gc/g1/g1EvacFailureObjectsSet.cpp

@@ -24,12 +24,13 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1EvacFailureObjectsSet.hpp"
-#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1SegmentedArray.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/heapRegion.hpp"
+#include "gc/shared/taskqueue.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
 #include "utilities/quickSort.hpp"
 
-
 const G1SegmentedArrayAllocOptions G1EvacFailureObjectsSet::_alloc_options =
   G1SegmentedArrayAllocOptions((uint)sizeof(OffsetInRegion), SegmentLength, UINT_MAX, Alignment);
 
@@ -55,69 +56,102 @@ G1EvacFailureObjectsSet::OffsetInRegion G1EvacFailureObjectsSet::to_offset(oop o
 }
 
 G1EvacFailureObjectsSet::G1EvacFailureObjectsSet(uint region_idx, HeapWord* bottom) :
-  DEBUG_ONLY(_region_idx(region_idx) COMMA)
+  _region_idx(region_idx),
   _bottom(bottom),
-  _offsets(&_alloc_options, &_free_segment_list)  {
+  _offsets(&_alloc_options, &_free_segment_list),
+  _helper(this, _region_idx),
+  _word_size(0) {
   assert(HeapRegion::LogOfHRGrainBytes < 32, "must be");
 }
 
-// Helper class to join, sort and iterate over the previously collected segmented
-// array of objects that failed evacuation.
-class G1EvacFailureObjectsIterationHelper {
-  typedef G1EvacFailureObjectsSet::OffsetInRegion OffsetInRegion;
+int G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::order_oop(OffsetInRegion a, OffsetInRegion b) {
+  return static_cast<int>(a-b);
+}
 
-  G1EvacFailureObjectsSet* _objects_set;
-  const G1SegmentedArray<OffsetInRegion, mtGC>* _segments;
-  OffsetInRegion* _offset_array;
-  uint _array_length;
+void G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::join_and_sort() {
+  _segments->iterate_segments(*this);
 
-  static int order_oop(OffsetInRegion a, OffsetInRegion b) {
-    return static_cast<int>(a-b);
-  }
-
-  void join_and_sort() {
-    _segments->iterate_segments(*this);
+  QuickSort::sort(_offset_array, _array_length, order_oop, true);
+}
 
-    QuickSort::sort(_offset_array, _array_length, order_oop, true);
+HeapWord* G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::previous_object_end(HeapRegion* region, uint start_idx) {
+  if (start_idx == 0) {
+    return region->bottom();
   }
+  oop obj = _objects_set->from_offset(_offset_array[start_idx - 1]);
+  HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
+  size_t obj_size = obj->size();
+  HeapWord* obj_end = obj_addr + obj_size;
+  return obj_end;
+}
 
-  void iterate(ObjectClosure* closure) {
-    for (uint i = 0; i < _array_length; i++) {
-      oop cur = _objects_set->from_offset(_offset_array[i]);
-      closure->do_object(cur);
-    }
+void G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::insert_queue(G1EvacFailureParScanTasksQueue* queue) {
+  assert(_array_length > 0, "must be");
+  uint i = 1;
+  uint start_idx = (uint)-1;
+  HeapWord* prev_end = nullptr;
+  HeapRegion* region = G1CollectedHeap::heap()->region_at(_region_idx);
+  for (; i*TASK_LIMIT < _array_length; i++) {
+    start_idx = (i-1)*TASK_LIMIT;
+    prev_end = previous_object_end(region, start_idx);
+    queue->push(G1EvacFailureParScanTask(region, prev_end, start_idx, i * TASK_LIMIT));
   }
+  start_idx = (i-1)*TASK_LIMIT;
+  prev_end = previous_object_end(region, start_idx);
+  queue->push(G1EvacFailureParScanTask(region, prev_end, start_idx, _array_length, true));
+}
 
-public:
-  G1EvacFailureObjectsIterationHelper(G1EvacFailureObjectsSet* collector) :
-    _objects_set(collector),
-    _segments(&_objects_set->_offsets),
-    _offset_array(nullptr),
-    _array_length(0) { }
+G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::G1EvacFailureObjectsIterationHelper(G1EvacFailureObjectsSet* collector, uint region_idx) :
+  _objects_set(collector),
+  _segments(&_objects_set->_offsets),
+  _offset_array(nullptr),
+  _array_length(0),
+  _region_idx(region_idx) { }
 
-  void process_and_drop(ObjectClosure* closure) {
-    uint num = _segments->num_allocated_slots();
-    _offset_array = NEW_C_HEAP_ARRAY(OffsetInRegion, num, mtGC);
+void G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::prepare(G1EvacFailureParScanTasksQueue* queue) {
+  uint num = _segments->num_allocated_slots();
+  _offset_array = NEW_C_HEAP_ARRAY(OffsetInRegion, num, mtGC);
 
-    join_and_sort();
-    assert(_array_length == num, "must be %u, %u", _array_length, num);
-    iterate(closure);
+  join_and_sort();
+  assert(_array_length == num, "must be %u, %u", _array_length, num);
 
-    FREE_C_HEAP_ARRAY(OffsetInRegion, _offset_array);
-  }
+  insert_queue(queue);
+}
 
-  // Callback of G1SegmentedArray::iterate_segments
-  void do_segment(G1SegmentedArraySegment<mtGC>* segment, uint length) {
-    segment->copy_to(&_offset_array[_array_length]);
-    _array_length += length;
+void G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::iterate(ObjectClosure* closure, G1EvacFailureParScanTask& task) {
+  assert(_region_idx == task.region()->hrm_index(), "must be");
+  for (uint i = task.start(); i < task.end(); i++) {
+    oop cur = _objects_set->from_offset(_offset_array[i]);
+    closure->do_object(cur);
   }
-};
+}
+
+void G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::reset() {
+  FREE_C_HEAP_ARRAY(OffsetInRegion, _offset_array);
+  _offset_array = nullptr;
+  _array_length = 0;
+}
+
+// Callback of G1SegmentedArray::iterate_segments
+void G1EvacFailureObjectsSet::G1EvacFailureObjectsIterationHelper::do_segment(G1SegmentedArraySegment<mtGC>* segment, uint length) {
+  segment->copy_to(&_offset_array[_array_length]);
+  _array_length += length;
+}
 
-void G1EvacFailureObjectsSet::process_and_drop(ObjectClosure* closure) {
+size_t G1EvacFailureObjectsSet::pre_iteration(G1EvacFailureParScanTasksQueue* queue) {
   assert_at_safepoint();
 
-  G1EvacFailureObjectsIterationHelper helper(this);
-  helper.process_and_drop(closure);
+  _helper.prepare(queue);
+
+  return Atomic::load(&_word_size) * HeapWordSize;
+}
+
+void G1EvacFailureObjectsSet::iterate(ObjectClosure* closure, G1EvacFailureParScanTask& task) {
+  _helper.iterate(closure, task);
+}
 
+void G1EvacFailureObjectsSet::post_iteration() {
+  _helper.reset();
   _offsets.drop_all();
+  Atomic::store(&_word_size, size_t(0));
 }

src/hotspot/share/gc/g1/g1EvacFailureObjectsSet.hpp

@@ -25,42 +25,78 @@
 #ifndef SHARE_GC_G1_G1EVACFAILUREOBJECTSSET_HPP
 #define SHARE_GC_G1_G1EVACFAILUREOBJECTSSET_HPP
 
+#include "gc/g1/g1EvacFailureParScanTask.hpp"
 #include "gc/g1/g1SegmentedArray.hpp"
 #include "memory/iterator.hpp"
 #include "oops/oop.hpp"
+#include "runtime/atomic.hpp"
 
-class G1EvacFailureObjectsIterationHelper;
+class HeapRegion;
 
 // This class collects addresses of objects that failed evacuation in a specific
 // heap region.
 // Provides sorted iteration of these objects for processing during the remove
 // self forwards phase.
 class G1EvacFailureObjectsSet {
-  friend class G1EvacFailureObjectsIterationHelper;
-
-public:
   // Storage type of an object that failed evacuation within a region. Given
   // heap region size and possible object locations within a region, it is
   // sufficient to use an uint here to save some space instead of full pointers.
   typedef uint OffsetInRegion;
 
-private:
+  // Helper class to join, sort and iterate over the previously collected segmented
+  // array of objects that failed evacuation.
+  class G1EvacFailureObjectsIterationHelper {
+    typedef G1EvacFailureObjectsSet::OffsetInRegion OffsetInRegion;
+
+    static const uint TASK_LIMIT = 1000;
+
+    G1EvacFailureObjectsSet* _objects_set;
+    const G1SegmentedArray<OffsetInRegion, mtGC>* _segments;
+    OffsetInRegion* _offset_array;
+    uint _array_length;
+    uint _region_idx;
+
+    static int order_oop(OffsetInRegion a, OffsetInRegion b);
+
+    void join_and_sort();
+
+    HeapWord* previous_object_end(HeapRegion* region, uint start_idx);
+
+    void insert_queue(G1EvacFailureParScanTasksQueue* queue);
+
+  public:
+    G1EvacFailureObjectsIterationHelper(G1EvacFailureObjectsSet* collector, uint region_idx);
+
+    void prepare(G1EvacFailureParScanTasksQueue* queue);
+    void iterate(ObjectClosure* closure, G1EvacFailureParScanTask& task);
+    void reset();
+
+    // Callback of G1SegmentedArray::iterate_segments
+    void do_segment(G1SegmentedArraySegment<mtGC>* segment, uint length);
+  };
+
   static const uint SegmentLength = 256;
+
   static const uint Alignment = 4;
 
   static const G1SegmentedArrayAllocOptions _alloc_options;
 
   // This free list is shared among evacuation failure process in all regions.
   static G1SegmentedArrayFreeList<mtGC> _free_segment_list;
 
-  DEBUG_ONLY(const uint _region_idx;)
+  const uint _region_idx;
 
   // Region bottom
   const HeapWord* _bottom;
 
   // Offsets within region containing objects that failed evacuation.
   G1SegmentedArray<OffsetInRegion, mtGC> _offsets;
 
+  G1EvacFailureObjectsIterationHelper _helper;
+
+  // Live words in the evacuation failure region.
+  volatile size_t _word_size;
+
   void assert_is_valid_offset(size_t offset) const NOT_DEBUG_RETURN;
   // Converts between an offset within a region and an oop address.
   oop from_offset(OffsetInRegion offset) const;
@@ -70,12 +106,18 @@ class G1EvacFailureObjectsSet {
   G1EvacFailureObjectsSet(uint region_idx, HeapWord* bottom);
 
   // Record an object that failed evacuation.
-  inline void record(oop obj);
-
-  // Apply the given ObjectClosure to all objects that failed evacuation and
-  // empties the list after processing.
-  // Objects are passed in increasing address order.
-  void process_and_drop(ObjectClosure* closure);
+  inline void record(oop obj, size_t word_size);
+
+  // Prepare parallel iteration by building and sorting list of evacuation
+  // failure objects, and constructing parallelizable tasks.
+  // Return live bytes in the evacuation failure region.
+  size_t pre_iteration(G1EvacFailureParScanTasksQueue* queue);
+  // Apply the given ObjectClosure to all previously recorded objects in the task
+  // that failed evacuation in ascending address order.
+  void iterate(ObjectClosure* closure, G1EvacFailureParScanTask& task);
+  // Empty the list of evacuation failure objects.
+  // Reset live words in the evacuation failure region.
+  void post_iteration();
 };
 
 

src/hotspot/share/gc/g1/g1EvacFailureObjectsSet.inline.hpp

@@ -26,15 +26,16 @@
 #define SHARE_GC_G1_G1EVACFAILUREOBJECTSSET_INLINE_HPP
 
 #include "gc/g1/g1EvacFailureObjectsSet.hpp"
-#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1SegmentedArray.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/heapRegion.hpp"
 
-void G1EvacFailureObjectsSet::record(oop obj) {
+inline void G1EvacFailureObjectsSet::record(oop obj, size_t word_size) {
   assert(obj != NULL, "must be");
   assert(_region_idx == G1CollectedHeap::heap()->heap_region_containing(obj)->hrm_index(), "must be");
   OffsetInRegion* e = _offsets.allocate();
   *e = to_offset(obj);
+  Atomic::add(&_word_size, word_size);
 }
 
 #endif //SHARE_GC_G1_G1EVACFAILUREOBJECTSSET_INLINE_HPP