8252105: parallel heap inspection for ZCollectedHeap

src/hotspot/share/gc/z/zCollectedHeap.cpp

@@ -253,6 +253,28 @@ void ZCollectedHeap::object_iterate(ObjectClosure* cl) {
   _heap.object_iterate(cl, true /* visit_weaks */);
 }
 
+class ZHeapParallelObjectIterator : public ParallelObjectIterator {
+private:
+  uint    _thread_num;
+  ZHeap*  _heap;
+  ZHeapIterator _iter;
+public:
+  ZHeapParallelObjectIterator(uint thread_num, ZHeap* heap) :
+      _thread_num(thread_num), _heap(heap), _iter(_thread_num) {
+    // prepare work queue, enqueue all roots serially.
+    _heap->process_roots_for_par_iterate(&_iter, true /* visit_weaks */);
+  }
+
+  virtual void object_iterate(ObjectClosure* cl, uint worker_id) {
+    // process references from roots.
+    _heap->par_references_iterate(cl, &_iter, worker_id, true /* visit_weaks*/);
+  }
+};
+
+ParallelObjectIterator* ZCollectedHeap::parallel_object_iterator(uint thread_num) {
+  return new ZHeapParallelObjectIterator(thread_num, &_heap);
+}
+
 void ZCollectedHeap::keep_alive(oop obj) {
   _heap.keep_alive(obj);
 }

src/hotspot/share/gc/z/zCollectedHeap.hpp

@@ -33,6 +33,7 @@
 
 class ZDirector;
 class ZDriver;
+class ZHeapParallelObjectIterator;
 class ZStat;
 
 class ZCollectedHeap : public CollectedHeap {
@@ -97,6 +98,7 @@ class ZCollectedHeap : public CollectedHeap {
   virtual GrowableArray<MemoryPool*> memory_pools();
 
   virtual void object_iterate(ObjectClosure* cl);
+  virtual ParallelObjectIterator* parallel_object_iterator(uint thread_num);
 
   virtual void keep_alive(oop obj);
 

src/hotspot/share/gc/z/zHeap.cpp

@@ -26,7 +26,6 @@
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
-#include "gc/z/zHeapIterator.hpp"
 #include "gc/z/zHeuristics.hpp"
 #include "gc/z/zMark.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
@@ -441,6 +440,17 @@ void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) {
   iter.objects_do(cl, visit_weaks);
 }
 
+void ZHeap::process_roots_for_par_iterate(ZHeapIterator* iter, bool visit_weaks) {
+  iter->enqueue_roots(visit_weaks);
+}
+
+void ZHeap::par_references_iterate(ObjectClosure* cl,
+                                   ZHeapIterator* iter,
+                                   uint worker_id,
+                                   bool visit_weaks) {
+  iter->drain_queue(cl, visit_weaks, worker_id);
+}
+
 void ZHeap::pages_do(ZPageClosure* cl) {
   ZPageTableIterator iter(&_page_table);
   for (ZPage* page; iter.next(&page);) {

src/hotspot/share/gc/z/zHeap.hpp

@@ -26,6 +26,7 @@
 
 #include "gc/z/zAllocationFlags.hpp"
 #include "gc/z/zForwardingTable.hpp"
+#include "gc/z/zHeapIterator.hpp"
 #include "gc/z/zMark.hpp"
 #include "gc/z/zObjectAllocator.hpp"
 #include "gc/z/zPage.hpp"
@@ -141,6 +142,9 @@ class ZHeap {
 
   // Iteration
   void object_iterate(ObjectClosure* cl, bool visit_weaks);
+  void process_roots_for_par_iterate(ZHeapIterator* iter, bool visit_weaks);
+  void par_references_iterate(ObjectClosure* cl, ZHeapIterator* iter,
+                              uint worker_id, bool visit_weaks);
   void pages_do(ZPageClosure* cl);
 
   // Serviceability

src/hotspot/share/gc/z/zHeapIterator.cpp

@@ -52,6 +52,10 @@ class ZHeapIteratorBitMap : public CHeapObj<mtGC> {
     _map.set_bit(index);
     return true;
   }
+
+  inline bool par_try_set_bit(size_t index) {
+    return _map.par_set_bit(index);
+  }
 };
 
 template <bool Concurrent, bool Weak>
@@ -85,11 +89,12 @@ class ZHeapIteratorRootOopClosure : public ZRootsIteratorClosure {
   }
 };
 
-template <bool VisitReferents>
+template <bool VisitReferents, bool ParallelIter>
 class ZHeapIteratorOopClosure : public ClaimMetadataVisitingOopIterateClosure {
 private:
   ZHeapIterator* const _iter;
   const oop            _base;
+  ZHeapIterTaskQueue*     _queue;
 
   oop load_oop(oop* p) {
     if (VisitReferents) {
@@ -100,18 +105,23 @@ class ZHeapIteratorOopClosure : public ClaimMetadataVisitingOopIterateClosure {
   }
 
 public:
-  ZHeapIteratorOopClosure(ZHeapIterator* iter, oop base) :
+  ZHeapIteratorOopClosure(ZHeapIterator* iter, oop base, ZHeapIterTaskQueue* q = NULL) :
       ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other),
       _iter(iter),
-      _base(base) {}
+      _base(base),
+      _queue(q) {}
 
   virtual ReferenceIterationMode reference_iteration_mode() {
     return VisitReferents ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
   }
 
   virtual void do_oop(oop* p) {
     const oop obj = load_oop(p);
-    _iter->push(obj);
+    if (ParallelIter == false) {
+      _iter->push(obj);
+    } else {
+      _iter->par_enqueue(obj, _queue);
+    }
   }
 
   virtual void do_oop(narrowOop* p) {
@@ -125,15 +135,40 @@ class ZHeapIteratorOopClosure : public ClaimMetadataVisitingOopIterateClosure {
 #endif
 };
 
-ZHeapIterator::ZHeapIterator() :
+ZHeapIterator::ZHeapIterator(uint num_workers) :
     _visit_stack(),
-    _visit_map(ZAddressOffsetMax) {}
+    _visit_map(ZAddressOffsetMax),
+    _num_workers(num_workers),
+    _map_lock(),
+    _task_queues(NULL) {
+  if (_num_workers > 1) {
+    // prepare process queue.
+    _task_queues = new ZHeapIterTaskQueueSet((int) _num_workers);
+    for (uint i = 0; i < _num_workers; i++) {
+      ZHeapIterTaskQueue* q = new ZHeapIterTaskQueue();
+      q->initialize();
+      _task_queues->register_queue(i, q);
+    }
+  }
+}
 
 ZHeapIterator::~ZHeapIterator() {
   ZVisitMapIterator iter(&_visit_map);
   for (ZHeapIteratorBitMap* map; iter.next(&map);) {
     delete map;
   }
+  // reclaim task queues
+  if (_task_queues != NULL) {
+    for (uint i = 0; i < _num_workers; i++) {
+      ZHeapIterTaskQueue* q = _task_queues->queue(i);
+      if (q != NULL) {
+        delete q;
+        q = NULL;
+      }
+    }
+    delete _task_queues;
+    _task_queues = NULL;
+  }
   ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_other);
 }
 
@@ -152,13 +187,22 @@ ZHeapIteratorBitMap* ZHeapIterator::object_map(oop obj) {
   const uintptr_t offset = ZAddress::offset(ZOop::to_address(obj));
   ZHeapIteratorBitMap* map = _visit_map.get(offset);
   if (map == NULL) {
-    map = new ZHeapIteratorBitMap(object_index_max());
-    _visit_map.put(offset, map);
+    if (_num_workers > 1) {
+      // Parallel iterate, holding lock to update _visit_map
+      ZLocker<ZLock> locker(&_map_lock);
+      if (map == NULL) {
+        map = new ZHeapIteratorBitMap(object_index_max());
+        _visit_map.put(offset, map);
+      }
+    } else {
+      map = new ZHeapIteratorBitMap(object_index_max());
+      _visit_map.put(offset, map);
+    }
   }
-
   return map;
 }
 
+// push objects in to _visit_stack, used by RootOopClosure.
 void ZHeapIterator::push(oop obj) {
   if (obj == NULL) {
     // Ignore
@@ -183,9 +227,9 @@ void ZHeapIterator::push_roots() {
   roots.oops_do(&cl);
 }
 
-template <bool VisitReferents>
-void ZHeapIterator::push_fields(oop obj) {
-  ZHeapIteratorOopClosure<VisitReferents> cl(this, obj);
+template <bool VisitReferents, bool ParallelIter>
+void ZHeapIterator::push_fields(oop obj, ZHeapIterTaskQueue* queue) {
+  ZHeapIteratorOopClosure<VisitReferents, ParallelIter> cl(this, obj, queue);
   obj->oop_iterate(&cl);
 }
 
@@ -213,10 +257,65 @@ void ZHeapIterator::objects_do(ObjectClosure* cl) {
   }
 }
 
+// Used only in serial iteration
 void ZHeapIterator::objects_do(ObjectClosure* cl, bool visit_weaks) {
   if (visit_weaks) {
     objects_do<true /* VisitWeaks */>(cl);
   } else {
     objects_do<false /* VisitWeaks */>(cl);
   }
 }
+
+// Parallel iteration support
+void ZHeapIterator::enqueue_roots(bool visit_weaks) {
+  ZStatTimerDisable disable;
+  // Push roots to visit
+  push_roots<ZRootsIterator,                     false /* Concurrent */, false /* Weak */>();
+  push_roots<ZConcurrentRootsIteratorClaimOther, true  /* Concurrent */, false /* Weak */>();
+  if (visit_weaks) {
+    push_roots<ZWeakRootsIterator,           false /* Concurrent */, true  /* Weak */>();
+    push_roots<ZConcurrentWeakRootsIterator, true  /* Concurrent */, true  /* Weak */>();
+  }
+  // Divide roots into thread queue.
+  size_t roots_num = _visit_stack.size();
+  for (uint i = 0; i < roots_num; i++) {
+    uint worker_id = i % _num_workers;
+    oop obj = _visit_stack.pop();
+    _task_queues->queue(worker_id)->push(obj);
+  }
+}
+
+void ZHeapIterator::drain_queue(ObjectClosure* cl, bool visit_weaks, uint worker_id) {
+  ZStatTimerDisable disable;
+  ZHeapIterTaskQueue* q = _task_queues->queue(worker_id);
+  assert(q != NULL, "Heap iteration task queue must not NULL");
+  // Drain stack
+  oop obj;
+  while (q->pop_overflow(obj) || q->pop_local(obj) || _task_queues->steal(worker_id, obj)) {
+    // Visit object
+    cl->do_object(obj);
+    // Push fields to visit
+    if (visit_weaks = true) {
+      push_fields<true, true /* ParallelIter */>(obj, q);
+    } else {
+      push_fields<false, true /* ParallelIter */>(obj, q);
+    }
+  }
+}
+
+void ZHeapIterator::par_enqueue(oop obj, ZHeapIterTaskQueue* q) {
+   if (obj == NULL) {
+    // Ignore
+    return;
+  }
+
+  ZHeapIteratorBitMap* const map = object_map(obj);
+  const size_t index = object_index(obj);
+  if (!map->par_try_set_bit(index)) {
+    // Already pushed
+    return;
+  }
+
+  // Push
+  q->push(obj);
+}

src/hotspot/share/gc/z/zHeapIterator.hpp

@@ -24,37 +24,53 @@
 #ifndef SHARE_GC_Z_ZHEAPITERATOR_HPP
 #define SHARE_GC_Z_ZHEAPITERATOR_HPP
 
+#include "gc/shared/taskqueue.inline.hpp"
 #include "gc/z/zGranuleMap.hpp"
+#include "gc/z/zLock.inline.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/stack.hpp"
 
 class ObjectClosure;
 class ZHeapIteratorBitMap;
+// Queue and QueueSet for parallel iteration
+typedef OverflowTaskQueue<oop, mtGC>                     ZHeapIterTaskQueue;
+typedef GenericTaskQueueSet<ZHeapIterTaskQueue, mtGC>    ZHeapIterTaskQueueSet;
 
 class ZHeapIterator : public StackObj {
   template<bool Concurrent, bool Weak> friend class ZHeapIteratorRootOopClosure;
-  template<bool VisitReferents> friend class ZHeapIteratorOopClosure;
+  template<bool VisitReferents, bool ParallelIter> friend class ZHeapIteratorOopClosure;
 
 private:
   typedef ZGranuleMap<ZHeapIteratorBitMap*>         ZVisitMap;
   typedef ZGranuleMapIterator<ZHeapIteratorBitMap*> ZVisitMapIterator;
   typedef Stack<oop, mtGC>                          ZVisitStack;
-
+  // For parallel iteration, _visit_stack only contains roots.
+  // For serial iteration, _visit_stack contains all references reached.
   ZVisitStack _visit_stack;
   ZVisitMap   _visit_map;
+  // For parallel iteration
+  uint            _num_workers;
+  ZLock           _map_lock;
+  ZHeapIterTaskQueueSet* _task_queues;
 
   ZHeapIteratorBitMap* object_map(oop obj);
   void push(oop obj);
 
   template <typename RootsIterator, bool Concurrent, bool Weak> void push_roots();
-  template <bool VisitReferents> void push_fields(oop obj);
+  // push_fields is different for serial iterate and parallel iterate.
+  template <bool VisitReferents, bool ParallelIter = false>
+  void push_fields(oop obj, ZHeapIterTaskQueue* queue = NULL);
   template <bool VisitReferents> void objects_do(ObjectClosure* cl);
 
 public:
-  ZHeapIterator();
+  ZHeapIterator(uint num_workers = 0);
   ~ZHeapIterator();
 
   void objects_do(ObjectClosure* cl, bool visit_weaks);
+  // For parallel iteration
+  void par_enqueue(oop obj, ZHeapIterTaskQueue* q);
+  void enqueue_roots(bool visit_weaks);
+  void drain_queue(ObjectClosure* cl, bool visit_weaks, uint worker_id);
 };
 
 #endif // SHARE_GC_Z_ZHEAPITERATOR_HPP