8229451: ZGC: Make some roots invisible to the heap iterator

Reviewed-by: eosterlund

Author: pliden

src/hotspot/share/gc/shared/gcThreadLocalData.hpp

@@ -40,6 +40,6 @@
 // should consider placing frequently accessed fields first in
 // T, so that field offsets relative to Thread are small, which
 // often allows for a more compact instruction encoding.
-typedef uint64_t GCThreadLocalData[18]; // 144 bytes
+typedef uint64_t GCThreadLocalData[19]; // 152 bytes
 
 #endif // SHARE_GC_SHARED_GCTHREADLOCALDATA_HPP

src/hotspot/share/gc/z/zHeapIterator.cpp

@@ -193,7 +193,7 @@ void ZHeapIterator::objects_do(ObjectClosure* cl) {
   ZStatTimerDisable disable;
 
   // Push roots to visit
-  push_roots<ZRootsIterator,                     false /* Concurrent */, false /* Weak */>();
+  push_roots<ZRootsIteratorNoInvisible,          false /* Concurrent */, false /* Weak */>();
   push_roots<ZConcurrentRootsIteratorClaimOther, true  /* Concurrent */, false /* Weak */>();
   if (VisitWeaks) {
     push_roots<ZWeakRootsIterator,           false /* Concurrent */, true  /* Weak */>();

src/hotspot/share/gc/z/zMark.cpp

@@ -156,7 +156,7 @@ class ZMarkRootsTask : public ZTask {
   ZMarkRootsTask(ZMark* mark) :
       ZTask("ZMarkRootsTask"),
       _mark(mark),
-      _roots() {}
+      _roots(true /* visit_invisible */, false /* visit_jvmti_weak_export */) {}
 
   virtual void work() {
     _roots.oops_do(&_cl);

src/hotspot/share/gc/z/zObjArrayAllocator.cpp

@@ -22,6 +22,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zThreadLocalData.hpp"
 #include "gc/z/zObjArrayAllocator.hpp"
 #include "gc/z/zUtils.inline.hpp"
 #include "memory/universe.hpp"
@@ -57,8 +58,12 @@ ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int lengt
     _final_klass(klass) {}
 
 oop ZObjArrayAllocator::finish(HeapWord* mem) const {
-  HandleMark hm;
-  Handle array(_thread, ObjArrayAllocator::finish(mem));
+  // Set mark word and initial klass pointer
+  ObjArrayAllocator::finish(mem);
+
+  // Keep the array alive across safepoints, but make it invisible
+  // to the heap itarator until the final klass pointer has been set
+  ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem);
 
   const size_t segment_max = ZUtils::bytes_to_words(64 * K);
   const size_t skip = arrayOopDesc::header_size(ArrayKlass::cast(_klass)->element_type());
@@ -67,7 +72,7 @@ oop ZObjArrayAllocator::finish(HeapWord* mem) const {
   while (remaining > 0) {
     // Clear segment
     const size_t segment = MIN2(remaining, segment_max);
-    Copy::zero_to_words((HeapWord*)array() + (_word_size - remaining), segment);
+    Copy::zero_to_words(mem + (_word_size - remaining), segment);
     remaining -= segment;
 
     if (remaining > 0) {
@@ -77,9 +82,12 @@ oop ZObjArrayAllocator::finish(HeapWord* mem) const {
   }
 
   if (_klass != _final_klass) {
-    // Set final klass
-    oopDesc::release_set_klass((HeapWord*)array(), _final_klass);
+    // Set final klass pointer
+    oopDesc::release_set_klass(mem, _final_klass);
   }
 
-  return array();
+  // Make the array visible to the heap iterator
+  ZThreadLocalData::clear_invisible_root(_thread);
+
+  return oop(mem);
 }

src/hotspot/share/gc/z/zRelocate.cpp

@@ -69,12 +69,12 @@ class ZRelocateRootsTask : public ZTask {
 public:
   ZRelocateRootsTask() :
       ZTask("ZRelocateRootsTask"),
-      _roots() {}
+      _roots(true /* visit_invisible */, true /* visit_jvmti_weak_export */) {}
 
   virtual void work() {
     // During relocation we need to visit the JVMTI
     // export weak roots to rehash the JVMTI tag map
-    _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */);
+    _roots.oops_do(&_cl);
   }
 };
 

src/hotspot/share/gc/z/zRootsIterator.cpp

@@ -159,19 +159,26 @@ class ZRootsIteratorCodeBlobClosure : public CodeBlobToOopClosure {
 class ZRootsIteratorThreadClosure : public ThreadClosure {
 private:
   ZRootsIteratorClosure* _cl;
+  const bool             _visit_invisible;
 
 public:
-  ZRootsIteratorThreadClosure(ZRootsIteratorClosure* cl) :
-      _cl(cl) {}
+  ZRootsIteratorThreadClosure(ZRootsIteratorClosure* cl, bool visit_invisible) :
+      _cl(cl),
+      _visit_invisible(visit_invisible) {}
 
   virtual void do_thread(Thread* thread) {
     ZRootsIteratorCodeBlobClosure code_cl(_cl);
     thread->oops_do(_cl, ClassUnloading ? &code_cl : NULL);
     _cl->do_thread(thread);
+    if (_visit_invisible && ZThreadLocalData::has_invisible_root(thread)) {
+      _cl->do_oop(ZThreadLocalData::invisible_root(thread));
+    }
   }
 };
 
-ZRootsIterator::ZRootsIterator() :
+ZRootsIterator::ZRootsIterator(bool visit_invisible, bool visit_jvmti_weak_export) :
+    _visit_invisible(visit_invisible),
+    _visit_jvmti_weak_export(visit_jvmti_weak_export),
     _universe(this),
     _object_synchronizer(this),
     _management(this),
@@ -239,7 +246,7 @@ void ZRootsIterator::do_system_dictionary(ZRootsIteratorClosure* cl) {
 void ZRootsIterator::do_threads(ZRootsIteratorClosure* cl) {
   ZStatTimer timer(ZSubPhasePauseRootsThreads);
   ResourceMark rm;
-  ZRootsIteratorThreadClosure thread_cl(cl);
+  ZRootsIteratorThreadClosure thread_cl(cl, _visit_invisible);
   Threads::possibly_parallel_threads_do(true, &thread_cl);
 }
 
@@ -248,7 +255,7 @@ void ZRootsIterator::do_code_cache(ZRootsIteratorClosure* cl) {
   ZNMethod::oops_do(cl);
 }
 
-void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_export) {
+void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl) {
   ZStatTimer timer(ZSubPhasePauseRoots);
   _universe.oops_do(cl);
   _object_synchronizer.oops_do(cl);
@@ -259,7 +266,7 @@ void ZRootsIterator::oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_ex
   if (!ClassUnloading) {
     _code_cache.oops_do(cl);
   }
-  if (visit_jvmti_weak_export) {
+  if (_visit_jvmti_weak_export) {
     _jvmti_weak_export.oops_do(cl);
   }
 }

src/hotspot/share/gc/z/zRootsIterator.hpp

@@ -84,6 +84,9 @@ class ZParallelWeakOopsDo {
 
 class ZRootsIterator {
 private:
+  bool _visit_invisible;
+  bool _visit_jvmti_weak_export;
+
   void do_universe(ZRootsIteratorClosure* cl);
   void do_object_synchronizer(ZRootsIteratorClosure* cl);
   void do_management(ZRootsIteratorClosure* cl);
@@ -103,10 +106,16 @@ class ZRootsIterator {
   ZParallelOopsDo<ZRootsIterator, &ZRootsIterator::do_code_cache>        _code_cache;
 
 public:
-  ZRootsIterator();
+  ZRootsIterator(bool visit_invisible = true, bool visit_jvmti_weak_export = false);
   ~ZRootsIterator();
 
-  void oops_do(ZRootsIteratorClosure* cl, bool visit_jvmti_weak_export = false);
+  void oops_do(ZRootsIteratorClosure* cl);
+};
+
+class ZRootsIteratorNoInvisible : public ZRootsIterator {
+public:
+  ZRootsIteratorNoInvisible() :
+      ZRootsIterator(false /* visit_invisible */, false /* visit_jvmti_weak_export */) {}
 };
 
 class ZConcurrentRootsIterator {

src/hotspot/share/gc/z/zThreadLocalData.hpp

@@ -34,10 +34,12 @@ class ZThreadLocalData {
 private:
   uintptr_t              _address_bad_mask;
   ZMarkThreadLocalStacks _stacks;
+  oop*                   _invisible_root;
 
   ZThreadLocalData() :
       _address_bad_mask(0),
-      _stacks() {}
+      _stacks(),
+      _invisible_root(NULL) {}
 
   static ZThreadLocalData* data(Thread* thread) {
     return thread->gc_data<ZThreadLocalData>();
@@ -60,6 +62,25 @@ class ZThreadLocalData {
     return &data(thread)->_stacks;
   }
 
+  static void set_invisible_root(Thread* thread, oop* root) {
+    assert(!has_invisible_root(thread), "Already set");
+    data(thread)->_invisible_root = root;
+  }
+
+  static void clear_invisible_root(Thread* thread) {
+    assert(has_invisible_root(thread), "Should be set");
+    data(thread)->_invisible_root = NULL;
+  }
+
+  static bool has_invisible_root(Thread* thread) {
+    return data(thread)->_invisible_root != NULL;
+  }
+
+  static oop* invisible_root(Thread* thread) {
+    assert(has_invisible_root(thread), "Should be set");
+    return data(thread)->_invisible_root;
+  }
+
   static ByteSize address_bad_mask_offset() {
     return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _address_bad_mask);
   }