8321619: Generational ZGC: ZColorStoreGoodOopClosure is only valid for young objects

Reviewed-by: stefank, sjohanss

Author: fisk

src/hotspot/share/gc/z/zBarrierSet.cpp

@@ -152,6 +152,19 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
   deoptimize_allocation(thread);
 }
 
+void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj, size_t size) {
+  volatile zpointer* src = (volatile zpointer*)src_obj->base();
+  volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
+
+  for (const zpointer* const end = cast_from_oop<const zpointer*>(src_obj) + size; src < end; src++, dst++) {
+    zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
+    // We avoid healing here because the store below colors the pointer store good,
+    // hence avoiding the cost of a CAS.
+    ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
+    Atomic::store(dst, ZAddress::store_good(elem));
+  }
+}
+
 void ZBarrierSet::print_on(outputStream* st) const {
   st->print_cr("ZBarrierSet");
 }

src/hotspot/share/gc/z/zBarrierSet.hpp

@@ -39,6 +39,8 @@ class ZBarrierSet : public BarrierSet {
   static ZBarrierSetAssembler* assembler();
   static bool barrier_needed(DecoratorSet decorators, BasicType type);
 
+  static void clone_obj_array(objArrayOop src, objArrayOop dst, size_t size);
+
   virtual void on_thread_create(Thread* thread);
   virtual void on_thread_destroy(Thread* thread);
   virtual void on_thread_attach(Thread* thread);

src/hotspot/share/gc/z/zBarrierSet.inline.hpp

@@ -403,14 +403,13 @@ inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_i
   return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
 }
 
-class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
+class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
 public:
   virtual void do_oop(oop* p_) {
     volatile zpointer* const p = (volatile zpointer*)p_;
     const zpointer ptr = ZBarrier::load_atomic(p);
     const zaddress addr = ZPointer::uncolor(ptr);
-    ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
-    *p = ZAddress::store_good(addr);
+    Atomic::store(p, ZAddress::store_good(addr));
   }
 
   virtual void do_oop(narrowOop* p) {
@@ -433,17 +432,28 @@ template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
   assert_is_valid(to_zaddress(src));
 
+  if (dst->is_objArray()) {
+    // Cloning an object array is similar to performing array copy.
+    // If an array is large enough to have its allocation segmented,
+    // this operation might require GC barriers. However, the intrinsics
+    // for cloning arrays transform the clone to an optimized allocation
+    // and arraycopy sequence, so the performance of this runtime call
+    // does not matter for object arrays.
+    clone_obj_array(objArrayOop(src), objArrayOop(dst), size);
+    return;
+  }
+
   // Fix the oops
   ZLoadBarrierOopClosure cl;
   ZIterator::oop_iterate(src, &cl);
 
   // Clone the object
   Raw::clone_in_heap(src, dst, size);
 
-  assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
+  assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
 
   // Color store good before handing out
-  ZStoreBarrierOopClosure cl_sg;
+  ZColorStoreGoodOopClosure cl_sg;
   ZIterator::oop_iterate(dst, &cl_sg);
 }