8227226: ZGC: Segmented array clearing

Co-authored-by: Stefan Karlsson <stefan.karlsson@oracle.com>
Co-authored-by: Erik Osterlund <erik.osterlund@oracle.com>
Co-authored-by: Ryan Sciampacone <sci@amazon.com>
Reviewed-by: eosterlund

Author: 4 people

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -226,18 +226,18 @@ void MemAllocator::Allocation::notify_allocation_jfr_sampler() {
   size_t size_in_bytes = _allocator._word_size * HeapWordSize;
 
   if (_allocated_outside_tlab) {
-    AllocTracer::send_allocation_outside_tlab(_allocator._klass, mem, size_in_bytes, _thread);
+    AllocTracer::send_allocation_outside_tlab(obj()->klass(), mem, size_in_bytes, _thread);
   } else if (_allocated_tlab_size != 0) {
     // TLAB was refilled
-    AllocTracer::send_allocation_in_new_tlab(_allocator._klass, mem, _allocated_tlab_size * HeapWordSize,
+    AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize,
                                              size_in_bytes, _thread);
   }
 }
 
 void MemAllocator::Allocation::notify_allocation_dtrace_sampler() {
   if (DTraceAllocProbes) {
     // support for Dtrace object alloc event (no-op most of the time)
-    Klass* klass = _allocator._klass;
+    Klass* klass = obj()->klass();
     size_t word_size = _allocator._word_size;
     if (klass != NULL && klass->name() != NULL) {
       SharedRuntime::dtrace_object_alloc(obj(), (int)word_size);

src/hotspot/share/gc/shared/memAllocator.hpp

@@ -59,7 +59,7 @@ class MemAllocator: StackObj {
   // This finish constructing an oop by installing the mark word and the Klass* pointer
   // last. At the point when the Klass pointer is initialized, this is a constructed object
   // that must be parseable as an oop by concurrent collectors.
-  oop finish(HeapWord* mem) const;
+  virtual oop finish(HeapWord* mem) const;
 
   // Raw memory allocation. This may or may not use TLAB allocations to satisfy the
   // allocation. A GC implementation may override this function to satisfy the allocation

src/hotspot/share/gc/z/zCollectedHeap.cpp

@@ -28,6 +28,7 @@
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zNMethod.hpp"
+#include "gc/z/zObjArrayAllocator.hpp"
 #include "gc/z/zServiceability.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zUtils.inline.hpp"
@@ -127,6 +128,15 @@ HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_si
   return (HeapWord*)addr;
 }
 
+oop ZCollectedHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) {
+  if (!do_zero) {
+    return CollectedHeap::array_allocate(klass, size, length, false /* do_zero */, THREAD);
+  }
+
+  ZObjArrayAllocator allocator(klass, size, length, THREAD);
+  return allocator.allocate();
+}
+
 HeapWord* ZCollectedHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) {
   const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(size));
   return (HeapWord*)_heap.alloc_object(size_in_bytes);

src/hotspot/share/gc/z/zCollectedHeap.hpp

@@ -75,6 +75,7 @@ class ZCollectedHeap : public CollectedHeap {
 
   virtual uint32_t hash_oop(oop obj) const;
 
+  virtual oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS);
   virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded);
   virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
                                                        size_t size,

src/hotspot/share/gc/z/zObjArrayAllocator.cpp

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zObjArrayAllocator.hpp"
+#include "gc/z/zUtils.inline.hpp"
+#include "memory/universe.hpp"
+#include "oops/arrayKlass.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/handles.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// To avoid delaying safepoints, clearing of arrays is split up in segments
+// with safepoint polling inbetween. However, we can't have a not-yet-cleared
+// array of oops on the heap when we safepoint since the GC will then stumble
+// across uninitialized oops. To avoid this we let an array of oops be an
+// array of a primitive type of the same size until the clearing has completed.
+// A max segment size of 64K was chosen because benchmarking suggests that is
+// offers a good trade-off between allocation time and time-to-safepoint.
+
+static Klass* substitute_object_array_klass(Klass* klass) {
+  if (!klass->is_objArray_klass()) {
+    return klass;
+  }
+
+  Klass* const substitute_klass = Universe::longArrayKlassObj();
+  const BasicType type = ArrayKlass::cast(klass)->element_type();
+  const BasicType substitute_type = ArrayKlass::cast(substitute_klass)->element_type();
+  assert(type2aelembytes(type) == type2aelembytes(substitute_type), "Element size mismatch");
+  return substitute_klass;
+}
+
+ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread) :
+    ObjArrayAllocator(substitute_object_array_klass(klass), word_size, length, false /* do_zero */, thread),
+    _final_klass(klass) {}
+
+oop ZObjArrayAllocator::finish(HeapWord* mem) const {
+  HandleMark hm;
+  Handle array(_thread, ObjArrayAllocator::finish(mem));
+
+  const size_t segment_max = ZUtils::bytes_to_words(64 * K);
+  const size_t skip = arrayOopDesc::header_size(ArrayKlass::cast(_klass)->element_type());
+  size_t remaining = _word_size - skip;
+
+  while (remaining > 0) {
+    // Clear segment
+    const size_t segment = MIN2(remaining, segment_max);
+    Copy::zero_to_words((HeapWord*)array() + (_word_size - remaining), segment);
+    remaining -= segment;
+
+    if (remaining > 0) {
+      // Safepoint
+      ThreadBlockInVM tbivm((JavaThread*)_thread);
+    }
+  }
+
+  if (_klass != _final_klass) {
+    // Set final klass
+    oopDesc::release_set_klass((HeapWord*)array(), _final_klass);
+  }
+
+  return array();
+}

src/hotspot/share/gc/z/zObjArrayAllocator.hpp

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP
+#define SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP
+
+#include "gc/shared/memAllocator.hpp"
+
+class ZObjArrayAllocator : public ObjArrayAllocator {
+private:
+  Klass* const _final_klass;
+
+public:
+  ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread);
+
+  virtual oop finish(HeapWord* mem) const;
+};
+
+#endif // SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP