8255234: ZGC: Bulk allocate forwarding data structures

Co-authored-by: Albert Mingkun Yang <ayang@openjdk.org>
Co-authored-by: Per Liden <pliden@openjdk.org>
Reviewed-by: ayang, stefank

Author: pliden

src/hotspot/share/gc/z/zAttachedArray.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,8 +32,12 @@ class ZAttachedArray {
   const size_t _length;
 
   static size_t object_size();
+  static size_t array_size(size_t length);
 
 public:
+  template <typename Allocator>
+  static void* alloc(Allocator* allocator, size_t length);
+
   static void* alloc(size_t length);
   static void free(ObjectT* obj);
 

src/hotspot/share/gc/z/zAttachedArray.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,13 +34,35 @@ inline size_t ZAttachedArray<ObjectT, ArrayT>::object_size() {
 }
 
 template <typename ObjectT, typename ArrayT>
-inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
-  const size_t array_size = sizeof(ArrayT) * length;
-  char* const addr = AllocateHeap(object_size() + array_size, mtGC);
-  ::new (addr + object_size()) ArrayT[length];
+inline size_t ZAttachedArray<ObjectT, ArrayT>::array_size(size_t length) {
+  return sizeof(ArrayT) * length;
+}
+
+template <typename ObjectT, typename ArrayT>
+template <typename Allocator>
+inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(Allocator* allocator, size_t length) {
+  // Allocate memory for object and array
+  const size_t size = object_size() + array_size(length);
+  void* const addr = allocator->alloc(size);
+
+  // Placement new array
+  void* const array_addr = reinterpret_cast<char*>(addr) + object_size();
+  ::new (array_addr) ArrayT[length];
+
+  // Return pointer to object
   return addr;
 }
 
+template <typename ObjectT, typename ArrayT>
+inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
+  struct Allocator {
+    void* alloc(size_t size) const {
+      return AllocateHeap(size, mtGC);
+    }
+  } allocator;
+  return alloc(&allocator, length);
+}
+
 template <typename ObjectT, typename ArrayT>
 inline void ZAttachedArray<ObjectT, ArrayT>::free(ObjectT* obj) {
   FreeHeap(obj);

src/hotspot/share/gc/z/zForwarding.cpp

@@ -23,31 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/z/zForwarding.inline.hpp"
-#include "gc/z/zPage.inline.hpp"
 #include "utilities/debug.hpp"
-#include "utilities/powerOfTwo.hpp"
-
-ZForwarding* ZForwarding::create(ZPage* page) {
-  // Allocate table for linear probing. The size of the table must be
-  // a power of two to allow for quick and inexpensive indexing/masking.
-  // The table is sized to have a load factor of 50%, i.e. sized to have
-  // double the number of entries actually inserted.
-  assert(page->live_objects() > 0, "Invalid value");
-  const size_t nentries = round_up_power_of_2(page->live_objects() * 2);
-  return ::new (AttachedArray::alloc(nentries)) ZForwarding(page, nentries);
-}
-
-void ZForwarding::destroy(ZForwarding* forwarding) {
-  AttachedArray::free(forwarding);
-}
-
-ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
-    _virtual(page->virtual_memory()),
-    _object_alignment_shift(page->object_alignment_shift()),
-    _entries(nentries),
-    _page(page),
-    _refcount(1),
-    _pinned(false) {}
 
 void ZForwarding::verify() const {
   guarantee(_refcount > 0, "Invalid refcount");

src/hotspot/share/gc/z/zForwarding.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "gc/z/zForwardingEntry.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 
+class ZForwardingAllocator;
 class ZPage;
 
 typedef size_t ZForwardingCursor;
@@ -57,8 +58,8 @@ class ZForwarding {
   ZForwarding(ZPage* page, size_t nentries);
 
 public:
-  static ZForwarding* create(ZPage* page);
-  static void destroy(ZForwarding* forwarding);
+  static uint32_t nentries(const ZPage* page);
+  static ZForwarding* alloc(ZForwardingAllocator* allocator, ZPage* page);
 
   uintptr_t start() const;
   size_t size() const;

src/hotspot/share/gc/z/zForwarding.inline.hpp

@@ -26,11 +26,38 @@
 
 #include "gc/z/zAttachedArray.inline.hpp"
 #include "gc/z/zForwarding.hpp"
+#include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zHash.inline.hpp"
 #include "gc/z/zHeap.hpp"
+#include "gc/z/zPage.inline.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+inline uint32_t ZForwarding::nentries(const ZPage* page) {
+  // The number returned by the function is used to size the hash table of
+  // forwarding entries for this page. This hash table uses linear probing.
+  // The size of the table must be a power of two to allow for quick and
+  // inexpensive indexing/masking. The table is also sized to have a load
+  // factor of 50%, i.e. sized to have double the number of entries actually
+  // inserted, to allow for good lookup/insert performance.
+  return round_up_power_of_2(page->live_objects() * 2);
+}
+
+inline ZForwarding* ZForwarding::alloc(ZForwardingAllocator* allocator, ZPage* page) {
+  const size_t nentries = ZForwarding::nentries(page);
+  void* const addr = AttachedArray::alloc(allocator, nentries);
+  return ::new (addr) ZForwarding(page, nentries);
+}
+
+inline ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
+    _virtual(page->virtual_memory()),
+    _object_alignment_shift(page->object_alignment_shift()),
+    _entries(nentries),
+    _page(page),
+    _refcount(1),
+    _pinned(false) {}
 
 inline uintptr_t ZForwarding::start() const {
   return _virtual.start();

src/hotspot/share/gc/z/zForwardingAllocator.cpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zForwardingAllocator.hpp"
+#include "memory/allocation.inline.hpp"
+
+ZForwardingAllocator::ZForwardingAllocator() :
+    _start(NULL),
+    _end(NULL),
+    _top(NULL) {}
+
+ZForwardingAllocator::~ZForwardingAllocator() {
+  FREE_C_HEAP_ARRAY(char, _start);
+}
+
+void ZForwardingAllocator::reset(size_t size) {
+  _start = _top = REALLOC_C_HEAP_ARRAY(char, _start, size, mtGC);
+  _end = _start + size;
+}

src/hotspot/share/gc/z/zForwardingAllocator.hpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP
+#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+class ZForwardingAllocator {
+private:
+  char* _start;
+  char* _end;
+  char* _top;
+
+public:
+  ZForwardingAllocator();
+  ~ZForwardingAllocator();
+
+  void reset(size_t size);
+  size_t size() const;
+  bool is_full() const;
+
+  void* alloc(size_t size);
+};
+
+#endif // SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP

src/hotspot/share/gc/z/zForwardingAllocator.inline.hpp

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
+#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
+
+#include "gc/z/zForwardingAllocator.hpp"
+#include "utilities/debug.hpp"
+
+inline size_t ZForwardingAllocator::size() const {
+  return _end - _start;
+}
+
+inline bool ZForwardingAllocator::is_full() const {
+  return _top == _end;
+}
+
+inline void* ZForwardingAllocator::alloc(size_t size) {
+  char* const addr = _top;
+  _top += size;
+  assert(_top <= _end, "Allocation should never fail");
+  return addr;
+}
+
+#endif // SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP

src/hotspot/share/gc/z/zRelocationSet.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,35 +22,53 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zForwarding.hpp"
+#include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zRelocationSet.hpp"
+#include "gc/z/zStat.hpp"
 #include "memory/allocation.hpp"
+#include "utilities/debug.hpp"
 
 ZRelocationSet::ZRelocationSet() :
+    _allocator(),
     _forwardings(NULL),
     _nforwardings(0) {}
 
-void ZRelocationSet::populate(ZPage* const* group0, size_t ngroup0,
-                              ZPage* const* group1, size_t ngroup1) {
-  _nforwardings = ngroup0 + ngroup1;
-  _forwardings = REALLOC_C_HEAP_ARRAY(ZForwarding*, _forwardings, _nforwardings, mtGC);
+void ZRelocationSet::populate(ZPage* const* small, size_t nsmall,
+                              ZPage* const* medium, size_t nmedium,
+                              size_t forwarding_entries) {
+  // Set relocation set length
+  _nforwardings = nsmall + nmedium;
 
+  // Initialize forwarding allocator to have room for the
+  // relocation set, all forwardings, and all forwarding entries.
+  const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
+  const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
+  const size_t forwarding_entries_size = forwarding_entries * sizeof(ZForwardingEntry);
+  _allocator.reset(relocation_set_size + forwardings_size + forwarding_entries_size);
+
+  // Allocate relocation set
+  _forwardings = new (_allocator.alloc(relocation_set_size)) ZForwarding*[_nforwardings];
+
+  // Populate relocation set array
   size_t j = 0;
 
-  // Populate group 0
-  for (size_t i = 0; i < ngroup0; i++) {
-    _forwardings[j++] = ZForwarding::create(group0[i]);
+  // Populate medium pages
+  for (size_t i = 0; i < nmedium; i++) {
+    _forwardings[j++] = ZForwarding::alloc(&_allocator, medium[i]);
   }
 
-  // Populate group 1
-  for (size_t i = 0; i < ngroup1; i++) {
-    _forwardings[j++] = ZForwarding::create(group1[i]);
+  // Populate small pages
+  for (size_t i = 0; i < nsmall; i++) {
+    _forwardings[j++] = ZForwarding::alloc(&_allocator, small[i]);
   }
+
+  assert(_allocator.is_full(), "Should be full");
+
+  // Update statistics
+  ZStatRelocation::set_at_populate_relocation_set(_allocator.size());
 }
 
 void ZRelocationSet::reset() {
-  for (size_t i = 0; i < _nforwardings; i++) {
-    ZForwarding::destroy(_forwardings[i]);
-    _forwardings[i] = NULL;
-  }
+  _nforwardings = 0;
 }