8255235: ZGC: Allocate and initialize forwarding data structures in parallel

Reviewed-by: ayang, eosterlund

Author: pliden

src/hotspot/share/gc/z/zForwardingAllocator.inline.hpp

@@ -25,6 +25,7 @@
 #define SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
 
 #include "gc/z/zForwardingAllocator.hpp"
+#include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
 
 inline size_t ZForwardingAllocator::size() const {
@@ -36,9 +37,8 @@ inline bool ZForwardingAllocator::is_full() const {
 }
 
 inline void* ZForwardingAllocator::alloc(size_t size) {
-  char* const addr = _top;
-  _top += size;
-  assert(_top <= _end, "Allocation should never fail");
+  char* const addr = Atomic::fetch_and_add(&_top, size);
+  assert(addr + size <= _end, "Allocation should never fail");
   return addr;
 }
 

src/hotspot/share/gc/z/zForwardingTable.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,12 +25,34 @@
 #define SHARE_GC_Z_ZFORWARDINGTABLE_INLINE_HPP
 
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zForwardingTable.hpp"
+#include "gc/z/zGlobals.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline ZForwardingTable::ZForwardingTable() :
+    _map(ZAddressOffsetMax) {}
 
 inline ZForwarding* ZForwardingTable::get(uintptr_t addr) const {
   assert(!ZAddress::is_null(addr), "Invalid address");
   return _map.get(ZAddress::offset(addr));
 }
 
+inline void ZForwardingTable::insert(ZForwarding* forwarding) {
+  const uintptr_t offset = forwarding->start();
+  const size_t size = forwarding->size();
+
+  assert(_map.get(offset) == NULL, "Invalid entry");
+  _map.put(offset, size, forwarding);
+}
+
+inline void ZForwardingTable::remove(ZForwarding* forwarding) {
+  const uintptr_t offset = forwarding->start();
+  const size_t size = forwarding->size();
+
+  assert(_map.get(offset) == forwarding, "Invalid entry");
+  _map.put(offset, size, NULL);
+}
+
 #endif // SHARE_GC_Z_ZFORWARDINGTABLE_INLINE_HPP

src/hotspot/share/gc/z/zForwardingTable.inline.hpp

@@ -65,7 +65,7 @@ ZHeap::ZHeap() :
     _reference_processor(&_workers),
     _weak_roots_processor(&_workers),
     _relocate(&_workers),
-    _relocation_set(),
+    _relocation_set(&_workers),
     _unload(&_workers),
     _serviceability(min_capacity(), max_capacity()) {
   // Install global heap instance
@@ -377,8 +377,11 @@ void ZHeap::select_relocation_set() {
   // Allow pages to be deleted
   _page_allocator.disable_deferred_delete();
 
-  // Select pages to relocate
-  selector.select(&_relocation_set);
+  // Select relocation set
+  selector.select();
+
+  // Install relocation set
+  _relocation_set.install(&selector);
 
   // Setup forwarding table
   ZRelocationSetIterator rs_iter(&_relocation_set);

src/hotspot/share/gc/z/zHeap.cpp

@@ -22,51 +22,106 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zArray.inline.hpp"
 #include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zRelocationSet.hpp"
+#include "gc/z/zRelocationSetSelector.inline.hpp"
 #include "gc/z/zStat.hpp"
-#include "memory/allocation.hpp"
+#include "gc/z/zTask.hpp"
+#include "gc/z/zWorkers.hpp"
+#include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
 
-ZRelocationSet::ZRelocationSet() :
-    _allocator(),
-    _forwardings(NULL),
-    _nforwardings(0) {}
+class ZRelocationSetInstallTask : public ZTask {
+private:
+  ZForwardingAllocator* const    _allocator;
+  ZForwarding**                  _forwardings;
+  const size_t                   _nforwardings;
+  ZArrayParallelIterator<ZPage*> _small_iter;
+  ZArrayParallelIterator<ZPage*> _medium_iter;
+  volatile size_t                _small_next;
+  volatile size_t                _medium_next;
+
+  void install(ZForwarding* forwarding, volatile size_t* next) {
+    const size_t index = Atomic::fetch_and_add(next, 1u);
+    assert(index < _nforwardings, "Invalid index");
+    _forwardings[index] = forwarding;
+  }
 
-void ZRelocationSet::populate(ZPage* const* small, size_t nsmall,
-                              ZPage* const* medium, size_t nmedium,
-                              size_t forwarding_entries) {
-  // Set relocation set length
-  _nforwardings = nsmall + nmedium;
+  void install_small(ZForwarding* forwarding) {
+    install(forwarding, &_small_next);
+  }
 
-  // Initialize forwarding allocator to have room for the
-  // relocation set, all forwardings, and all forwarding entries.
-  const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
-  const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
-  const size_t forwarding_entries_size = forwarding_entries * sizeof(ZForwardingEntry);
-  _allocator.reset(relocation_set_size + forwardings_size + forwarding_entries_size);
+  void install_medium(ZForwarding* forwarding) {
+    install(forwarding, &_medium_next);
+  }
 
-  // Allocate relocation set
-  _forwardings = new (_allocator.alloc(relocation_set_size)) ZForwarding*[_nforwardings];
+public:
+  ZRelocationSetInstallTask(ZForwardingAllocator* allocator, const ZRelocationSetSelector* selector) :
+      ZTask("ZRelocationSetInstallTask"),
+      _allocator(allocator),
+      _forwardings(NULL),
+      _nforwardings(selector->small()->length() + selector->medium()->length()),
+      _small_iter(selector->small()),
+      _medium_iter(selector->medium()),
+      _small_next(selector->medium()->length()),
+      _medium_next(0) {
 
-  // Populate relocation set array
-  size_t j = 0;
+    // Reset the allocator to have room for the relocation
+    // set, all forwardings, and all forwarding entries.
+    const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
+    const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
+    const size_t forwarding_entries_size = selector->forwarding_entries() * sizeof(ZForwardingEntry);
+    _allocator->reset(relocation_set_size + forwardings_size + forwarding_entries_size);
 
-  // Populate medium pages
-  for (size_t i = 0; i < nmedium; i++) {
-    _forwardings[j++] = ZForwarding::alloc(&_allocator, medium[i]);
+    // Allocate relocation set
+    _forwardings = new (_allocator->alloc(relocation_set_size)) ZForwarding*[_nforwardings];
   }
 
-  // Populate small pages
-  for (size_t i = 0; i < nsmall; i++) {
-    _forwardings[j++] = ZForwarding::alloc(&_allocator, small[i]);
+  ~ZRelocationSetInstallTask() {
+    assert(_allocator->is_full(), "Should be full");
+  }
+
+  virtual void work() {
+    // Allocate and install forwardings for small pages
+    for (ZPage* page; _small_iter.next(&page);) {
+      ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page);
+      install_small(forwarding);
+    }
+
+    // Allocate and install forwardings for medium pages
+    for (ZPage* page; _medium_iter.next(&page);) {
+      ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page);
+      install_medium(forwarding);
+    }
   }
 
-  assert(_allocator.is_full(), "Should be full");
+  ZForwarding** forwardings() const {
+    return _forwardings;
+  }
+
+  size_t nforwardings() const {
+    return _nforwardings;
+  }
+};
+
+ZRelocationSet::ZRelocationSet(ZWorkers* workers) :
+    _workers(workers),
+    _allocator(),
+    _forwardings(NULL),
+    _nforwardings(0) {}
+
+void ZRelocationSet::install(const ZRelocationSetSelector* selector) {
+  // Install relocation set
+  ZRelocationSetInstallTask task(&_allocator, selector);
+  _workers->run_concurrent(&task);
+
+  _forwardings = task.forwardings();
+  _nforwardings = task.nforwardings();
 
   // Update statistics
-  ZStatRelocation::set_at_populate_relocation_set(_allocator.size());
+  ZStatRelocation::set_at_install_relocation_set(_allocator.size());
 }
 
 void ZRelocationSet::reset() {

src/hotspot/share/gc/z/zRelocationSet.cpp

@@ -26,25 +26,24 @@
 
 #include "gc/z/zArray.hpp"
 #include "gc/z/zForwardingAllocator.hpp"
-#include "memory/allocation.hpp"
 
 class ZForwarding;
-class ZPage;
+class ZRelocationSetSelector;
+class ZWorkers;
 
 class ZRelocationSet {
   template <bool> friend class ZRelocationSetIteratorImpl;
 
 private:
+  ZWorkers*            _workers;
   ZForwardingAllocator _allocator;
   ZForwarding**        _forwardings;
   size_t               _nforwardings;
 
 public:
-  ZRelocationSet();
+  ZRelocationSet(ZWorkers* workers);
 
-  void populate(ZPage* const* small, size_t nsmall,
-                ZPage* const* medium, size_t nmedium,
-                size_t forwarding_entries);
+  void install(const ZRelocationSetSelector* selector);
   void reset();
 };