From bf66d734bc70d68200e8aaeb525c92c70b0b6baa Mon Sep 17 00:00:00 2001
From: Per Liden <pliden@openjdk.org>
Date: Thu, 26 Nov 2020 12:30:28 +0000
Subject: [PATCH] 8257073: ZGC: Try forward object before retaining page

Reviewed-by: ayang, eosterlund, stefank
---
 src/hotspot/share/gc/z/zForwarding.inline.hpp |  4 ++-
 src/hotspot/share/gc/z/zRelocate.cpp          | 26 +++++++++----------
 src/hotspot/share/gc/z/zRelocate.hpp          |  1 -
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/hotspot/share/gc/z/zForwarding.inline.hpp b/src/hotspot/share/gc/z/zForwarding.inline.hpp
index cd6c73571e963..dc9998e625246 100644
--- a/src/hotspot/share/gc/z/zForwarding.inline.hpp
+++ b/src/hotspot/share/gc/z/zForwarding.inline.hpp
@@ -94,7 +94,9 @@ inline ZForwardingEntry* ZForwarding::entries() const {
 }
 
 inline ZForwardingEntry ZForwarding::at(ZForwardingCursor* cursor) const {
-  return Atomic::load(entries() + *cursor);
+  // Load acquire for correctness with regards to
+  // accesses to the contents of the forwarded object.
+  return Atomic::load_acquire(entries() + *cursor);
 }
 
 inline ZForwardingEntry ZForwarding::first(uintptr_t from_index, ZForwardingCursor* cursor) const {
diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp
index bd12d4f2f2879..f956e2026e345 100644
--- a/src/hotspot/share/gc/z/zRelocate.cpp
+++ b/src/hotspot/share/gc/z/zRelocate.cpp
@@ -58,21 +58,12 @@ static uintptr_t forwarding_insert(ZForwarding* forwarding, uintptr_t from_addr,
   return ZAddress::good(to_offset_final);
 }
 
-uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t from_addr) const {
-  ZForwardingCursor cursor;
-
-  // Lookup forwarding
-  uintptr_t to_addr = forwarding_find(forwarding, from_addr, &cursor);
-  if (to_addr != 0) {
-    // Already relocated
-    return to_addr;
-  }
-
+static uintptr_t relocate_object_inner(ZForwarding* forwarding, uintptr_t from_addr, ZForwardingCursor* cursor) {
   assert(ZHeap::heap()->is_object_live(from_addr), "Should be live");
 
   // Allocate object
   const size_t size = ZUtils::object_size(from_addr);
-  to_addr = ZHeap::heap()->alloc_object_non_blocking(size);
+  const uintptr_t to_addr = ZHeap::heap()->alloc_object_non_blocking(size);
   if (to_addr == 0) {
     // Allocation failed
     return 0;
@@ -82,7 +73,7 @@ uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t fr
   ZUtils::object_copy_disjoint(from_addr, to_addr, size);
 
   // Insert forwarding
-  const uintptr_t to_addr_final = forwarding_insert(forwarding, from_addr, to_addr, &cursor);
+  const uintptr_t to_addr_final = forwarding_insert(forwarding, from_addr, to_addr, cursor);
   if (to_addr_final != to_addr) {
     // Already relocated, try undo allocation
     ZHeap::heap()->undo_alloc_object(to_addr, size);
@@ -92,9 +83,18 @@ uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t fr
 }
 
 uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const {
+  ZForwardingCursor cursor;
+
+  // Lookup forwarding
+  uintptr_t to_addr = forwarding_find(forwarding, from_addr, &cursor);
+  if (to_addr != 0) {
+    // Already relocated
+    return to_addr;
+  }
+
   // Relocate object
   if (forwarding->retain_page()) {
-    const uintptr_t to_addr = relocate_object_inner(forwarding, from_addr);
+    to_addr = relocate_object_inner(forwarding, from_addr, &cursor);
     forwarding->release_page();
 
     if (to_addr != 0) {
diff --git a/src/hotspot/share/gc/z/zRelocate.hpp b/src/hotspot/share/gc/z/zRelocate.hpp
index f701ce80396b5..0400837d45298 100644
--- a/src/hotspot/share/gc/z/zRelocate.hpp
+++ b/src/hotspot/share/gc/z/zRelocate.hpp
@@ -35,7 +35,6 @@ class ZRelocate {
 private:
   ZWorkers* const _workers;
 
-  uintptr_t relocate_object_inner(ZForwarding* forwarding, uintptr_t from_addr) const;
   void work(ZRelocationSetParallelIterator* iter);
 
 public: