From 619e3fa4512001ba52a165f2630ac93687cf0605 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Wed, 3 May 2023 13:01:44 +0000
Subject: [PATCH 01/16] 8305898: Alternative self-forwarding mechanism

---
 .../share/gc/g1/g1ParScanThreadState.cpp      |  2 +-
 .../share/gc/parallel/psPromotionManager.cpp  |  2 +-
 .../share/gc/serial/defNewGeneration.cpp      |  3 +-
 src/hotspot/share/oops/markWord.hpp           | 25 +++++++--
 src/hotspot/share/oops/oop.hpp                |  6 ++-
 src/hotspot/share/oops/oop.inline.hpp         | 54 ++++++++++++++++++-
 6 files changed, 80 insertions(+), 12 deletions(-)

diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index a3740982c45e8..da7231d52a262 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -620,7 +620,7 @@ NOINLINE
 oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) {
   assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
 
-  oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
+  oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
   if (forward_ptr == NULL) {
     // Forward-to-self succeeded. We are the "owner" of the object.
     HeapRegion* r = _g1h->heap_region_containing(old);
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index 55db6bbbc6d9a..3277f7008f8e6 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -328,7 +328,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
   // this started.  If it is the same (i.e., no forwarding
   // pointer has been installed), then this thread owns
   // it.
-  if (obj->forward_to_atomic(obj, obj_mark) == nullptr) {
+  if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
     // We won any races, we "own" this object.
     assert(obj == obj->forwardee(), "Sanity");
 
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index bd256c9c5551b..6acc1db3a61c1 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -905,8 +905,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
 
   ContinuationGCSupport::transform_stack_chunk(old);
 
-  // forward to self
-  old->forward_to(old);
+  old->forward_to_self();
 
   _promo_failure_scan_stack.push(old);
 
diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp
index 2250d2a95bf0d..8c64dd6a7718c 100644
--- a/src/hotspot/share/oops/markWord.hpp
+++ b/src/hotspot/share/oops/markWord.hpp
@@ -25,6 +25,7 @@
 #ifndef SHARE_OOPS_MARKWORD_HPP
 #define SHARE_OOPS_MARKWORD_HPP
 
+#include "gc/shared/gc_globals.hpp"
 #include "metaprogramming/primitiveConversions.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "runtime/globals.hpp"
@@ -102,17 +103,20 @@ class markWord {
   // Constants
   static const int age_bits                       = 4;
   static const int lock_bits                      = 2;
-  static const int first_unused_gap_bits          = 1;
-  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits;
+  static const int self_forwarded_bits            = 1;
+  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - self_forwarded_bits;
   static const int hash_bits                      = max_hash_bits > 31 ? 31 : max_hash_bits;
-  static const int second_unused_gap_bits         = LP64_ONLY(1) NOT_LP64(0);
+  static const int unused_gap_bits                = LP64_ONLY(1) NOT_LP64(0);
 
   static const int lock_shift                     = 0;
-  static const int age_shift                      = lock_bits + first_unused_gap_bits;
-  static const int hash_shift                     = age_shift + age_bits + second_unused_gap_bits;
+  static const int self_forwarded_shift           = lock_shift + lock_bits;
+  static const int age_shift                      = self_forwarded_shift + self_forwarded_bits;
+  static const int hash_shift                     = age_shift + age_bits + unused_gap_bits;
 
   static const uintptr_t lock_mask                = right_n_bits(lock_bits);
   static const uintptr_t lock_mask_in_place       = lock_mask << lock_shift;
+  static const uintptr_t self_forwarded_mask      = right_n_bits(self_forwarded_bits);
+  static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift;
   static const uintptr_t age_mask                 = right_n_bits(age_bits);
   static const uintptr_t age_mask_in_place        = age_mask << age_shift;
   static const uintptr_t hash_mask                = right_n_bits(hash_bits);
@@ -245,6 +249,17 @@ class markWord {
 
   // Recover address of oop from encoded form used in mark
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
+
+  inline bool self_forwarded() const {
+    bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
+    assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
+    return self_fwd;
+  }
+
+  inline markWord set_self_forwarded() const {
+    assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
+    return markWord(value() | self_forwarded_mask_in_place | marked_value);
+  }
 };
 
 // Support atomic operations.
diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp
index a7be060b00744..8082da5609d1b 100644
--- a/src/hotspot/share/oops/oop.hpp
+++ b/src/hotspot/share/oops/oop.hpp
@@ -63,7 +63,9 @@ class oopDesc {
   // make use of the C++ copy/assign incorrect.
   NONCOPYABLE(oopDesc);
 
- public:
+  inline oop forwardee(markWord header) const;
+
+public:
   // Must be trivial; see verifying static assert after the class.
   oopDesc() = default;
 
@@ -258,12 +260,14 @@ class oopDesc {
   inline bool is_forwarded() const;
 
   inline void forward_to(oop p);
+  inline void forward_to_self();
 
   // Like "forward_to", but inserts the forwarding pointer atomically.
   // Exactly one thread succeeds in inserting the forwarding pointer, and
   // this call returns null for that thread; any other thread has the
   // value of the forwarding pointer returned and does not modify "this".
   inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
+  inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
 
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 4988846f1724d..543b058a59507 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -38,6 +38,7 @@
 #include "oops/oopsHierarchy.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/globals.hpp"
+#include "runtime/safepoint.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/macros.hpp"
@@ -271,6 +272,23 @@ void oopDesc::forward_to(oop p) {
   set_mark(m);
 }
 
+void oopDesc::forward_to_self() {
+  if (UseAltGCForwarding) {
+    markWord m = mark();
+    // If mark is displaced, we need to preserve real header during GC.
+    // It will be restored to the displaced header after GC.
+    assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
+    if (m.has_displaced_mark_helper()) {
+      m = m.displaced_mark_helper();
+    }
+    m = m.set_self_forwarded();
+    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable");
+    set_mark(m);
+  } else {
+    forward_to(oop(this));
+  }
+}
+
 oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(m.decode_pointer() == p, "encoding must be reversible");
@@ -282,12 +300,44 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
   }
 }
 
+oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
+  if (UseAltGCForwarding) {
+    markWord m = compare;
+    // If mark is displaced, we need to preserve real header during GC.
+    // It will be restored to the displaced header after GC.
+    assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
+    if (m.has_displaced_mark_helper()) {
+      m = m.displaced_mark_helper();
+    }
+    m = m.set_self_forwarded();
+    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable");
+    markWord old_mark = cas_set_mark(m, compare, order);
+    if (old_mark == compare) {
+      return nullptr;
+    } else {
+      assert(old_mark.is_marked(), "must be marked here");
+      return forwardee(old_mark);
+    }
+  } else {
+    return forward_to_atomic(oop(this), compare, order);
+  }
+}
+
+oop oopDesc::forwardee(markWord header) const {
+  assert(header.is_marked(), "only decode when actually forwarded");
+  if (header.self_forwarded()) {
+    assert(UseAltGCForwarding, "Only use self-fwd bits when using alt GC forwarding");
+    return cast_to_oop(this);
+  } else {
+    return cast_to_oop(header.decode_pointer());
+  }
+}
+
 // Note that the forwardee is not the same thing as the displaced_mark.
 // The forwardee is used when copying during scavenge and mark-sweep.
 // It does need to clear the low two locking- and GC-related bits.
 oop oopDesc::forwardee() const {
-  assert(is_forwarded(), "only decode when actually forwarded");
-  return cast_to_oop(mark().decode_pointer());
+  return forwardee(mark());
 }
 
 // The following method needs to be MT safe.

From 7dca17ac06ee3c091a6540d29b54abdf1d687676 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Wed, 3 May 2023 13:45:06 +0000
Subject: [PATCH 02/16] Replace uses of decode_pointer() with forwardee()

---
 src/hotspot/share/gc/g1/g1OopClosures.inline.hpp            | 2 +-
 src/hotspot/share/gc/g1/g1ParScanThreadState.cpp            | 2 +-
 src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp | 2 +-
 src/hotspot/share/oops/oop.hpp                              | 5 ++---
 src/hotspot/share/oops/oop.inline.hpp                       | 2 +-
 5 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
index fb7b0dafa159d..6e7762fc89825 100644
--- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
@@ -228,7 +228,7 @@ void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
     oop forwardee;
     markWord m = obj->mark();
     if (m.is_marked()) {
-      forwardee = cast_to_oop(m.decode_pointer());
+      forwardee = obj->forwardee(m);
     } else {
       forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
     }
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index da7231d52a262..1a2cd90f050df 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -206,7 +206,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
 
   markWord m = obj->mark();
   if (m.is_marked()) {
-    obj = cast_to_oop(m.decode_pointer());
+    obj = obj->forwardee(m);
   } else {
     obj = do_copy_to_survivor_space(region_attr, obj, m);
   }
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
index f702bc483481c..826ae394c373b 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
@@ -147,7 +147,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
     // other thread.
     OrderAccess::acquire();
     // Return the already installed forwardee.
-    return cast_to_oop(m.decode_pointer());
+    return o->forwardee(m);
   }
 }
 
diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp
index 8082da5609d1b..1b741f3a79d46 100644
--- a/src/hotspot/share/oops/oop.hpp
+++ b/src/hotspot/share/oops/oop.hpp
@@ -63,9 +63,7 @@ class oopDesc {
   // make use of the C++ copy/assign incorrect.
   NONCOPYABLE(oopDesc);
 
-  inline oop forwardee(markWord header) const;
-
-public:
+ public:
   // Must be trivial; see verifying static assert after the class.
   oopDesc() = default;
 
@@ -270,6 +268,7 @@ class oopDesc {
   inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
+  inline oop forwardee(markWord header) const;
 
   // Age of object during scavenge
   inline uint age() const;
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 543b058a59507..ef98a8d8b8653 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -268,7 +268,7 @@ bool oopDesc::is_forwarded() const {
 // Used by scavengers
 void oopDesc::forward_to(oop p) {
   markWord m = markWord::encode_pointer_as_mark(p);
-  assert(m.decode_pointer() == p, "encoding must be reversible");
+  assert(forwardee(m) == p, "encoding must be reversable");
   set_mark(m);
 }
 

From efb08a6e84c603b430e0415f431f3bdbfb1764c2 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.com>
Date: Thu, 4 May 2023 15:22:27 +0000
Subject: [PATCH 03/16] Use forwardee() in forward_to_atomic() method

---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index ef98a8d8b8653..d6db515562328 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -296,7 +296,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
   if (old_mark == compare) {
     return nullptr;
   } else {
-    return cast_to_oop(old_mark.decode_pointer());
+    return forwardee(old_mark);
   }
 }
 

From 1313a45f15c71ef8049cb08a87e154c7bd5fc575 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Tue, 9 May 2023 22:00:27 +0200
Subject: [PATCH 04/16] Update src/hotspot/share/oops/oop.inline.hpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aleksey Shipilëv <shipilev@amazon.de>
---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index d6db515562328..72c3548355ba8 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -275,7 +275,7 @@ void oopDesc::forward_to(oop p) {
 void oopDesc::forward_to_self() {
   if (UseAltGCForwarding) {
     markWord m = mark();
-    // If mark is displaced, we need to preserve real header during GC.
+    // If mark is displaced, we need to preserve the real header during GC.
     // It will be restored to the displaced header after GC.
     assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
     if (m.has_displaced_mark_helper()) {

From 06e555d52f166c6ed220af1dff1f04283f13acc9 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Tue, 9 May 2023 22:00:35 +0200
Subject: [PATCH 05/16] Update src/hotspot/share/oops/oop.inline.hpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aleksey Shipilëv <shipilev@amazon.de>
---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 72c3548355ba8..36df374337028 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -303,7 +303,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
 oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
   if (UseAltGCForwarding) {
     markWord m = compare;
-    // If mark is displaced, we need to preserve real header during GC.
+    // If mark is displaced, we need to preserve the real header during GC.
     // It will be restored to the displaced header after GC.
     assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
     if (m.has_displaced_mark_helper()) {

From 1d3e8331aba037a95d1ce79ad28285dca87e22f7 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Tue, 9 May 2023 22:01:46 +0200
Subject: [PATCH 06/16] Update src/hotspot/share/oops/oop.inline.hpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aleksey Shipilëv <shipilev@amazon.de>
---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 36df374337028..ec4077c9e55b9 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -268,7 +268,7 @@ bool oopDesc::is_forwarded() const {
 // Used by scavengers
 void oopDesc::forward_to(oop p) {
   markWord m = markWord::encode_pointer_as_mark(p);
-  assert(forwardee(m) == p, "encoding must be reversable");
+  assert(forwardee(m) == p, "encoding must be reversible");
   set_mark(m);
 }
 

From a559e8d6245b8596ec9728149b1319553c69a601 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Tue, 9 May 2023 22:02:13 +0200
Subject: [PATCH 07/16] Update src/hotspot/share/oops/oop.inline.hpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aleksey Shipilëv <shipilev@amazon.de>
---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index ec4077c9e55b9..6d66898db12d9 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -319,7 +319,7 @@ oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order)
       return forwardee(old_mark);
     }
   } else {
-    return forward_to_atomic(oop(this), compare, order);
+    return forward_to_atomic(cast_to_oop(this), compare, order);
   }
 }
 

From 94e3d071db25d210055861e518a04871d3d96ef2 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.com>
Date: Tue, 9 May 2023 20:06:08 +0000
Subject: [PATCH 08/16] @shipilev suggestions

---
 src/hotspot/share/oops/oop.inline.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 6d66898db12d9..3ea17c3b37557 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -267,6 +267,7 @@ bool oopDesc::is_forwarded() const {
 
 // Used by scavengers
 void oopDesc::forward_to(oop p) {
+  assert(p != cast_to_oop(p), "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(forwardee(m) == p, "encoding must be reversible");
   set_mark(m);
@@ -290,6 +291,7 @@ void oopDesc::forward_to_self() {
 }
 
 oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
+  assert(p != cast_to_oop(p), "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(m.decode_pointer() == p, "encoding must be reversible");
   markWord old_mark = cas_set_mark(m, compare, order);
@@ -326,7 +328,6 @@ oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order)
 oop oopDesc::forwardee(markWord header) const {
   assert(header.is_marked(), "only decode when actually forwarded");
   if (header.self_forwarded()) {
-    assert(UseAltGCForwarding, "Only use self-fwd bits when using alt GC forwarding");
     return cast_to_oop(this);
   } else {
     return cast_to_oop(header.decode_pointer());

From 915c20bc6f0f408b5684c9a565026cd0bdb9b842 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.com>
Date: Tue, 9 May 2023 20:21:05 +0000
Subject: [PATCH 09/16] Fix assert

---
 src/hotspot/share/oops/oop.inline.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 3ea17c3b37557..0d5283c81fc47 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -267,7 +267,7 @@ bool oopDesc::is_forwarded() const {
 
 // Used by scavengers
 void oopDesc::forward_to(oop p) {
-  assert(p != cast_to_oop(p), "Must not be called with self-forwarding");
+  assert(p != cast_to_oop(this), "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(forwardee(m) == p, "encoding must be reversible");
   set_mark(m);
@@ -291,7 +291,7 @@ void oopDesc::forward_to_self() {
 }
 
 oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
-  assert(p != cast_to_oop(p), "Must not be called with self-forwarding");
+  assert(p != cast_to_oop(this), "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(m.decode_pointer() == p, "encoding must be reversible");
   markWord old_mark = cas_set_mark(m, compare, order);

From 6d39d5751f9d109091b769cda905797e5b4719b7 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.com>
Date: Tue, 9 May 2023 20:49:26 +0000
Subject: [PATCH 10/16] Fix asserts (again)

---
 src/hotspot/share/oops/oop.inline.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 0d5283c81fc47..990deb3bf24d4 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -267,7 +267,7 @@ bool oopDesc::is_forwarded() const {
 
 // Used by scavengers
 void oopDesc::forward_to(oop p) {
-  assert(p != cast_to_oop(this), "Must not be called with self-forwarding");
+  assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(forwardee(m) == p, "encoding must be reversible");
   set_mark(m);
@@ -291,7 +291,7 @@ void oopDesc::forward_to_self() {
 }
 
 oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
-  assert(p != cast_to_oop(this), "Must not be called with self-forwarding");
+  assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(m.decode_pointer() == p, "encoding must be reversible");
   markWord old_mark = cas_set_mark(m, compare, order);

From eb04cd7d54f7f1b5fd83723c82a59d1dda15b797 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Wed, 10 May 2023 12:23:21 +0200
Subject: [PATCH 11/16] Update src/hotspot/share/oops/oop.inline.hpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aleksey Shipilëv <shipilev@amazon.de>
---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 990deb3bf24d4..45e56db154cba 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -283,7 +283,7 @@ void oopDesc::forward_to_self() {
       m = m.displaced_mark_helper();
     }
     m = m.set_self_forwarded();
-    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable");
+    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
     set_mark(m);
   } else {
     forward_to(oop(this));

From 40c1b0be03b5b1d56031d09f9a924ef44acd1697 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Wed, 10 May 2023 12:23:39 +0200
Subject: [PATCH 12/16] Update src/hotspot/share/oops/oop.inline.hpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aleksey Shipilëv <shipilev@amazon.de>
---
 src/hotspot/share/oops/oop.inline.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 45e56db154cba..7d265312c77a6 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -312,7 +312,7 @@ oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order)
       m = m.displaced_mark_helper();
     }
     m = m.set_self_forwarded();
-    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable");
+    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
     markWord old_mark = cas_set_mark(m, compare, order);
     if (old_mark == compare) {
       return nullptr;

From 4d9713ca239da8e294c63887426bfb97240d3130 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Wed, 10 May 2023 12:12:27 +0000
Subject: [PATCH 13/16] Rename self-forwarded -> forward-failed

---
 .../share/gc/g1/g1ParScanThreadState.cpp      |  2 +-
 .../share/gc/parallel/psPromotionManager.cpp  |  2 +-
 .../share/gc/serial/defNewGeneration.cpp      |  2 +-
 src/hotspot/share/oops/markWord.hpp           | 24 +++++++++----------
 src/hotspot/share/oops/oop.hpp                |  4 ++--
 src/hotspot/share/oops/oop.inline.hpp         | 10 ++++----
 6 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index b5eadec999865..cedfd00eb8944 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -620,7 +620,7 @@ NOINLINE
 oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) {
   assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
 
-  oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
+  oop forward_ptr = old->forward_failed_atomic(m, memory_order_relaxed);
   if (forward_ptr == nullptr) {
     // Forward-to-self succeeded. We are the "owner" of the object.
     HeapRegion* r = _g1h->heap_region_containing(old);
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index 3277f7008f8e6..2700e8b0469c5 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -328,7 +328,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
   // this started.  If it is the same (i.e., no forwarding
   // pointer has been installed), then this thread owns
   // it.
-  if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
+  if (obj->forward_failed_atomic(obj_mark) == nullptr) {
     // We won any races, we "own" this object.
     assert(obj == obj->forwardee(), "Sanity");
 
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index 6acc1db3a61c1..f0ca89b45ed05 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -905,7 +905,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
 
   ContinuationGCSupport::transform_stack_chunk(old);
 
-  old->forward_to_self();
+  old->forward_failed();
 
   _promo_failure_scan_stack.push(old);
 
diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp
index 8ee1c0ed158c7..791c4461554c0 100644
--- a/src/hotspot/share/oops/markWord.hpp
+++ b/src/hotspot/share/oops/markWord.hpp
@@ -104,20 +104,20 @@ class markWord {
   // Constants
   static const int age_bits                       = 4;
   static const int lock_bits                      = 2;
-  static const int self_forwarded_bits            = 1;
-  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - self_forwarded_bits;
+  static const int forward_failed_bits            = 1;
+  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - forward_failed_bits;
   static const int hash_bits                      = max_hash_bits > 31 ? 31 : max_hash_bits;
   static const int unused_gap_bits                = LP64_ONLY(1) NOT_LP64(0);
 
   static const int lock_shift                     = 0;
-  static const int self_forwarded_shift           = lock_shift + lock_bits;
-  static const int age_shift                      = self_forwarded_shift + self_forwarded_bits;
+  static const int forward_failed_shift           = lock_shift + lock_bits;
+  static const int age_shift                      = forward_failed_shift + forward_failed_bits;
   static const int hash_shift                     = age_shift + age_bits + unused_gap_bits;
 
   static const uintptr_t lock_mask                = right_n_bits(lock_bits);
   static const uintptr_t lock_mask_in_place       = lock_mask << lock_shift;
-  static const uintptr_t self_forwarded_mask      = right_n_bits(self_forwarded_bits);
-  static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift;
+  static const uintptr_t forward_failed_mask      = right_n_bits(forward_failed_bits);
+  static const uintptr_t forward_failed_mask_in_place = forward_failed_mask << forward_failed_shift;
   static const uintptr_t age_mask                 = right_n_bits(age_bits);
   static const uintptr_t age_mask_in_place        = age_mask << age_shift;
   static const uintptr_t hash_mask                = right_n_bits(hash_bits);
@@ -265,15 +265,15 @@ class markWord {
   // Recover address of oop from encoded form used in mark
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
 
-  inline bool self_forwarded() const {
-    bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
-    assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
-    return self_fwd;
+  inline bool forward_failed() const {
+    bool fwd_failed = mask_bits(value(), forward_failed_mask_in_place) != 0;
+    assert(!fwd_failed || UseAltGCForwarding, "Only set fwd-failed bit when using alt GC forwarding");
+    return fwd_failed;
   }
 
-  inline markWord set_self_forwarded() const {
+  inline markWord set_forward_failed() const {
     assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
-    return markWord(value() | self_forwarded_mask_in_place | marked_value);
+    return markWord(value() | forward_failed_mask_in_place | marked_value);
   }
 };
 
diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp
index 1b741f3a79d46..74c0f4e875b13 100644
--- a/src/hotspot/share/oops/oop.hpp
+++ b/src/hotspot/share/oops/oop.hpp
@@ -258,14 +258,14 @@ class oopDesc {
   inline bool is_forwarded() const;
 
   inline void forward_to(oop p);
-  inline void forward_to_self();
+  inline void forward_failed();
 
   // Like "forward_to", but inserts the forwarding pointer atomically.
   // Exactly one thread succeeds in inserting the forwarding pointer, and
   // this call returns null for that thread; any other thread has the
   // value of the forwarding pointer returned and does not modify "this".
   inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
-  inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
+  inline oop forward_failed_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
   inline oop forwardee(markWord header) const;
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 990deb3bf24d4..bd5c4dcc0a7e5 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -273,7 +273,7 @@ void oopDesc::forward_to(oop p) {
   set_mark(m);
 }
 
-void oopDesc::forward_to_self() {
+void oopDesc::forward_failed() {
   if (UseAltGCForwarding) {
     markWord m = mark();
     // If mark is displaced, we need to preserve the real header during GC.
@@ -282,7 +282,7 @@ void oopDesc::forward_to_self() {
     if (m.has_displaced_mark_helper()) {
       m = m.displaced_mark_helper();
     }
-    m = m.set_self_forwarded();
+    m = m.set_forward_failed();
     assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable");
     set_mark(m);
   } else {
@@ -302,7 +302,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
   }
 }
 
-oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
+oop oopDesc::forward_failed_atomic(markWord compare, atomic_memory_order order) {
   if (UseAltGCForwarding) {
     markWord m = compare;
     // If mark is displaced, we need to preserve the real header during GC.
@@ -311,7 +311,7 @@ oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order)
     if (m.has_displaced_mark_helper()) {
       m = m.displaced_mark_helper();
     }
-    m = m.set_self_forwarded();
+    m = m.set_forward_failed();
     assert(forwardee(m) == cast_to_oop(this), "encoding must be reversable");
     markWord old_mark = cas_set_mark(m, compare, order);
     if (old_mark == compare) {
@@ -327,7 +327,7 @@ oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order)
 
 oop oopDesc::forwardee(markWord header) const {
   assert(header.is_marked(), "only decode when actually forwarded");
-  if (header.self_forwarded()) {
+  if (header.forward_failed()) {
     return cast_to_oop(this);
   } else {
     return cast_to_oop(header.decode_pointer());

From 866771c35d14f6bb475b935e541cc62c1127bb90 Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Thu, 11 May 2023 11:48:42 +0000
Subject: [PATCH 14/16] wqRevert "Rename self-forwarded -> forward-failed"

This reverts commit 4d9713ca239da8e294c63887426bfb97240d3130.
---
 .../share/gc/g1/g1ParScanThreadState.cpp      |  2 +-
 .../share/gc/parallel/psPromotionManager.cpp  |  2 +-
 .../share/gc/serial/defNewGeneration.cpp      |  2 +-
 src/hotspot/share/oops/markWord.hpp           | 24 +++++++++----------
 src/hotspot/share/oops/oop.hpp                |  4 ++--
 src/hotspot/share/oops/oop.inline.hpp         | 10 ++++----
 6 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index cedfd00eb8944..b5eadec999865 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -620,7 +620,7 @@ NOINLINE
 oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) {
   assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
 
-  oop forward_ptr = old->forward_failed_atomic(m, memory_order_relaxed);
+  oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
   if (forward_ptr == nullptr) {
     // Forward-to-self succeeded. We are the "owner" of the object.
     HeapRegion* r = _g1h->heap_region_containing(old);
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index 2700e8b0469c5..3277f7008f8e6 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -328,7 +328,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
   // this started.  If it is the same (i.e., no forwarding
   // pointer has been installed), then this thread owns
   // it.
-  if (obj->forward_failed_atomic(obj_mark) == nullptr) {
+  if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
     // We won any races, we "own" this object.
     assert(obj == obj->forwardee(), "Sanity");
 
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index f0ca89b45ed05..6acc1db3a61c1 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -905,7 +905,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
 
   ContinuationGCSupport::transform_stack_chunk(old);
 
-  old->forward_failed();
+  old->forward_to_self();
 
   _promo_failure_scan_stack.push(old);
 
diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp
index 791c4461554c0..8ee1c0ed158c7 100644
--- a/src/hotspot/share/oops/markWord.hpp
+++ b/src/hotspot/share/oops/markWord.hpp
@@ -104,20 +104,20 @@ class markWord {
   // Constants
   static const int age_bits                       = 4;
   static const int lock_bits                      = 2;
-  static const int forward_failed_bits            = 1;
-  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - forward_failed_bits;
+  static const int self_forwarded_bits            = 1;
+  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - self_forwarded_bits;
   static const int hash_bits                      = max_hash_bits > 31 ? 31 : max_hash_bits;
   static const int unused_gap_bits                = LP64_ONLY(1) NOT_LP64(0);
 
   static const int lock_shift                     = 0;
-  static const int forward_failed_shift           = lock_shift + lock_bits;
-  static const int age_shift                      = forward_failed_shift + forward_failed_bits;
+  static const int self_forwarded_shift           = lock_shift + lock_bits;
+  static const int age_shift                      = self_forwarded_shift + self_forwarded_bits;
   static const int hash_shift                     = age_shift + age_bits + unused_gap_bits;
 
   static const uintptr_t lock_mask                = right_n_bits(lock_bits);
   static const uintptr_t lock_mask_in_place       = lock_mask << lock_shift;
-  static const uintptr_t forward_failed_mask      = right_n_bits(forward_failed_bits);
-  static const uintptr_t forward_failed_mask_in_place = forward_failed_mask << forward_failed_shift;
+  static const uintptr_t self_forwarded_mask      = right_n_bits(self_forwarded_bits);
+  static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift;
   static const uintptr_t age_mask                 = right_n_bits(age_bits);
   static const uintptr_t age_mask_in_place        = age_mask << age_shift;
   static const uintptr_t hash_mask                = right_n_bits(hash_bits);
@@ -265,15 +265,15 @@ class markWord {
   // Recover address of oop from encoded form used in mark
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
 
-  inline bool forward_failed() const {
-    bool fwd_failed = mask_bits(value(), forward_failed_mask_in_place) != 0;
-    assert(!fwd_failed || UseAltGCForwarding, "Only set fwd-failed bit when using alt GC forwarding");
-    return fwd_failed;
+  inline bool self_forwarded() const {
+    bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
+    assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
+    return self_fwd;
   }
 
-  inline markWord set_forward_failed() const {
+  inline markWord set_self_forwarded() const {
     assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
-    return markWord(value() | forward_failed_mask_in_place | marked_value);
+    return markWord(value() | self_forwarded_mask_in_place | marked_value);
   }
 };
 
diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp
index 74c0f4e875b13..1b741f3a79d46 100644
--- a/src/hotspot/share/oops/oop.hpp
+++ b/src/hotspot/share/oops/oop.hpp
@@ -258,14 +258,14 @@ class oopDesc {
   inline bool is_forwarded() const;
 
   inline void forward_to(oop p);
-  inline void forward_failed();
+  inline void forward_to_self();
 
   // Like "forward_to", but inserts the forwarding pointer atomically.
   // Exactly one thread succeeds in inserting the forwarding pointer, and
   // this call returns null for that thread; any other thread has the
   // value of the forwarding pointer returned and does not modify "this".
   inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
-  inline oop forward_failed_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
+  inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
   inline oop forwardee(markWord header) const;
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index e43c9db462f92..7d265312c77a6 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -273,7 +273,7 @@ void oopDesc::forward_to(oop p) {
   set_mark(m);
 }
 
-void oopDesc::forward_failed() {
+void oopDesc::forward_to_self() {
   if (UseAltGCForwarding) {
     markWord m = mark();
     // If mark is displaced, we need to preserve the real header during GC.
@@ -282,7 +282,7 @@ void oopDesc::forward_failed() {
     if (m.has_displaced_mark_helper()) {
       m = m.displaced_mark_helper();
     }
-    m = m.set_forward_failed();
+    m = m.set_self_forwarded();
     assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
     set_mark(m);
   } else {
@@ -302,7 +302,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
   }
 }
 
-oop oopDesc::forward_failed_atomic(markWord compare, atomic_memory_order order) {
+oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
   if (UseAltGCForwarding) {
     markWord m = compare;
     // If mark is displaced, we need to preserve the real header during GC.
@@ -311,7 +311,7 @@ oop oopDesc::forward_failed_atomic(markWord compare, atomic_memory_order order)
     if (m.has_displaced_mark_helper()) {
       m = m.displaced_mark_helper();
     }
-    m = m.set_forward_failed();
+    m = m.set_self_forwarded();
     assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
     markWord old_mark = cas_set_mark(m, compare, order);
     if (old_mark == compare) {
@@ -327,7 +327,7 @@ oop oopDesc::forward_failed_atomic(markWord compare, atomic_memory_order order)
 
 oop oopDesc::forwardee(markWord header) const {
   assert(header.is_marked(), "only decode when actually forwarded");
-  if (header.forward_failed()) {
+  if (header.self_forwarded()) {
     return cast_to_oop(this);
   } else {
     return cast_to_oop(header.decode_pointer());

From d35cfb474d30d6a9e52818efde972985247f0a4d Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Sat, 13 May 2023 22:00:58 +0000
Subject: [PATCH 15/16] Fix tests on 32bit builds

---
 src/hotspot/share/oops/markWord.hpp   |  2 ++
 src/hotspot/share/oops/oop.inline.hpp | 15 ++++++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp
index 8ee1c0ed158c7..2be3312537bc6 100644
--- a/src/hotspot/share/oops/markWord.hpp
+++ b/src/hotspot/share/oops/markWord.hpp
@@ -265,6 +265,7 @@ class markWord {
   // Recover address of oop from encoded form used in mark
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
 
+#ifdef _LP64
   inline bool self_forwarded() const {
     bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
     assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
@@ -275,6 +276,7 @@ class markWord {
     assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
     return markWord(value() | self_forwarded_mask_in_place | marked_value);
   }
+#endif
 };
 
 // Support atomic operations.
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 1a7df6097a105..20b6da48f6660 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -278,6 +278,7 @@ void oopDesc::forward_to(oop p) {
 }
 
 void oopDesc::forward_to_self() {
+#ifdef _LP64
   if (UseAltGCForwarding) {
     markWord m = mark();
     // If mark is displaced, we need to preserve the real header during GC.
@@ -289,7 +290,9 @@ void oopDesc::forward_to_self() {
     m = m.set_self_forwarded();
     assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
     set_mark(m);
-  } else {
+  } else
+#endif
+  {
     forward_to(oop(this));
   }
 }
@@ -307,6 +310,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
 }
 
 oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
+#ifdef _LP64
   if (UseAltGCForwarding) {
     markWord m = compare;
     // If mark is displaced, we need to preserve the real header during GC.
@@ -324,16 +328,21 @@ oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order)
       assert(old_mark.is_marked(), "must be marked here");
       return forwardee(old_mark);
     }
-  } else {
+  } else
+#endif
+  {
     return forward_to_atomic(cast_to_oop(this), compare, order);
   }
 }
 
 oop oopDesc::forwardee(markWord header) const {
   assert(header.is_marked(), "only decode when actually forwarded");
+#ifdef _LP64
   if (header.self_forwarded()) {
     return cast_to_oop(this);
-  } else {
+  } else
+#endif
+  {
     return cast_to_oop(header.decode_pointer());
   }
 }

From 4895ad865a2f13ce03481524b77ce5cfa3a6af1f Mon Sep 17 00:00:00 2001
From: Roman Kennke <rkennke@amazon.de>
Date: Wed, 17 May 2023 21:31:03 +0000
Subject: [PATCH 16/16] Update comment about mark-word layout

---
 src/hotspot/share/oops/markWord.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp
index 2be3312537bc6..5fc2c18ccbe9a 100644
--- a/src/hotspot/share/oops/markWord.hpp
+++ b/src/hotspot/share/oops/markWord.hpp
@@ -44,6 +44,10 @@
 //  --------
 //  unused:25 hash:31 -->| unused_gap:1  age:4  unused_gap:1  lock:2 (normal object)
 //
+//  64 bits (alternative GC forwarding):
+//  ------------------------------------
+//  unused:25 hash:31 -->| unused_gap:1  age:4  self-fwd:1  lock:2 (normal object)
+//
 //  - hash contains the identity hash value: largest value is
 //    31 bits, see os::random().  Also, 64-bit vm's require
 //    a hash value no bigger than 32 bits because they will not