8316108: [Lilliput/JDK21] Cherry-pick: 8305898: Alternative self-forwarding mechanism

Reviewed-by: shade

Author: rkennke

src/hotspot/share/gc/g1/g1OopClosures.inline.hpp

@@ -228,7 +228,7 @@ void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
     oop forwardee;
     markWord m = obj->mark();
     if (m.is_marked()) {
-      forwardee = cast_to_oop(m.decode_pointer());
+      forwardee = obj->forwardee(m);
     } else {
       forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
     }

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -207,7 +207,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
 
   markWord m = obj->mark();
   if (m.is_marked()) {
-    obj = cast_to_oop(m.decode_pointer());
+    obj = obj->forwardee(m);
   } else {
     obj = do_copy_to_survivor_space(region_attr, obj, m);
   }
@@ -620,7 +620,7 @@ NOINLINE
 oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) {
   assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
 
-  oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
+  oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
   if (forward_ptr == nullptr) {
     // Forward-to-self succeeded. We are the "owner" of the object.
     HeapRegion* r = _g1h->heap_region_containing(old);

src/hotspot/share/gc/parallel/psPromotionManager.cpp

@@ -333,7 +333,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
   // this started.  If it is the same (i.e., no forwarding
   // pointer has been installed), then this thread owns
   // it.
-  if (obj->forward_to_atomic(obj, obj_mark) == nullptr) {
+  if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
     // We won any races, we "own" this object.
     assert(obj == obj->forwardee(), "Sanity");
 

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -147,7 +147,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
     // other thread.
     OrderAccess::acquire();
     // Return the already installed forwardee.
-    return cast_to_oop(m.decode_pointer());
+    return o->forwardee(m);
   }
 }
 

src/hotspot/share/gc/serial/defNewGeneration.cpp

@@ -905,8 +905,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
 
   ContinuationGCSupport::transform_stack_chunk(old);
 
-  // forward to self
-  old->forward_to(old);
+  old->forward_to_self();
 
   _promo_failure_scan_stack.push(old);
 

src/hotspot/share/oops/markWord.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_OOPS_MARKWORD_HPP
 #define SHARE_OOPS_MARKWORD_HPP
 
+#include "gc/shared/gc_globals.hpp"
 #include "metaprogramming/primitiveConversions.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "runtime/globals.hpp"
@@ -43,6 +44,10 @@
 //  --------
 //  unused:25 hash:31 -->| unused_gap:1  age:4  unused_gap:1  lock:2 (normal object)
 //
+//  64 bits (alternative GC forwarding):
+//  ------------------------------------
+//  unused:25 hash:31 -->| unused_gap:1  age:4  self-fwd:1  lock:2 (normal object)
+//
 //  - hash contains the identity hash value: largest value is
 //    31 bits, see os::random().  Also, 64-bit vm's require
 //    a hash value no bigger than 32 bits because they will not
@@ -103,17 +108,20 @@ class markWord {
   // Constants
   static const int age_bits                       = 4;
   static const int lock_bits                      = 2;
-  static const int first_unused_gap_bits          = 1;
-  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits;
+  static const int self_forwarded_bits            = 1;
+  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - self_forwarded_bits;
   static const int hash_bits                      = max_hash_bits > 31 ? 31 : max_hash_bits;
-  static const int second_unused_gap_bits         = LP64_ONLY(1) NOT_LP64(0);
+  static const int unused_gap_bits                = LP64_ONLY(1) NOT_LP64(0);
 
   static const int lock_shift                     = 0;
-  static const int age_shift                      = lock_bits + first_unused_gap_bits;
-  static const int hash_shift                     = age_shift + age_bits + second_unused_gap_bits;
+  static const int self_forwarded_shift           = lock_shift + lock_bits;
+  static const int age_shift                      = self_forwarded_shift + self_forwarded_bits;
+  static const int hash_shift                     = age_shift + age_bits + unused_gap_bits;
 
   static const uintptr_t lock_mask                = right_n_bits(lock_bits);
   static const uintptr_t lock_mask_in_place       = lock_mask << lock_shift;
+  static const uintptr_t self_forwarded_mask      = right_n_bits(self_forwarded_bits);
+  static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift;
   static const uintptr_t age_mask                 = right_n_bits(age_bits);
   static const uintptr_t age_mask_in_place        = age_mask << age_shift;
   static const uintptr_t hash_mask                = right_n_bits(hash_bits);
@@ -260,6 +268,19 @@ class markWord {
 
   // Recover address of oop from encoded form used in mark
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
+
+#ifdef _LP64
+  inline bool self_forwarded() const {
+    bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
+    assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
+    return self_fwd;
+  }
+
+  inline markWord set_self_forwarded() const {
+    assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
+    return markWord(value() | self_forwarded_mask_in_place | marked_value);
+  }
+#endif
 };
 
 // Support atomic operations.

src/hotspot/share/oops/oop.hpp

@@ -259,14 +259,17 @@ class oopDesc {
   inline bool is_forwarded() const;
 
   inline void forward_to(oop p);
+  inline void forward_to_self();
 
   // Like "forward_to", but inserts the forwarding pointer atomically.
   // Exactly one thread succeeds in inserting the forwarding pointer, and
   // this call returns null for that thread; any other thread has the
   // value of the forwarding pointer returned and does not modify "this".
   inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
+  inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
+  inline oop forwardee(markWord header) const;
 
   // Age of object during scavenge
   inline uint age() const;

src/hotspot/share/oops/oop.inline.hpp

@@ -38,6 +38,7 @@
 #include "oops/oopsHierarchy.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/globals.hpp"
+#include "runtime/safepoint.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/macros.hpp"
@@ -270,28 +271,87 @@ bool oopDesc::is_forwarded() const {
 
 // Used by scavengers
 void oopDesc::forward_to(oop p) {
+  assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
-  assert(m.decode_pointer() == p, "encoding must be reversible");
+  assert(forwardee(m) == p, "encoding must be reversible");
   set_mark(m);
 }
 
+void oopDesc::forward_to_self() {
+#ifdef _LP64
+  if (UseAltGCForwarding) {
+    markWord m = mark();
+    // If mark is displaced, we need to preserve the real header during GC.
+    // It will be restored to the displaced header after GC.
+    assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
+    if (m.has_displaced_mark_helper()) {
+      m = m.displaced_mark_helper();
+    }
+    m = m.set_self_forwarded();
+    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
+    set_mark(m);
+  } else
+#endif
+  {
+    forward_to(oop(this));
+  }
+}
+
 oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
+  assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding");
   markWord m = markWord::encode_pointer_as_mark(p);
   assert(m.decode_pointer() == p, "encoding must be reversible");
   markWord old_mark = cas_set_mark(m, compare, order);
   if (old_mark == compare) {
     return nullptr;
   } else {
-    return cast_to_oop(old_mark.decode_pointer());
+    return forwardee(old_mark);
+  }
+}
+
+oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
+#ifdef _LP64
+  if (UseAltGCForwarding) {
+    markWord m = compare;
+    // If mark is displaced, we need to preserve the real header during GC.
+    // It will be restored to the displaced header after GC.
+    assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
+    if (m.has_displaced_mark_helper()) {
+      m = m.displaced_mark_helper();
+    }
+    m = m.set_self_forwarded();
+    assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
+    markWord old_mark = cas_set_mark(m, compare, order);
+    if (old_mark == compare) {
+      return nullptr;
+    } else {
+      assert(old_mark.is_marked(), "must be marked here");
+      return forwardee(old_mark);
+    }
+  } else
+#endif
+  {
+    return forward_to_atomic(cast_to_oop(this), compare, order);
+  }
+}
+
+oop oopDesc::forwardee(markWord header) const {
+  assert(header.is_marked(), "only decode when actually forwarded");
+#ifdef _LP64
+  if (header.self_forwarded()) {
+    return cast_to_oop(this);
+  } else
+#endif
+  {
+    return cast_to_oop(header.decode_pointer());
   }
 }
 
 // Note that the forwardee is not the same thing as the displaced_mark.
 // The forwardee is used when copying during scavenge and mark-sweep.
 // It does need to clear the low two locking- and GC-related bits.
 oop oopDesc::forwardee() const {
-  assert(is_forwarded(), "only decode when actually forwarded");
-  return cast_to_oop(mark().decode_pointer());
+  return forwardee(mark());
 }
 
 // The following method needs to be MT safe.