diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp index ea7f690d325fc..b77056d254f25 100644 --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp @@ -228,7 +228,7 @@ void G1ParCopyClosure::do_oop_work(T* p) { oop forwardee; markWord m = obj->mark(); if (m.is_marked()) { - forwardee = cast_to_oop(m.decode_pointer()); + forwardee = obj->forwardee(m); } else { forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 8d7ee0b1dfdd0..278609d522707 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -212,7 +212,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) { markWord m = obj->mark(); if (m.is_marked()) { - obj = cast_to_oop(m.decode_pointer()); + obj = obj->forwardee(m); } else { obj = do_copy_to_survivor_space(region_attr, obj, m); } @@ -632,7 +632,7 @@ NOINLINE oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz, bool cause_pinned) { assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old)); - oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed); + oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed); if (forward_ptr == nullptr) { // Forward-to-self succeeded. We are the "owner" of the object. HeapRegion* r = _g1h->heap_region_containing(old); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index 7610a6e13d77b..2194ab8e59270 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -325,7 +325,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) { // this started. If it is the same (i.e., no forwarding // pointer has been installed), then this thread owns // it. - if (obj->forward_to_atomic(obj, obj_mark) == nullptr) { + if (obj->forward_to_self_atomic(obj_mark) == nullptr) { // We won any races, we "own" this object. assert(obj == obj->forwardee(), "Sanity"); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index c1cbeb0f597bb..b65eaa39674b6 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -152,7 +152,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) { // other thread. OrderAccess::acquire(); // Return the already installed forwardee. - return cast_to_oop(m.decode_pointer()); + return o->forwardee(m); } } diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp index b569fbac48f0f..e0e0dfb55b754 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -898,8 +898,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) { ContinuationGCSupport::transform_stack_chunk(old); - // forward to self - old->forward_to(old); + old->forward_to_self(); _promo_failure_scan_stack.push(old); diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index be85fb8ba82ad..f5540975c8697 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_OOPS_MARKWORD_HPP #define SHARE_OOPS_MARKWORD_HPP +#include "gc/shared/gc_globals.hpp" #include "metaprogramming/primitiveConversions.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/globals.hpp" @@ -43,6 +44,10 @@ // -------- // unused:25 hash:31 -->| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object) // +// 64 bits (alternative GC forwarding): +// ------------------------------------ +// unused:25 hash:31 -->| unused_gap:1 age:4 self-fwd:1 lock:2 (normal object) +// // - hash contains the identity hash value: largest value is // 31 bits, see os::random(). Also, 64-bit vm's require // a hash value no bigger than 32 bits because they will not @@ -103,17 +108,20 @@ class markWord { // Constants static const int age_bits = 4; static const int lock_bits = 2; - static const int first_unused_gap_bits = 1; - static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits; + static const int self_forwarded_bits = 1; + static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - self_forwarded_bits; static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits; - static const int second_unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); + static const int unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); static const int lock_shift = 0; - static const int age_shift = lock_bits + first_unused_gap_bits; - static const int hash_shift = age_shift + age_bits + second_unused_gap_bits; + static const int self_forwarded_shift = lock_shift + lock_bits; + static const int age_shift = self_forwarded_shift + self_forwarded_bits; + static const int hash_shift = age_shift + age_bits + unused_gap_bits; static const uintptr_t lock_mask = right_n_bits(lock_bits); static const uintptr_t lock_mask_in_place = lock_mask << lock_shift; + static const uintptr_t self_forwarded_mask = right_n_bits(self_forwarded_bits); + static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift; static const uintptr_t age_mask = right_n_bits(age_bits); static const uintptr_t age_mask_in_place = age_mask << age_shift; static const uintptr_t hash_mask = right_n_bits(hash_bits); @@ -260,6 +268,19 @@ class markWord { // Recover address of oop from encoded form used in mark inline void* decode_pointer() { return (void*)clear_lock_bits().value(); } + +#ifdef _LP64 + inline bool self_forwarded() const { + bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0; + assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding"); + return self_fwd; + } + + inline markWord set_self_forwarded() const { + assert(UseAltGCForwarding, "Only call this with alt GC forwarding"); + return markWord(value() | self_forwarded_mask_in_place | marked_value); + } +#endif }; // Support atomic operations. diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 50d8b6041b674..bab00194c7dd4 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -260,14 +260,17 @@ class oopDesc { inline bool is_forwarded() const; inline void forward_to(oop p); + inline void forward_to_self(); // Like "forward_to", but inserts the forwarding pointer atomically. // Exactly one thread succeeds in inserting the forwarding pointer, and // this call returns null for that thread; any other thread has the // value of the forwarding pointer returned and does not modify "this". inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative); + inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative); inline oop forwardee() const; + inline oop forwardee(markWord header) const; // Age of object during scavenge inline uint age() const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index a9b88e32651d8..f5a63cd25e7c7 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -38,6 +38,7 @@ #include "oops/oopsHierarchy.hpp" #include "runtime/atomic.hpp" #include "runtime/globals.hpp" +#include "runtime/safepoint.hpp" #include "utilities/align.hpp" #include "utilities/debug.hpp" #include "utilities/macros.hpp" @@ -270,19 +271,79 @@ bool oopDesc::is_forwarded() const { // Used by scavengers void oopDesc::forward_to(oop p) { + assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding"); markWord m = markWord::encode_pointer_as_mark(p); - assert(m.decode_pointer() == p, "encoding must be reversible"); + assert(forwardee(m) == p, "encoding must be reversible"); set_mark(m); } +void oopDesc::forward_to_self() { +#ifdef _LP64 + if (UseAltGCForwarding) { + markWord m = mark(); + // If mark is displaced, we need to preserve the real header during GC. + // It will be restored to the displaced header after GC. + assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint"); + if (m.has_displaced_mark_helper()) { + m = m.displaced_mark_helper(); + } + m = m.set_self_forwarded(); + assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible"); + set_mark(m); + } else +#endif + { + forward_to(oop(this)); + } +} + oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) { + assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding"); markWord m = markWord::encode_pointer_as_mark(p); assert(m.decode_pointer() == p, "encoding must be reversible"); markWord old_mark = cas_set_mark(m, compare, order); if (old_mark == compare) { return nullptr; } else { - return cast_to_oop(old_mark.decode_pointer()); + return forwardee(old_mark); + } +} + +oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) { +#ifdef _LP64 + if (UseAltGCForwarding) { + markWord m = compare; + // If mark is displaced, we need to preserve the real header during GC. + // It will be restored to the displaced header after GC. + assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint"); + if (m.has_displaced_mark_helper()) { + m = m.displaced_mark_helper(); + } + m = m.set_self_forwarded(); + assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible"); + markWord old_mark = cas_set_mark(m, compare, order); + if (old_mark == compare) { + return nullptr; + } else { + assert(old_mark.is_marked(), "must be marked here"); + return forwardee(old_mark); + } + } else +#endif + { + return forward_to_atomic(cast_to_oop(this), compare, order); + } +} + +oop oopDesc::forwardee(markWord header) const { + assert(header.is_marked(), "only decode when actually forwarded"); +#ifdef _LP64 + if (header.self_forwarded()) { + return cast_to_oop(this); + } else +#endif + { + return cast_to_oop(header.decode_pointer()); } } @@ -290,8 +351,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde // The forwardee is used when copying during scavenge and mark-sweep. // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { - assert(is_forwarded(), "only decode when actually forwarded"); - return cast_to_oop(mark().decode_pointer()); + return forwardee(mark()); } // The following method needs to be MT safe.