8305895: Implement JEP 450: Compact Object Headers (Experimental)

src/hotspot/share/gc/g1/g1Arguments.cpp

@@ -34,8 +34,8 @@
 #include "gc/g1/g1HeapRegionRemSet.hpp"
 #include "gc/g1/g1HeapVerifier.hpp"
 #include "gc/shared/cardTable.hpp"
+#include "gc/shared/fullGCForwarding.hpp"
 #include "gc/shared/gcArguments.hpp"
-#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/workerPolicy.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/globals_extension.hpp"
@@ -248,7 +248,7 @@ void G1Arguments::initialize() {
 
 void G1Arguments::initialize_heap_flags_and_sizes() {
   GCArguments::initialize_heap_flags_and_sizes();
-  GCForwarding::initialize_flags(heap_reserved_size_bytes());
+  FullGCForwarding::initialize_flags(heap_reserved_size_bytes());
 }
 
 CollectedHeap* G1Arguments::create_heap() {

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -77,8 +77,8 @@
 #include "gc/g1/g1YoungGCAllocationFailureInjector.hpp"
 #include "gc/shared/classUnloadingContext.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
+#include "gc/shared/fullGCForwarding.hpp"
 #include "gc/shared/gcBehaviours.hpp"
-#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcTimer.hpp"
@@ -1435,7 +1435,7 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
-  GCForwarding::initialize();
+  FullGCForwarding::initialize();
 
   return JNI_OK;
 }

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -29,7 +29,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/g1HeapRegion.inline.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
@@ -42,7 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
 
 size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
   size_t size = obj->size();
-  if (GCForwarding::is_forwarded(obj)) {
+  if (FullGCForwarding::is_forwarded(obj)) {
     G1FullGCCompactTask::copy_object_to_new_location(obj);
   }
 
@@ -53,13 +53,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
 }
 
 void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
-  assert(GCForwarding::is_forwarded(obj), "Sanity!");
-  assert(GCForwarding::forwardee(obj) != obj, "Object must have a new location");
+  assert(FullGCForwarding::is_forwarded(obj), "Sanity!");
+  assert(FullGCForwarding::forwardee(obj) != obj, "Object must have a new location");
 
   size_t size = obj->size();
   // Copy object and reinit its mark.
   HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
-  HeapWord* destination = cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj));
+  HeapWord* destination = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj));
   Copy::aligned_conjoint_words(obj_addr, destination, size);
 
   // There is no need to transform stack chunks - marking already did that.
@@ -122,7 +122,7 @@ void G1FullGCCompactTask::compact_humongous_obj(G1HeapRegion* src_hr) {
   size_t word_size = obj->size();
 
   uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
-  HeapWord* destination = cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj));
+  HeapWord* destination = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj));
 
   assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
   collector()->mark_bitmap()->clear(obj);

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -26,7 +26,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1HeapRegion.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
@@ -107,10 +107,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
     if (!object->is_forwarded()) {
       preserved_stack()->push_if_necessary(object, object->mark());
     }
-    GCForwarding::forward_to(object, cast_to_oop(_compaction_top));
-    assert(GCForwarding::is_forwarded(object), "must be forwarded");
+    FullGCForwarding::forward_to(object, cast_to_oop(_compaction_top));
+    assert(FullGCForwarding::is_forwarded(object), "must be forwarded");
   } else {
-    assert(!GCForwarding::is_forwarded(object), "must not be forwarded");
+    assert(!FullGCForwarding::is_forwarded(object), "must not be forwarded");
   }
 
   // Update compaction values.
@@ -173,8 +173,8 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) {
   preserved_stack()->push_if_necessary(obj, obj->mark());
 
   G1HeapRegion* dest_hr = _compaction_regions->at(range_begin);
-  GCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom()));
-  assert(GCForwarding::is_forwarded(obj), "Object must be forwarded!");
+  FullGCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom()));
+  assert(FullGCForwarding::is_forwarded(obj), "Object must be forwarded!");
 
   // Add the humongous object regions to the compaction point.
   add_humongous(hr);

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -32,7 +32,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/g1HeapRegionRemSet.inline.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"
@@ -66,8 +66,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
     return;
   }
 
-  if (GCForwarding::is_forwarded(obj)) {
-    oop forwardee = GCForwarding::forwardee(obj);
+  if (FullGCForwarding::is_forwarded(obj)) {
+    oop forwardee = FullGCForwarding::forwardee(obj);
     // Forwarded, just update.
     assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
     RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);

src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp

@@ -32,7 +32,7 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCScope.hpp"
 #include "gc/g1/g1HeapRegion.inline.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 
 void G1DetermineCompactionQueueClosure::free_empty_humongous_region(G1HeapRegion* hr) {
   _g1h->free_humongous_region(hr, nullptr);
@@ -115,10 +115,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(G1HeapRegion* hr)
 }
 
 inline size_t G1SerialRePrepareClosure::apply(oop obj) {
-  if (GCForwarding::is_forwarded(obj)) {
+  if (FullGCForwarding::is_forwarded(obj)) {
     // We skip objects compiled into the first region or
     // into regions not part of the serial compaction point.
-    if (cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj)) < _dense_prefix_top) {
+    if (cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj)) < _dense_prefix_top) {
       return obj->size();
     }
   }

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -469,12 +469,12 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
   assert(region_attr.is_in_cset(),
          "Unexpected region attr type: %s", region_attr.get_type_str());
 
-  // NOTE: With compact headers, it is not safe to load the Klass* from o, because
-  // that would access the mark-word, and the mark-word might change at any time by
-  // concurrent promotion. The promoted mark-word would point to the forwardee, which
-  // may not yet have completed copying. Therefore we must load the Klass* from
-  // the mark-word that we have already loaded. This is safe, because we have checked
-  // that this is not yet forwarded in the caller.
+  // NOTE: With compact headers, it is not safe to load the Klass* from old, because
+  // that would access the mark-word, that might change at any time by concurrent
+  // workers.
+  // This mark word would refer to a forwardee, which may not yet have completed
+  // copying. Therefore we must load the Klass* from the mark-word that we already
+  // loaded. This is safe, because we only enter here if not yet forwarded.
   assert(!old_mark.is_forwarded(), "precondition");
   Klass* klass = UseCompactObjectHeaders
       ? old_mark.klass()

src/hotspot/share/gc/parallel/mutableSpace.cpp

@@ -218,7 +218,7 @@ void MutableSpace::object_iterate(ObjectClosure* cl) {
     // so we can encounter objects with "forwarded" markword.
     // They are essentially dead, so skipping them
     if (obj->is_forwarded()) {
-      assert(obj->forwardee() != obj, "must not be self-forwarded");
+      assert(!obj->is_self_forwarded(), "must not be self-forwarded");
       // It is safe to use the forwardee here. Parallel GC only uses
       // header-based forwarding during promotion. Full GC doesn't
       // use the object header for forwarding at all.

src/hotspot/share/gc/parallel/parallelArguments.cpp

@@ -27,8 +27,8 @@
 #include "gc/parallel/parallelArguments.hpp"
 #include "gc/parallel/parallelScavengeHeap.hpp"
 #include "gc/shared/adaptiveSizePolicy.hpp"
+#include "gc/shared/fullGCForwarding.hpp"
 #include "gc/shared/gcArguments.hpp"
-#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/genArguments.hpp"
 #include "gc/shared/workerPolicy.hpp"
 #include "logging/log.hpp"
@@ -128,7 +128,7 @@ void ParallelArguments::initialize_heap_flags_and_sizes() {
     // Redo everything from the start
     initialize_heap_flags_and_sizes_one_pass();
   }
-  GCForwarding::initialize_flags(heap_reserved_size_bytes());
+  FullGCForwarding::initialize_flags(heap_reserved_size_bytes());
 }
 
 size_t ParallelArguments::heap_reserved_size_bytes() {

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -33,7 +33,7 @@
 #include "gc/parallel/psPromotionManager.hpp"
 #include "gc/parallel/psScavenge.hpp"
 #include "gc/parallel/psVMOperations.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
 #include "gc/shared/gcWhen.hpp"
@@ -130,7 +130,7 @@ jint ParallelScavengeHeap::initialize() {
 
   ParallelInitLogger::print();
 
-  GCForwarding::initialize();
+  FullGCForwarding::initialize();
 
   return JNI_OK;
 }

src/hotspot/share/gc/parallel/psParallelCompact.cpp

@@ -44,8 +44,8 @@
 #include "gc/parallel/psStringDedup.hpp"
 #include "gc/parallel/psYoungGen.hpp"
 #include "gc/shared/classUnloadingContext.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "gc/shared/gcCause.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcLocker.hpp"
@@ -1597,7 +1597,7 @@ void PSParallelCompact::forward_to_new_addr() {
             oop obj = cast_to_oop(cur_addr);
             if (new_addr != cur_addr) {
               cm->preserved_marks()->push_if_necessary(obj, obj->mark());
-              GCForwarding::forward_to(obj, cast_to_oop(new_addr));
+              FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
             }
             size_t obj_size = obj->size();
             live_words += obj_size;
@@ -1640,7 +1640,7 @@ void PSParallelCompact::verify_forward() {
       }
       oop obj = cast_to_oop(cur_addr);
       if (cur_addr != bump_ptr) {
-        assert(GCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv");
+        assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv");
       }
       bump_ptr += obj->size();
       cur_addr += obj->size();
@@ -2403,8 +2403,8 @@ void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) {
   if (copy_destination() != source()) {
     DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
     assert(source() != destination(), "inv");
-    assert(GCForwarding::is_forwarded(cast_to_oop(source())), "inv");
-    assert(GCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
+    assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv");
+    assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
     Copy::aligned_conjoint_words(source(), copy_destination(), words);
     cast_to_oop(copy_destination())->init_mark();
   }

src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp

@@ -31,7 +31,7 @@
 #include "gc/parallel/parMarkBitMap.inline.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/klass.hpp"
@@ -80,7 +80,7 @@ inline void PSParallelCompact::adjust_pointer(T* p) {
     if (!obj->is_forwarded()) {
       return;
     }
-    oop new_obj = GCForwarding::forwardee(obj);
+    oop new_obj = FullGCForwarding::forwardee(obj);
     assert(new_obj != nullptr, "non-null address for live objects");
     assert(new_obj != obj, "inv");
     assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj),

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -166,12 +166,12 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
   oop new_obj = nullptr;
   bool new_obj_is_tenured = false;
 
-  // NOTE: With compact headers, it is not safe to load the Klass* from o, because
-  // that would access the mark-word, and the mark-word might change at any time by
-  // concurrent promotion. The promoted mark-word would point to the forwardee, which
-  // may not yet have completed copying. Therefore we must load the Klass* from
-  // the mark-word that we have already loaded. This is safe, because we have checked
-  // that this is not yet forwarded in the caller.)
+  // NOTE: With compact headers, it is not safe to load the Klass* from old, because
+  // that would access the mark-word, that might change at any time by concurrent
+  // workers.
+  // This mark word would refer to a forwardee, which may not yet have completed
+  // copying. Therefore we must load the Klass* from the mark-word that we already
+  // loaded. This is safe, because we only enter here if not yet forwarded.
   assert(!test_mark.is_forwarded(), "precondition");
   Klass* klass = UseCompactObjectHeaders
       ? test_mark.klass()

src/hotspot/share/gc/serial/serialArguments.cpp

@@ -23,14 +23,14 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/shared/gcForwarding.hpp"
+#include "gc/shared/fullGCForwarding.hpp"
 #include "gc/shared/genArguments.hpp"
 #include "gc/serial/serialArguments.hpp"
 #include "gc/serial/serialHeap.hpp"
 
 void SerialArguments::initialize_heap_flags_and_sizes() {
   GenArguments::initialize_heap_flags_and_sizes();
-  GCForwarding::initialize_flags(MaxHeapSize);
+  FullGCForwarding::initialize_flags(MaxHeapSize);
 }
 
 CollectedHeap* SerialArguments::create_heap() {

src/hotspot/share/gc/serial/serialFullGC.cpp

@@ -43,7 +43,7 @@
 #include "gc/shared/classUnloadingContext.hpp"
 #include "gc/shared/collectedHeap.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
-#include "gc/shared/gcForwarding.inline.hpp"
+#include "gc/shared/fullGCForwarding.inline.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcTimer.hpp"
 #include "gc/shared/gcTrace.hpp"
@@ -231,7 +231,7 @@ class Compacter {
   static void forward_obj(oop obj, HeapWord* new_addr) {
     prefetch_write_scan(obj);
     if (cast_from_oop<HeapWord*>(obj) != new_addr) {
-      GCForwarding::forward_to(obj, cast_to_oop(new_addr));
+      FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
     } else {
       assert(obj->is_gc_marked(), "inv");
       // This obj will stay in-place. Fix the markword.
@@ -256,7 +256,7 @@ class Compacter {
     prefetch_read_scan(addr);
 
     oop obj = cast_to_oop(addr);
-    oop new_obj = GCForwarding::forwardee(obj);
+    oop new_obj = FullGCForwarding::forwardee(obj);
     HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
     assert(addr != new_addr, "inv");
     prefetch_write_copy(new_addr);
@@ -353,13 +353,13 @@ class Compacter {
       HeapWord* top = space->top();
 
       // Check if the first obj inside this space is forwarded.
-      if (!GCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
+      if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
         // Jump over consecutive (in-place) live-objs-chunk
         cur_addr = get_first_dead(i);
       }
 
       while (cur_addr < top) {
-        if (!GCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
+        if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
           cur_addr = *(HeapWord**) cur_addr;
           continue;
         }
@@ -625,8 +625,8 @@ template <class T> void SerialFullGC::adjust_pointer(T* p) {
     oop obj = CompressedOops::decode_not_null(heap_oop);
     assert(Universe::heap()->is_in(obj), "should be in heap");
 
-    if (GCForwarding::is_forwarded(obj)) {
-      oop new_obj = GCForwarding::forwardee(obj);
+    if (FullGCForwarding::is_forwarded(obj)) {
+      oop new_obj = FullGCForwarding::forwardee(obj);
       assert(is_object_aligned(new_obj), "oop must be aligned");
       RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
     }

src/hotspot/share/gc/serial/serialHeap.cpp

@@ -40,7 +40,7 @@
 #include "gc/shared/collectedHeap.inline.hpp"
 #include "gc/shared/collectorCounters.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
-#include "gc/shared/gcForwarding.hpp"
+#include "gc/shared/fullGCForwarding.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcInitLogger.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
@@ -201,7 +201,7 @@ jint SerialHeap::initialize() {
 
   GCInitLogger::print();
 
-  GCForwarding::initialize();
+  FullGCForwarding::initialize();
 
   return JNI_OK;
 }

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -706,7 +706,7 @@ int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
   int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
                             instanceOopDesc::base_offset_in_bytes();
   // base_off:
-  // 8  - 32-bit VM
+  // 8  - 32-bit VM or 64-bit VM, compact headers
   // 12 - 64-bit VM, compressed klass
   // 16 - 64-bit VM, normal klass
   if (base_off % BytesPerLong != 0) {