8315046: [Lilliput/JDK21] Cherry-pick: 8305896: Alternative full GC f…

…orwarding

Reviewed-by: shade

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -87,6 +87,7 @@
 #include "gc/shared/oopStorageParState.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/referenceProcessor.inline.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "gc/shared/taskTerminator.hpp"
@@ -1524,6 +1525,8 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
+  SlidingForwarding::initialize(heap_rs.region(), HeapRegion::GrainWords);
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -42,6 +42,7 @@
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/referenceProcessor.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/verifyOption.hpp"
 #include "gc/shared/weakProcessor.inline.hpp"
 #include "gc/shared/workerPolicy.hpp"
@@ -210,6 +211,8 @@ void G1FullCollector::collect() {
   // Don't add any more derived pointers during later phases
   deactivate_derived_pointers();
 
+  SlidingForwarding::begin();
+
   phase2_prepare_compaction();
 
   if (has_compaction_targets()) {
@@ -222,6 +225,8 @@ void G1FullCollector::collect() {
     log_info(gc, phases) ("No Regions selected for compaction. Skipping Phase 3: Adjust pointers and Phase 4: Compact heap");
   }
 
+  SlidingForwarding::end();
+
   phase5_reset_metadata();
 
   G1CollectedHeap::finish_codecache_marking_cycle();
@@ -394,7 +399,8 @@ uint G1FullCollector::truncate_parallel_cps() {
   return lowest_current;
 }
 
-void G1FullCollector::phase2c_prepare_serial_compaction() {
+template <bool ALT_FWD>
+void G1FullCollector::phase2c_prepare_serial_compaction_impl() {
   GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare serial compaction", scope()->timer());
   // At this point, we know that after parallel compaction there will be regions that
   // are partially compacted into. Thus, the last compaction region of all
@@ -419,7 +425,7 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
   serial_cp->initialize(start_hr);
 
   HeapWord* dense_prefix_top = compaction_top(start_hr);
-  G1SerialRePrepareClosure re_prepare(serial_cp, dense_prefix_top);
+  G1SerialRePrepareClosure<ALT_FWD> re_prepare(serial_cp, dense_prefix_top);
 
   for (uint i = start_serial + 1; i < _heap->max_reserved_regions(); i++) {
     if (is_compaction_target(i)) {
@@ -432,7 +438,16 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
   serial_cp->update();
 }
 
-void G1FullCollector::phase2d_prepare_humongous_compaction() {
+void G1FullCollector::phase2c_prepare_serial_compaction() {
+  if (UseAltGCForwarding) {
+    phase2c_prepare_serial_compaction_impl<true>();
+  } else {
+    phase2c_prepare_serial_compaction_impl<false>();
+  }
+}
+
+template <bool ALT_FWD>
+void G1FullCollector::phase2d_prepare_humongous_compaction_impl() {
   GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare humongous compaction", scope()->timer());
   G1FullGCCompactionPoint* serial_cp = serial_compaction_point();
   assert(serial_cp->has_regions(), "Sanity!" );
@@ -450,7 +465,7 @@ void G1FullCollector::phase2d_prepare_humongous_compaction() {
       region_index++;
       continue;
     } else if (hr->is_starts_humongous()) {
-      uint num_regions = humongous_cp->forward_humongous(hr);
+      uint num_regions = humongous_cp->forward_humongous<ALT_FWD>(hr);
       region_index += num_regions; // Skip over the continues humongous regions.
       continue;
     } else if (is_compaction_target(region_index)) {
@@ -461,6 +476,14 @@ void G1FullCollector::phase2d_prepare_humongous_compaction() {
   }
 }
 
+void G1FullCollector::phase2d_prepare_humongous_compaction() {
+  if (UseAltGCForwarding) {
+    phase2d_prepare_humongous_compaction_impl<true>();
+  } else {
+    phase2d_prepare_humongous_compaction_impl<false>();
+  }
+}
+
 void G1FullCollector::phase3_adjust_pointers() {
   // Adjust the pointers to reflect the new locations
   GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer());

src/hotspot/share/gc/g1/g1FullCollector.hpp

@@ -158,7 +158,11 @@ class G1FullCollector : StackObj {
 
   void phase2a_determine_worklists();
   bool phase2b_forward_oops();
+  template <bool ALT_FWD>
+  void phase2c_prepare_serial_compaction_impl();
   void phase2c_prepare_serial_compaction();
+  template <bool ALT_FWD>
+  void phase2d_prepare_humongous_compaction_impl();
   void phase2d_prepare_humongous_compaction();
 
   void phase3_adjust_pointers();

src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp

@@ -40,10 +40,11 @@
 #include "memory/iterator.inline.hpp"
 #include "runtime/atomic.hpp"
 
+template <bool ALT_FWD>
 class G1AdjustLiveClosure : public StackObj {
-  G1AdjustClosure* _adjust_closure;
+  G1AdjustClosure<ALT_FWD>* _adjust_closure;
 public:
-  G1AdjustLiveClosure(G1AdjustClosure* cl) :
+  G1AdjustLiveClosure(G1AdjustClosure<ALT_FWD>* cl) :
     _adjust_closure(cl) { }
 
   size_t apply(oop object) {
@@ -62,15 +63,25 @@ class G1AdjustRegionClosure : public HeapRegionClosure {
     _worker_id(worker_id) { }
 
   bool do_heap_region(HeapRegion* r) {
-    G1AdjustClosure cl(_collector);
+    if (UseAltGCForwarding) {
+      return do_heap_region_impl<true>(r);
+    } else {
+      return do_heap_region_impl<false>(r);
+    }
+  }
+
+private:
+  template <bool ALT_FWD>
+  bool do_heap_region_impl(HeapRegion* r) {
+    G1AdjustClosure<ALT_FWD> cl(_collector);
     if (r->is_humongous()) {
       // Special handling for humongous regions to get somewhat better
       // work distribution.
       oop obj = cast_to_oop(r->humongous_start_region()->bottom());
       obj->oop_iterate(&cl, MemRegion(r->bottom(), r->top()));
     } else if (!r->is_free()) {
       // Free regions do not contain objects to iterate. So skip them.
-      G1AdjustLiveClosure adjust(&cl);
+      G1AdjustLiveClosure<ALT_FWD> adjust(&cl);
       r->apply_to_marked_objects(_bitmap, &adjust);
     }
     return false;
@@ -81,31 +92,40 @@ G1FullGCAdjustTask::G1FullGCAdjustTask(G1FullCollector* collector) :
     G1FullGCTask("G1 Adjust", collector),
     _root_processor(G1CollectedHeap::heap(), collector->workers()),
     _weak_proc_task(collector->workers()),
-    _hrclaimer(collector->workers()),
-    _adjust(collector) {
+    _hrclaimer(collector->workers()) {
   ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
 }
 
-void G1FullGCAdjustTask::work(uint worker_id) {
+template <bool ALT_FWD>
+void G1FullGCAdjustTask::work_impl(uint worker_id) {
   Ticks start = Ticks::now();
   ResourceMark rm;
 
   // Adjust preserved marks first since they are not balanced.
   G1FullGCMarker* marker = collector()->marker(worker_id);
   marker->preserved_stack()->adjust_during_full_gc();
 
+  G1AdjustClosure<ALT_FWD> adjust(collector());
   {
     // Adjust the weak roots.
     AlwaysTrueClosure always_alive;
-    _weak_proc_task.work(worker_id, &always_alive, &_adjust);
+    _weak_proc_task.work(worker_id, &always_alive, &adjust);
   }
 
-  CLDToOopClosure adjust_cld(&_adjust, ClassLoaderData::_claim_stw_fullgc_adjust);
-  CodeBlobToOopClosure adjust_code(&_adjust, CodeBlobToOopClosure::FixRelocations);
-  _root_processor.process_all_roots(&_adjust, &adjust_cld, &adjust_code);
+  CLDToOopClosure adjust_cld(&adjust, ClassLoaderData::_claim_stw_fullgc_adjust);
+  CodeBlobToOopClosure adjust_code(&adjust, CodeBlobToOopClosure::FixRelocations);
+  _root_processor.process_all_roots(&adjust, &adjust_cld, &adjust_code);
 
   // Now adjust pointers region by region
   G1AdjustRegionClosure blk(collector(), worker_id);
   G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
   log_task("Adjust task", worker_id, start);
 }
+
+void G1FullGCAdjustTask::work(uint worker_id) {
+  if (UseAltGCForwarding) {
+    work_impl<true>(worker_id);
+  } else {
+    work_impl<false>(worker_id);
+  }
+}

src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp

@@ -38,8 +38,9 @@ class G1FullGCAdjustTask : public G1FullGCTask {
   G1RootProcessor          _root_processor;
   WeakProcessor::Task      _weak_proc_task;
   HeapRegionClaimer        _hrclaimer;
-  G1AdjustClosure          _adjust;
 
+  template <bool ALT_FWD>
+  void work_impl(uint worker_id);
 public:
   G1FullGCAdjustTask(G1FullCollector* collector);
   void work(uint worker_id);

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -30,19 +30,22 @@
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/ticks.hpp"
 
-void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
+template <bool ALT_FWD>
+void G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::clear_in_bitmap(oop obj) {
   assert(_bitmap->is_marked(obj), "Should only compact marked objects");
   _bitmap->clear(obj);
 }
 
-size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
+template <bool ALT_FWD>
+size_t G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::apply(oop obj) {
   size_t size = obj->size();
-  if (obj->is_forwarded()) {
-    G1FullGCCompactTask::copy_object_to_new_location(obj);
+  if (SlidingForwarding::is_forwarded(obj)) {
+    G1FullGCCompactTask::copy_object_to_new_location<ALT_FWD>(obj);
   }
 
   // Clear the mark for the compacted object to allow reuse of the
@@ -51,14 +54,15 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
   return size;
 }
 
+template <bool ALT_FWD>
 void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
-  assert(obj->is_forwarded(), "Sanity!");
-  assert(obj->forwardee() != obj, "Object must have a new location");
+  assert(SlidingForwarding::is_forwarded(obj), "Sanity!");
+  assert(SlidingForwarding::forwardee<ALT_FWD>(obj) != obj, "Object must have a new location");
 
   size_t size = obj->size();
   // Copy object and reinit its mark.
   HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
-  HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj));
   Copy::aligned_conjoint_words(obj_addr, destination, size);
 
   // There is no need to transform stack chunks - marking already did that.
@@ -77,8 +81,13 @@ void G1FullGCCompactTask::compact_region(HeapRegion* hr) {
     // showed that it was better overall to clear bit by bit, compared
     // to clearing the whole region at the end. This difference was
     // clearly seen for regions with few marks.
-    G1CompactRegionClosure compact(collector()->mark_bitmap());
-    hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+    if (UseAltGCForwarding) {
+      G1CompactRegionClosure<true> compact(collector()->mark_bitmap());
+      hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+    } else {
+      G1CompactRegionClosure<false> compact(collector()->mark_bitmap());
+      hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+    }
   }
 
   hr->reset_compacted_after_full_gc(_collector->compaction_top(hr));
@@ -104,28 +113,37 @@ void G1FullGCCompactTask::serial_compaction() {
   }
 }
 
-void G1FullGCCompactTask::humongous_compaction() {
-  GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer());
-
+template <bool ALT_FWD>
+void G1FullGCCompactTask::humongous_compaction_impl() {
   for (HeapRegion* hr : collector()->humongous_compaction_regions()) {
     assert(collector()->is_compaction_target(hr->hrm_index()), "Sanity");
-    compact_humongous_obj(hr);
+    compact_humongous_obj<ALT_FWD>(hr);
+  }
+}
+
+void G1FullGCCompactTask::humongous_compaction() {
+  GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer());
+  if (UseAltGCForwarding) {
+    humongous_compaction_impl<true>();
+  } else {
+    humongous_compaction_impl<false>();
   }
 }
 
+template <bool ALT_FWD>
 void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) {
   assert(src_hr->is_starts_humongous(), "Should be start region of the humongous object");
 
   oop obj = cast_to_oop(src_hr->bottom());
   size_t word_size = obj->size();
 
   uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
-  HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+  HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj));
 
   assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
   collector()->mark_bitmap()->clear(obj);
 
-  copy_object_to_new_location(obj);
+  copy_object_to_new_location<ALT_FWD>(obj);
 
   uint dest_start_idx = _g1h->addr_to_region(destination);
   // Update the metadata for the destination regions.

src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp

@@ -41,11 +41,16 @@ class G1FullGCCompactTask : public G1FullGCTask {
   G1CollectedHeap* _g1h;
 
   void compact_region(HeapRegion* hr);
+  template <bool ALT_FWD>
   void compact_humongous_obj(HeapRegion* hr);
   void free_non_overlapping_regions(uint src_start_idx, uint dest_start_idx, uint num_regions);
 
+  template <bool ALT_FWD>
   static void copy_object_to_new_location(oop obj);
 
+  template <bool ALT_FWD>
+  void humongous_compaction_impl();
+
 public:
   G1FullGCCompactTask(G1FullCollector* collector) :
     G1FullGCTask("G1 Compact Task", collector),
@@ -57,6 +62,7 @@ class G1FullGCCompactTask : public G1FullGCTask {
   void serial_compaction();
   void humongous_compaction();
 
+  template <bool ALT_FWD>
   class G1CompactRegionClosure : public StackObj {
     G1CMBitMap* _bitmap;
     void clear_in_bitmap(oop object);