8306122: [Lilliput] Refactor full GC forwarding

Reviewed-by: shade

Author: rkennke

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -76,6 +76,7 @@
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcBehaviours.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
@@ -86,8 +87,6 @@
 #include "gc/shared/locationPrinter.inline.hpp"
 #include "gc/shared/oopStorageParState.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
-#include "gc/shared/slidingForwarding.inline.hpp"
-#include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/referenceProcessor.inline.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
@@ -1522,8 +1521,6 @@ jint G1CollectedHeap::initialize() {
 
   initialize_reserved_region(heap_rs);
 
-  _forwarding = new SlidingForwarding(heap_rs.region(), HeapRegion::LogOfHRGrainBytes - LogHeapWordSize);
-
   // Create the barrier set for the entire reserved region.
   G1CardTable* ct = new G1CardTable(heap_rs.region());
   ct->initialize();
@@ -1673,6 +1670,8 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
+  GCForwarding::initialize(heap_rs.region(), HeapRegion::LogOfHRGrainBytes - LogHeapWordSize);
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -85,7 +85,6 @@ class MemoryPool;
 class nmethod;
 class ReferenceProcessor;
 class STWGCTimer;
-class SlidingForwarding;
 class WorkerThreads;
 
 typedef OverflowTaskQueue<ScannerTask, mtGC>           G1ScannerTasksQueue;
@@ -253,8 +252,6 @@ class G1CollectedHeap : public CollectedHeap {
   // Helper for monitoring and management support.
   G1MonitoringSupport* _monitoring_support;
 
-  SlidingForwarding* _forwarding;
-
   uint _num_humongous_objects; // Current amount of (all) humongous objects found in the heap.
   uint _num_humongous_reclaim_candidates; // Number of humongous object eager reclaim candidates.
 public:
@@ -268,10 +265,6 @@ class G1CollectedHeap : public CollectedHeap {
   void set_collection_set_candidates_stats(G1MonotonicArenaMemoryStats& stats);
   void set_young_gen_card_set_stats(const G1MonotonicArenaMemoryStats& stats);
 
-  SlidingForwarding* forwarding() const {
-    return _forwarding;
-  }
-
 private:
 
   G1HRPrinter _hr_printer;

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -39,10 +39,10 @@
 #include "gc/g1/g1OopClosures.hpp"
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
+#include "gc/shared/gcForwarding.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/referenceProcessor.hpp"
-#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/verifyOption.hpp"
 #include "gc/shared/weakProcessor.inline.hpp"
 #include "gc/shared/workerPolicy.hpp"
@@ -208,6 +208,7 @@ void G1FullCollector::collect() {
   // Don't add any more derived pointers during later phases
   deactivate_derived_pointers();
 
+  GCForwarding::begin();
   phase2_prepare_compaction();
 
   if (has_compaction_targets()) {
@@ -220,6 +221,8 @@ void G1FullCollector::collect() {
     log_info(gc, phases) ("No Regions selected for compaction. Skipping Phase 3: Adjust pointers and Phase 4: Compact heap");
   }
 
+  GCForwarding::end();
+
   phase5_reset_metadata();
 
   G1CollectedHeap::finish_codecache_marking_cycle();
@@ -331,8 +334,6 @@ void G1FullCollector::phase1_mark_live_objects() {
 void G1FullCollector::phase2_prepare_compaction() {
   GCTraceTime(Info, gc, phases) info("Phase 2: Prepare compaction", scope()->timer());
 
-  _heap->forwarding()->clear();
-
   phase2a_determine_worklists();
 
   if (!has_compaction_targets()) {
@@ -344,10 +345,9 @@ void G1FullCollector::phase2_prepare_compaction() {
   // Try to avoid OOM immediately after Full GC in case there are no free regions
   // left after determining the result locations (i.e. this phase). Prepare to
   // maximally compact the tail regions of the compaction queues serially.
-  // TODO: Disabled for now because it violates sliding-forwarding assumption.
-//  if (scope()->do_maximal_compaction() || !has_free_compaction_targets) {
-//    phase2c_prepare_serial_compaction();
-//  }
+  if (!UseCompactObjectHeaders && (scope()->do_maximal_compaction() || !has_free_compaction_targets)) {
+    phase2c_prepare_serial_compaction();
+  }
 }
 
 void G1FullCollector::phase2a_determine_worklists() {
@@ -366,61 +366,61 @@ bool G1FullCollector::phase2b_forward_oops() {
   return task.has_free_compaction_targets();
 }
 
-//uint G1FullCollector::truncate_parallel_cps() {
-//  uint lowest_current = (uint)-1;
-//  for (uint i = 0; i < workers(); i++) {
-//    G1FullGCCompactionPoint* cp = compaction_point(i);
-//    if (cp->has_regions()) {
-//      lowest_current = MIN2(lowest_current, cp->current_region()->hrm_index());
-//    }
-//  }
-
-//  for (uint i = 0; i < workers(); i++) {
-//    G1FullGCCompactionPoint* cp = compaction_point(i);
-//    if (cp->has_regions()) {
-//      cp->remove_at_or_above(lowest_current);
-//    }
-//  }
-//  return lowest_current;
-//}
-
-//void G1FullCollector::phase2c_prepare_serial_compaction() {
-//  GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare serial compaction", scope()->timer());
-//  // At this point, we know that after parallel compaction there will be regions that
-//  // are partially compacted into. Thus, the last compaction region of all
-//  // compaction queues still have space in them. We try to re-compact these regions
-//  // in serial to avoid a premature OOM when the mutator wants to allocate the first
-//  // eden region after gc.
-//
-//  // For maximum compaction, we need to re-prepare all objects above the lowest
-//  // region among the current regions for all thread compaction points. It may
-//  // happen that due to the uneven distribution of objects to parallel threads, holes
-//  // have been created as threads compact to different target regions between the
-//  // lowest and the highest region in the tails of the compaction points.
-//
-//  uint start_serial = truncate_parallel_cps();
-//  assert(start_serial < _heap->max_reserved_regions(), "Called on empty parallel compaction queues");
-//
-//  G1FullGCCompactionPoint* serial_cp = serial_compaction_point();
-//  assert(!serial_cp->is_initialized(), "sanity!");
-//
-//  HeapRegion* start_hr = _heap->region_at(start_serial);
-//  serial_cp->add(start_hr);
-//  serial_cp->initialize(start_hr);
-//
-//  HeapWord* dense_prefix_top = compaction_top(start_hr);
-//  G1SerialRePrepareClosure re_prepare(serial_cp, dense_prefix_top);
-//
-//  for (uint i = start_serial + 1; i < _heap->max_reserved_regions(); i++) {
-//    if (is_compaction_target(i)) {
-//      HeapRegion* current = _heap->region_at(i);
-//      set_compaction_top(current, current->bottom());
-//      serial_cp->add(current);
-//      current->apply_to_marked_objects(mark_bitmap(), &re_prepare);
-//    }
-//  }
-//  serial_cp->update();
-//}
+uint G1FullCollector::truncate_parallel_cps() {
+  uint lowest_current = (uint)-1;
+  for (uint i = 0; i < workers(); i++) {
+    G1FullGCCompactionPoint* cp = compaction_point(i);
+    if (cp->has_regions()) {
+      lowest_current = MIN2(lowest_current, cp->current_region()->hrm_index());
+    }
+  }
+
+  for (uint i = 0; i < workers(); i++) {
+    G1FullGCCompactionPoint* cp = compaction_point(i);
+    if (cp->has_regions()) {
+      cp->remove_at_or_above(lowest_current);
+    }
+  }
+  return lowest_current;
+}
+
+void G1FullCollector::phase2c_prepare_serial_compaction() {
+  GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare serial compaction", scope()->timer());
+  // At this point, we know that after parallel compaction there will be regions that
+  // are partially compacted into. Thus, the last compaction region of all
+  // compaction queues still have space in them. We try to re-compact these regions
+  // in serial to avoid a premature OOM when the mutator wants to allocate the first
+  // eden region after gc.
+
+  // For maximum compaction, we need to re-prepare all objects above the lowest
+  // region among the current regions for all thread compaction points. It may
+  // happen that due to the uneven distribution of objects to parallel threads, holes
+  // have been created as threads compact to different target regions between the
+  // lowest and the highest region in the tails of the compaction points.
+
+  uint start_serial = truncate_parallel_cps();
+  assert(start_serial < _heap->max_reserved_regions(), "Called on empty parallel compaction queues");
+
+  G1FullGCCompactionPoint* serial_cp = serial_compaction_point();
+  assert(!serial_cp->is_initialized(), "sanity!");
+
+  HeapRegion* start_hr = _heap->region_at(start_serial);
+  serial_cp->add(start_hr);
+  serial_cp->initialize(start_hr);
+
+  HeapWord* dense_prefix_top = compaction_top(start_hr);
+  G1SerialRePrepareClosure re_prepare(serial_cp, dense_prefix_top);
+
+  for (uint i = start_serial + 1; i < _heap->max_reserved_regions(); i++) {
+    if (is_compaction_target(i)) {
+      HeapRegion* current = _heap->region_at(i);
+      set_compaction_top(current, current->bottom());
+      serial_cp->add(current);
+      current->apply_to_marked_objects(mark_bitmap(), &re_prepare);
+    }
+  }
+  serial_cp->update();
+}
 
 void G1FullCollector::phase3_adjust_pointers() {
   // Adjust the pointers to reflect the new locations

src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp

@@ -94,8 +94,7 @@ void G1FullGCAdjustTask::work(uint worker_id) {
 
   // Adjust preserved marks first since they are not balanced.
   G1FullGCMarker* marker = collector()->marker(worker_id);
-  const SlidingForwarding* const forwarding = G1CollectedHeap::heap()->forwarding();
-  marker->preserved_stack()->adjust_during_full_gc(forwarding);
+  marker->preserved_stack()->adjust_during_full_gc();
 
   {
     // Adjust the weak roots.

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -29,8 +29,8 @@
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
-#include "gc/shared/slidingForwarding.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/ticks.hpp"
@@ -42,8 +42,8 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
 
 size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
   size_t size = obj->size();
-  if (obj->is_forwarded()) {
-    HeapWord* destination = cast_from_oop<HeapWord*>(_forwarding->forwardee(obj));
+  if (GCForwarding::is_forwarded(obj)) {
+    HeapWord* destination = cast_from_oop<HeapWord*>(GCForwarding::forwardee(obj));
 
     // copy object and reinit its mark
     HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);

src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp

@@ -33,7 +33,6 @@
 
 class G1CollectedHeap;
 class G1CMBitMap;
-class SlidingForwarding;
 class G1FullCollector;
 
 class G1FullGCCompactTask : public G1FullGCTask {
@@ -52,12 +51,9 @@ class G1FullGCCompactTask : public G1FullGCTask {
 
   class G1CompactRegionClosure : public StackObj {
     G1CMBitMap* _bitmap;
-    const SlidingForwarding* const _forwarding;
     void clear_in_bitmap(oop object);
   public:
-    G1CompactRegionClosure(G1CMBitMap* bitmap) :
-      _bitmap(bitmap),
-      _forwarding(G1CollectedHeap::heap()->forwarding()) { }
+    G1CompactRegionClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) { }
     size_t apply(oop object);
   };
 };

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -26,7 +26,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/shared/slidingForwarding.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
 
@@ -92,7 +92,7 @@ void G1FullGCCompactionPoint::switch_region() {
   initialize_values();
 }
 
-void G1FullGCCompactionPoint::forward(SlidingForwarding* const forwarding, oop object, size_t size) {
+void G1FullGCCompactionPoint::forward(oop object, size_t size) {
   assert(_current_region != NULL, "Must have been initialized");
 
   // Ensure the object fit in the current region.
@@ -102,10 +102,10 @@ void G1FullGCCompactionPoint::forward(SlidingForwarding* const forwarding, oop o
 
   // Store a forwarding pointer if the object should be moved.
   if (cast_from_oop<HeapWord*>(object) != _compaction_top) {
-    forwarding->forward_to(object, cast_to_oop(_compaction_top));
-    assert(object->is_forwarded(), "must be forwarded");
+    GCForwarding::forward_to(object, cast_to_oop(_compaction_top));
+    assert(GCForwarding::is_forwarded(object), "must be forwarded");
   } else {
-    assert(!object->is_forwarded(), "must not be forwarded");
+    assert(GCForwarding::is_not_forwarded(object), "must not be forwarded");
   }
 
   // Update compaction values.

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp

@@ -31,7 +31,6 @@
 
 class G1FullCollector;
 class HeapRegion;
-class SlidingForwarding;
 
 class G1FullGCCompactionPoint : public CHeapObj<mtGC> {
   G1FullCollector* _collector;
@@ -53,7 +52,7 @@ class G1FullGCCompactionPoint : public CHeapObj<mtGC> {
   bool is_initialized();
   void initialize(HeapRegion* hr);
   void update();
-  void forward(SlidingForwarding* const forwarding, oop object, size_t size);
+  void forward(oop object, size_t size);
   void add(HeapRegion* hr);
 
   void remove_at_or_above(uint bottom);

src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp

@@ -25,15 +25,13 @@
 #ifndef SHARE_GC_G1_G1FULLGCOOPCLOSURES_HPP
 #define SHARE_GC_G1_G1FULLGCOOPCLOSURES_HPP
 
-#include "gc/g1/g1CollectedHeap.hpp"
 #include "gc/shared/verifyOption.hpp"
 #include "memory/iterator.hpp"
 
 class G1CollectedHeap;
 class G1FullCollector;
 class G1CMBitMap;
 class G1FullGCMarker;
-class SlidingForwarding;
 
 // Below are closures used by the G1 Full GC.
 class G1IsAliveClosure : public BoolObjectClosure {
@@ -77,13 +75,10 @@ class G1MarkAndPushClosure : public ClaimMetadataVisitingOopIterateClosure {
 
 class G1AdjustClosure : public BasicOopIterateClosure {
   G1FullCollector* _collector;
-  const SlidingForwarding* const _forwarding;
 
   template <class T> inline void adjust_pointer(T* p);
 public:
-  G1AdjustClosure(G1FullCollector* collector) :
-    _collector(collector),
-    _forwarding(G1CollectedHeap::heap()->forwarding()) { }
+  G1AdjustClosure(G1FullCollector* collector) : _collector(collector) { }
   template <class T> void do_oop_work(T* p) { adjust_pointer(p); }
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -32,7 +32,7 @@
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/shared/slidingForwarding.inline.hpp"
+#include "gc/shared/gcForwarding.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"
@@ -66,8 +66,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
     return;
   }
 
-  if (obj->is_forwarded()) {
-    oop forwardee = _forwarding->forwardee(obj);
+  if (GCForwarding::is_forwarded(obj)) {
+    oop forwardee = GCForwarding::forwardee(obj);
     // Forwarded, just update.
     assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
     RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);