JDK-8263495: Gather liveness info in the mark phase of G1 full gc

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -35,6 +35,7 @@
 #include "gc/g1/g1FullGCScope.hpp"
 #include "gc/g1/g1OopClosures.hpp"
 #include "gc/g1/g1Policy.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/g1/g1StringDedup.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/preservedMarks.hpp"
@@ -121,8 +122,14 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap, bool explicit_gc, bool c
   _preserved_marks_set.init(_num_workers);
   _markers = NEW_C_HEAP_ARRAY(G1FullGCMarker*, _num_workers, mtGC);
   _compaction_points = NEW_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _num_workers, mtGC);
+
+  _live_stats = NEW_C_HEAP_ARRAY(G1RegionMarkStats, _heap->max_regions(), mtGC);
+  for (uint j = 0; j < heap->max_regions(); j++) {
+    _live_stats[j].clear();
+  }
+
   for (uint i = 0; i < _num_workers; i++) {
-    _markers[i] = new G1FullGCMarker(this, i, _preserved_marks_set.get(i));
+    _markers[i] = new G1FullGCMarker(this, i, _preserved_marks_set.get(i), _live_stats);
     _compaction_points[i] = new G1FullGCCompactionPoint();
     _oop_queue_set.register_queue(i, marker(i)->oop_stack());
     _array_queue_set.register_queue(i, marker(i)->objarray_stack());
@@ -137,6 +144,7 @@ G1FullCollector::~G1FullCollector() {
   }
   FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers);
   FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points);
+  FREE_C_HEAP_ARRAY(G1RegionMarkStats, _live_stats);
 }
 
 class PrepareRegionsClosure : public HeapRegionClosure {
@@ -237,6 +245,13 @@ void G1FullCollector::phase1_mark_live_objects() {
     // Do the actual marking.
     G1FullGCMarkTask marking_task(this);
     run_task(&marking_task);
+
+    // collect live bytes.
+    uint sum = 0;
+    for (uint j = 0; j < _heap->max_regions(); j++) {
+      sum += _live_stats[j]._live_words;
+    }
+    _heap->set_used(sum * HeapWordSize);
   }
 
   {

src/hotspot/share/gc/g1/g1FullCollector.hpp

@@ -67,6 +67,7 @@ class G1FullCollector : StackObj {
   G1FullGCCompactionPoint   _serial_compaction_point;
   G1IsAliveClosure          _is_alive;
   ReferenceProcessorIsAliveMutator _is_alive_mutator;
+  G1RegionMarkStats*        _live_stats;
 
   static uint calc_active_workers();
 

src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp

@@ -30,6 +30,7 @@
 #include "gc/g1/g1FullGCMarkTask.hpp"
 #include "gc/g1/g1FullGCOopClosures.inline.hpp"
 #include "gc/g1/g1FullGCReferenceProcessorExecutor.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/referenceProcessor.hpp"
 #include "memory/iterator.inline.hpp"
@@ -60,6 +61,8 @@ void G1FullGCMarkTask::work(uint worker_id) {
 
   // Mark stack is populated, now process and drain it.
   marker->complete_marking(collector()->oop_queue_set(), collector()->array_queue_set(), &_terminator);
+  // flush live bytes to regions
+  marker->flush_mark_region_cache();
 
   // This is the point where the entire marking should have completed.
   assert(marker->oop_stack()->is_empty(), "Marking should have completed");

src/hotspot/share/gc/g1/g1FullGCMarker.cpp

@@ -25,14 +25,16 @@
 #include "precompiled.hpp"
 #include "classfile/classLoaderData.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/shared/referenceProcessor.hpp"
 #include "gc/shared/taskTerminator.hpp"
 #include "gc/shared/verifyOption.hpp"
 #include "memory/iterator.inline.hpp"
 
 G1FullGCMarker::G1FullGCMarker(G1FullCollector* collector,
                                uint worker_id,
-                               PreservedMarks* preserved_stack) :
+                               PreservedMarks* preserved_stack,
+                               G1RegionMarkStats* mark_stats) :
     _collector(collector),
     _worker_id(worker_id),
     _bitmap(collector->mark_bitmap()),
@@ -42,7 +44,9 @@ G1FullGCMarker::G1FullGCMarker(G1FullCollector* collector,
     _mark_closure(worker_id, this, G1CollectedHeap::heap()->ref_processor_stw()),
     _verify_closure(VerifyOption_G1UseFullMarking),
     _stack_closure(this),
-    _cld_closure(mark_closure(), ClassLoaderData::_claim_strong) {
+    _cld_closure(mark_closure(), ClassLoaderData::_claim_strong),
+    _mark_region_cache(mark_stats, RegionMarkStatsCacheSize) {
+  _mark_region_cache.initialize();
   _oop_stack.initialize();
   _objarray_stack.initialize();
 }

src/hotspot/share/gc/g1/g1FullGCMarker.hpp

@@ -63,6 +63,12 @@ class G1FullGCMarker : public CHeapObj<mtGC> {
   G1FollowStackClosure _stack_closure;
   CLDToOopClosure      _cld_closure;
 
+
+  G1RegionMarkStatsCache _mark_region_cache;
+  // Number of entries in the per-task stats entry. This seems enough to have a very
+  // low cache miss rate.
+  static const uint RegionMarkStatsCacheSize = 1024;
+
   inline bool is_empty();
   inline bool pop_object(oop& obj);
   inline bool pop_objarray(ObjArrayTask& array);
@@ -74,7 +80,8 @@ class G1FullGCMarker : public CHeapObj<mtGC> {
   inline void follow_array(objArrayOop array);
   inline void follow_array_chunk(objArrayOop array, int index);
 public:
-  G1FullGCMarker(G1FullCollector* collector, uint worker_id, PreservedMarks* preserved_stack);
+  G1FullGCMarker(G1FullCollector* collector, uint worker_id,
+                 PreservedMarks* preserved_stack, G1RegionMarkStats* mark_stats);
   ~G1FullGCMarker();
 
   // Stack getters
@@ -96,6 +103,10 @@ class G1FullGCMarker : public CHeapObj<mtGC> {
   CLDToOopClosure*      cld_closure()   { return &_cld_closure; }
   G1MarkAndPushClosure* mark_closure()  { return &_mark_closure; }
   G1FollowStackClosure* stack_closure() { return &_stack_closure; }
+
+  void flush_mark_region_cache() {
+    _mark_region_cache.evict_all();
+  }
 };
 
 #endif // SHARE_GC_G1_G1FULLGCMARKER_HPP

src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp

@@ -64,6 +64,11 @@ inline bool G1FullGCMarker::mark_object(oop obj) {
       java_lang_String::is_instance_inlined(obj)) {
     G1StringDedup::enqueue_from_mark(obj, _worker_id);
   }
+
+  // Collect live words.
+  uint hr_index = G1CollectedHeap::heap()->addr_to_region(cast_from_oop<HeapWord*>(obj));
+  _mark_region_cache.add_live_words(hr_index, (size_t)obj->size());
+
   return true;
 }
 

src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp

@@ -27,6 +27,7 @@
 #include "gc/g1/g1FullCollector.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/g1FullGCOopClosures.inline.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "logging/logStream.hpp"
 #include "memory/iterator.inline.hpp"
 #include "oops/access.inline.hpp"

src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1CollectedHeap.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "memory/allocation.inline.hpp"
 #include "utilities/powerOfTwo.hpp"
@@ -44,6 +45,17 @@ G1RegionMarkStatsCache::~G1RegionMarkStatsCache() {
   FREE_C_HEAP_ARRAY(G1RegionMarkStatsCacheEntry, _cache);
 }
 
+// cache size is equal to or bigger than region size to intialize region_index
+void G1RegionMarkStatsCache::initialize() {
+  _cache_hits = 0;
+  _cache_misses = 0;
+
+  for (uint i = 0; i < _num_cache_entries; i++) {
+   _cache[i].clear();
+   _cache[i]._region_idx = i;
+  }
+}
+
 // Evict all remaining statistics, returning cache hits and misses.
 Pair<size_t, size_t> G1RegionMarkStatsCache::evict_all() {
   for (uint i = 0; i < _num_cache_entries; i++) {

src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp

@@ -102,6 +102,8 @@ class G1RegionMarkStatsCache {
 
   ~G1RegionMarkStatsCache();
 
+  void initialize();
+
   void add_live_words(uint region_idx, size_t live_words) {
     G1RegionMarkStatsCacheEntry* const cur = find_for_add(region_idx);
     cur->_stats._live_words += live_words;

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -27,6 +27,7 @@
 
 #include "gc/g1/g1BlockOffsetTable.hpp"
 #include "gc/g1/g1HeapRegionTraceType.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.hpp"
 #include "gc/g1/g1SurvRateGroup.hpp"
 #include "gc/g1/heapRegionTracer.hpp"
 #include "gc/g1/heapRegionType.hpp"