8342382: Implement JEP 522: G1 GC: Improve Throughput by Reducing Synchronization

src/hotspot/share/gc/g1/g1CardTable.cpp

@@ -34,14 +34,18 @@ void G1CardTable::verify_region(MemRegion mr, CardValue val, bool val_equals) {
   }
   CardValue* start    = byte_for(mr.start());
   CardValue* end      = byte_for(mr.last());
+
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+  G1HeapRegion* r = g1h->heap_region_containing(mr.start());
+
+  assert(r == g1h->heap_region_containing(mr.last()), "MemRegion crosses region");
+
   bool failures = false;
   for (CardValue* curr = start; curr <= end; ++curr) {
     CardValue curr_val = *curr;
     bool failed = (val_equals) ? (curr_val != val) : (curr_val == val);
     if (failed) {
       if (!failures) {
-        G1CollectedHeap* g1h = G1CollectedHeap::heap();
-        G1HeapRegion* r = g1h->heap_region_containing(mr.start());
         log_error(gc, verify)("== CT verification failed: [" PTR_FORMAT "," PTR_FORMAT "] r: %d (%s) %sexpecting value: %d",
                               p2i(start), p2i(end), r->hrm_index(), r->get_short_type_str(),
                               (val_equals) ? "" : "not ", val);

src/hotspot/share/gc/g1/g1CardTable.inline.hpp

@@ -51,7 +51,7 @@ inline bool G1CardTable::mark_clean_as_from_remset(CardValue* card) {
 // mask   = 0xFF00FF00
 // result = 0xBBAABBAA
 inline size_t blend(size_t a, size_t b, size_t mask) {
-  return a ^ ((a ^ b) & mask);
+  return (a & ~mask) | (b & mask);
 }
 
 inline size_t G1CardTable::mark_clean_range_as_from_remset(size_t start_card_index, size_t num_cards) {

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -913,7 +913,7 @@ class G1CollectedHeap : public CollectedHeap {
   void safepoint_synchronize_begin() override;
   void safepoint_synchronize_end() override;
 
-  jlong synchronized_duration() const { return _safepoint_duration; }
+  jlong safepoint_duration() const { return _safepoint_duration; }
   jlong last_refinement_epoch_start() const { return _last_refinement_epoch_start; }
   void set_last_refinement_epoch_start(jlong epoch_start, jlong last_yield_duration);
   jlong yield_duration_in_refinement_epoch();

src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp

@@ -330,8 +330,8 @@ bool G1ConcurrentRefineWorkState::complete(bool concurrent, bool print_log) {
                           s->cards_not_parsable(),
                           s->cards_refer_to_cset(),
                           percent_of(s->cards_refer_to_cset(), s->cards_not_clean()),
-                          s->cards_still_refer_to_cset(),
-                          percent_of(s->cards_still_refer_to_cset(), s->cards_not_clean()),
+                          s->cards_already_refer_to_cset(),
+                          percent_of(s->cards_already_refer_to_cset(), s->cards_not_clean()),
                           s->cards_clean_again(),
                           s->cards_pending()
                          );

src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp

@@ -34,12 +34,12 @@
 #include "utilities/macros.hpp"
 
 // Forward decl
+class G1CardTableClaimTable;
 class G1CollectedHeap;
 class G1ConcurrentRefine;
 class G1ConcurrentRefineThread;
 class G1HeapRegion;
 class G1Policy;
-class G1CardTableClaimTable;
 class ThreadClosure;
 class WorkerTask;
 class WorkerThreads;

src/hotspot/share/gc/g1/g1ConcurrentRefineStats.cpp

@@ -32,7 +32,7 @@ G1ConcurrentRefineStats::G1ConcurrentRefineStats() :
   _cards_scanned(0),
   _cards_clean(0),
   _cards_not_parsable(0),
-  _cards_still_refer_to_cset(0),
+  _cards_already_refer_to_cset(0),
   _cards_refer_to_cset(0),
   _cards_clean_again(0),
   _refine_duration(0)
@@ -45,7 +45,7 @@ void G1ConcurrentRefineStats::add_atomic(G1ConcurrentRefineStats* other) {
   Atomic::add(&_cards_scanned, other->_cards_scanned, memory_order_relaxed);
   Atomic::add(&_cards_clean, other->_cards_clean, memory_order_relaxed);
   Atomic::add(&_cards_not_parsable, other->_cards_not_parsable, memory_order_relaxed);
-  Atomic::add(&_cards_still_refer_to_cset, other->_cards_still_refer_to_cset, memory_order_relaxed);
+  Atomic::add(&_cards_already_refer_to_cset, other->_cards_already_refer_to_cset, memory_order_relaxed);
   Atomic::add(&_cards_refer_to_cset, other->_cards_refer_to_cset, memory_order_relaxed);
   Atomic::add(&_cards_clean_again, other->_cards_clean_again, memory_order_relaxed);
 
@@ -60,7 +60,7 @@ G1ConcurrentRefineStats::operator+=(const G1ConcurrentRefineStats& other) {
   _cards_scanned += other._cards_scanned;
   _cards_clean += other._cards_clean;
   _cards_not_parsable += other._cards_not_parsable;
-  _cards_still_refer_to_cset += other._cards_still_refer_to_cset;
+  _cards_already_refer_to_cset += other._cards_already_refer_to_cset;
   _cards_refer_to_cset += other._cards_refer_to_cset;
   _cards_clean_again += other._cards_clean_again;
 
@@ -81,7 +81,7 @@ G1ConcurrentRefineStats::operator-=(const G1ConcurrentRefineStats& other) {
   _cards_scanned = saturated_sub(_cards_scanned, other._cards_scanned);
   _cards_clean = saturated_sub(_cards_clean, other._cards_clean);
   _cards_not_parsable = saturated_sub(_cards_not_parsable, other._cards_not_parsable);
-  _cards_still_refer_to_cset = saturated_sub(_cards_still_refer_to_cset, other._cards_still_refer_to_cset);
+  _cards_already_refer_to_cset = saturated_sub(_cards_already_refer_to_cset, other._cards_already_refer_to_cset);
   _cards_refer_to_cset = saturated_sub(_cards_refer_to_cset, other._cards_refer_to_cset);
   _cards_clean_again = saturated_sub(_cards_clean_again, other._cards_clean_again);
 

src/hotspot/share/gc/g1/g1ConcurrentRefineStats.hpp

@@ -40,8 +40,8 @@ class G1ConcurrentRefineStats : public CHeapObj<mtGC> {
   size_t _cards_scanned;              // Total number of cards scanned.
   size_t _cards_clean;                // Number of cards found clean.
   size_t _cards_not_parsable;         // Number of cards we could not parse and left unrefined.
-  size_t _cards_still_refer_to_cset;  // Number of cards marked still young.
-  size_t _cards_refer_to_cset;        // Number of dirty cards that contain a to-cset reference.
+  size_t _cards_already_refer_to_cset;// Number of cards marked found to be already young.
+  size_t _cards_refer_to_cset;        // Number of dirty cards that were recently found to contain a to-cset reference.
   size_t _cards_clean_again;          // Dirtied cards that were cleaned.
 
   jlong _refine_duration;             // Time spent during actual refinement.
@@ -62,14 +62,14 @@ class G1ConcurrentRefineStats : public CHeapObj<mtGC> {
   size_t cards_clean() const { return _cards_clean; }
   size_t cards_not_clean() const { return _cards_scanned - _cards_clean; }
   size_t cards_not_parsable() const { return _cards_not_parsable; }
-  size_t cards_still_refer_to_cset() const { return _cards_still_refer_to_cset; }
+  size_t cards_already_refer_to_cset() const { return _cards_already_refer_to_cset; }
   size_t cards_refer_to_cset() const { return _cards_refer_to_cset; }
   size_t cards_clean_again() const { return _cards_clean_again; }
   // Number of cards that were marked dirty and in need of refinement. This includes cards recently
   // found to refer to the collection set as they originally were dirty.
-  size_t cards_pending() const { return cards_not_clean() - _cards_still_refer_to_cset; }
+  size_t cards_pending() const { return cards_not_clean() - _cards_already_refer_to_cset; }
 
-  size_t cards_to_cset() const { return _cards_still_refer_to_cset + _cards_refer_to_cset; }
+  size_t cards_to_cset() const { return _cards_already_refer_to_cset + _cards_refer_to_cset; }
 
   void inc_sweep_time(jlong t) { _sweep_duration += t; }
   void inc_yield_duration(jlong t) { _yield_duration += t; }
@@ -78,7 +78,7 @@ class G1ConcurrentRefineStats : public CHeapObj<mtGC> {
   void inc_cards_scanned(size_t increment = 1) { _cards_scanned += increment; }
   void inc_cards_clean(size_t increment = 1) { _cards_clean += increment; }
   void inc_cards_not_parsable() { _cards_not_parsable++; }
-  void inc_cards_still_refer_to_cset() { _cards_still_refer_to_cset++; }
+  void inc_cards_already_refer_to_cset() { _cards_already_refer_to_cset++; }
   void inc_cards_refer_to_cset() { _cards_refer_to_cset++; }
   void inc_cards_clean_again() { _cards_clean_again++; }
 

src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp

@@ -169,7 +169,7 @@ void G1ConcurrentRefineThread::do_refinement() {
   jlong epoch_yield_duration = g1h->yield_duration_in_refinement_epoch();
   jlong next_epoch_start = os::elapsed_counter();
 
-  jlong synchronize_duration_at_sweep_start = g1h->synchronized_duration();
+  jlong synchronize_duration_at_sweep_start = g1h->safepoint_duration();
 
   // 4. Snapshot heap.
   state.snapshot_heap();
@@ -204,7 +204,7 @@ void G1ConcurrentRefineThread::do_refinement() {
   }
 
   if (!interrupted_by_gc) {
-    state.add_yield_duration(G1CollectedHeap::heap()->synchronized_duration() - synchronize_duration_at_sweep_start);
+    state.add_yield_duration(G1CollectedHeap::heap()->safepoint_duration() - synchronize_duration_at_sweep_start);
 
     state.complete(true);
 

src/hotspot/share/gc/g1/g1ConcurrentRefineWorkTask.cpp

@@ -60,7 +60,7 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
       _refine_stats.inc_cards_not_parsable();
     } else if (res == G1RemSet::AlreadyToCSet) {
       *dest_card = G1CardTable::g1_to_cset_card;
-      _refine_stats.inc_cards_still_refer_to_cset();
+      _refine_stats.inc_cards_already_refer_to_cset();
     } else if (res == G1RemSet::HasToCSetRef) {
       *dest_card = G1CardTable::g1_to_cset_card;
       _refine_stats.inc_cards_refer_to_cset();