8275055: Improve HeapRegionRemSet::split_card()

Reviewed-by: sjohanss, ayang

src/hotspot/share/gc/g1/g1CardSetContainers.cpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// The only limitation is from the G1CardSetArray.
+uint G1CardSetContainer::LogCardsPerRegionLimit = sizeof(G1CardSetArray::EntryDataType) * BitsPerByte;

src/hotspot/share/gc/g1/g1CardSetContainers.hpp

@@ -177,6 +177,9 @@ class G1CardSetContainer {
   void set_next(G1CardSetContainer* next) {
     _next = next;
   }
+
+  // Log of largest card index that can be stored in any G1CardSetContainer
+  static uint LogCardsPerRegionLimit;
 };
 
 class G1CardSetArray : public G1CardSetContainer {

src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp

@@ -27,6 +27,7 @@
 
 #include "gc/g1/g1CardSetContainers.hpp"
 #include "gc/g1/g1GCPhaseTimes.hpp"
+#include "utilities/globalDefinitions.hpp"
 
 inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
   assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx);

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1671,6 +1671,8 @@ jint G1CollectedHeap::initialize() {
   guarantee(HeapRegion::CardsPerRegion < max_cards_per_region,
             "too many cards per region");
 
+  HeapRegionRemSet::initialize(_reserved);
+
   FreeRegionList::set_unrealistically_long_length(max_regions() + 1);
 
   _bot = new G1BlockOffsetTable(reserved(), bot_storage);

src/hotspot/share/gc/g1/heapRegionRemSet.cpp

@@ -35,16 +35,47 @@
 #include "oops/oop.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/globals_extension.hpp"
+#include "runtime/java.hpp"
 #include "utilities/bitMap.inline.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/formatBuffer.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/growableArray.hpp"
 #include "utilities/powerOfTwo.hpp"
 
+uint HeapRegionRemSet::_split_card_shift = 0;
+size_t HeapRegionRemSet::_split_card_mask = 0;
+HeapWord* HeapRegionRemSet::_heap_base_address = nullptr;
+
 const char* HeapRegionRemSet::_state_strings[] =  {"Untracked", "Updating", "Complete"};
 const char* HeapRegionRemSet::_short_state_strings[] =  {"UNTRA", "UPDAT", "CMPLT"};
 
+void HeapRegionRemSet::initialize(MemRegion reserved) {
+  const uint BitsInUint = sizeof(uint) * BitsPerByte;
+  const uint CardBitsWithinCardRegion = MIN2((uint)HeapRegion::LogCardsPerRegion, G1CardSetContainer::LogCardsPerRegionLimit);
+
+  // Check if the number of cards within a region fits an uint.
+  if (CardBitsWithinCardRegion > BitsInUint) {
+    vm_exit_during_initialization("Can not represent all cards in a card region within uint.");
+  }
+
+  _split_card_shift = CardBitsWithinCardRegion + CardTable::card_shift;
+  _split_card_mask = ((size_t)1 << _split_card_shift) - 1;
+
+  // Check if the card region/region within cards combination can cover the heap.
+  const uint HeapSizeBits = log2i_exact(round_up_power_of_2(reserved.byte_size()));
+  if (HeapSizeBits > (BitsInUint + _split_card_shift)) {
+    FormatBuffer<> fmt("Can not represent all cards in the heap with card region/card within region. "
+                       "Heap %zuB (%u bits) Remembered set covers %u bits.",
+                       reserved.byte_size(),
+                       HeapSizeBits,
+                       BitsInUint + _split_card_shift);
+    vm_exit_during_initialization(fmt, "Decrease heap size.");
+  }
+
+  _heap_base_address = reserved.start();
+}
+
 HeapRegionRemSet::HeapRegionRemSet(HeapRegion* hr,
                                    G1CardSetConfiguration* config) :
   _m(Mutex::service - 1, FormatBuffer<128>("HeapRegionRemSet#%u_lock", hr->hrm_index())),

src/hotspot/share/gc/g1/heapRegionRemSet.hpp

@@ -50,6 +50,17 @@ class HeapRegionRemSet : public CHeapObj<mtGC> {
 
   HeapRegion* _hr;
 
+  // When splitting addresses into region and card within that region, the logical
+  // shift value to get the region.
+  static uint _split_card_shift;
+  // When splitting addresses into region and card within that region, the mask
+  // to get the offset within the region.
+  static size_t _split_card_mask;
+  // Cached value of heap base address.
+  static HeapWord* _heap_base_address;
+
+  // Split the given address into region of that card and the card within that
+  // region.
   inline void split_card(OopOrNarrowOopStar from, uint& card_region, uint& card_within_region) const;
   void clear_fcc();
 
@@ -78,6 +89,8 @@ class HeapRegionRemSet : public CHeapObj<mtGC> {
     return _card_set.occupied();
   }
 
+  static void initialize(MemRegion reserved);
+
   // Coarsening statistics since VM start.
   static G1CardSetCoarsenStats coarsen_stats() { return G1CardSet::coarsen_stats(); }
 

src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp

@@ -62,9 +62,10 @@ inline void HeapRegionRemSet::iterate_for_merge(CardOrRangeVisitor& cl) {
 }
 
 void HeapRegionRemSet::split_card(OopOrNarrowOopStar from, uint& card_region, uint& card_within_region) const {
-  HeapRegion* hr = G1CollectedHeap::heap()->heap_region_containing(from);
-  card_region = hr->hrm_index();
-  card_within_region = (uint)(pointer_delta((HeapWord*)from, hr->bottom()) >> (CardTable::card_shift - LogHeapWordSize));
+  size_t offset = pointer_delta(from, _heap_base_address, 1);
+  card_region = (uint)(offset >> _split_card_shift);
+  card_within_region = (uint)((offset & _split_card_mask) >> CardTable::card_shift);
+  assert(card_within_region < ((uint)1 << G1CardSetContainer::LogCardsPerRegionLimit), "must be");
 }
 
 void HeapRegionRemSet::add_reference(OopOrNarrowOopStar from, uint tid) {