8272773: Configurable card table card size

src/hotspot/share/gc/g1/g1CardTable.cpp

@@ -28,6 +28,11 @@
 #include "gc/shared/memset_with_concurrent_readers.hpp"
 #include "logging/log.hpp"
 
+uint G1CardTable::min_card_size(size_t region_size) {
+  int region_size_log = exact_log2_long((jlong)region_size);
+  return 1 << (region_size_log - LogOfMaxCardsPerRegion);
+}
+
 void G1CardTable::g1_mark_as_young(const MemRegion& mr) {
   CardValue *const first = byte_for(mr.start());
   CardValue *const last = byte_after(mr.last());

src/hotspot/share/gc/g1/g1CardTable.hpp

@@ -26,6 +26,7 @@
 #define SHARE_GC_G1_G1CARDTABLE_HPP
 
 #include "gc/g1/g1RegionToSpaceMapper.hpp"
+#include "gc/g1/g1CardSetContainers.hpp"
 #include "gc/shared/cardTable.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/macros.hpp"
@@ -75,6 +76,7 @@ class G1CardTable : public CardTable {
 
   static const size_t WordAllClean = SIZE_MAX;
   static const size_t WordAllDirty = 0;
+  static const size_t LogOfMaxCardsPerRegion = BitsPerByte * sizeof(G1CardSetArray::EntryDataType);
 
   STATIC_ASSERT(BitsPerByte == 8);
   static const size_t WordAlreadyScanned = (SIZE_MAX / 255) * g1_card_already_scanned;
@@ -86,6 +88,8 @@ class G1CardTable : public CardTable {
   static CardValue g1_young_card_val() { return g1_young_gen; }
   static CardValue g1_scanned_card_val() { return g1_card_already_scanned; }
 
+  static uint min_card_size(size_t region_size);
+
   void verify_g1_young_region(MemRegion mr) PRODUCT_RETURN;
   void g1_mark_as_young(const MemRegion& mr);
 

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1661,7 +1661,7 @@ jint G1CollectedHeap::initialize() {
 
   // The G1FromCardCache reserves card with value 0 as "invalid", so the heap must not
   // start within the first card.
-  guarantee(heap_rs.base() >= (char*)G1CardTable::card_size, "Java heap must not start within the first card.");
+  guarantee(heap_rs.base() >= (char*)(uintptr_t)(G1CardTable::card_size), "Java heap must not start within the first card.");
   G1FromCardCache::initialize(max_reserved_regions());
   // Also create a G1 rem set.
   _rem_set = new G1RemSet(this, _card_table, _hot_card_cache);

src/hotspot/share/gc/g1/heapRegion.cpp

@@ -91,6 +91,10 @@ void HeapRegion::setup_heap_region_size(size_t max_heap_size) {
   guarantee(GrainWords == 0, "we should only set it once");
   GrainWords = GrainBytes >> LogHeapWordSize;
 
+  // Initialize card size based on the region size.
+  // Maximum no. of cards per region is 2^16.
+  CardTable::initialize_card_size(G1CardTable::min_card_size(region_size));
+
   guarantee(CardsPerRegion == 0, "we should only set it once");
   CardsPerRegion = GrainBytes >> G1CardTable::card_shift;
 

src/hotspot/share/gc/parallel/objectStartArray.cpp

@@ -31,11 +31,21 @@
 #include "services/memTracker.hpp"
 #include "utilities/align.hpp"
 
+uint ObjectStartArray::block_shift = 0;
+uint ObjectStartArray::block_size = 0;
+uint ObjectStartArray::block_size_in_words = 0;
+
+void ObjectStartArray::initialize_block_size(uint card_shift) {
+  block_shift = card_shift;
+  block_size = 1 << block_shift;
+  block_size_in_words = block_size / sizeof(HeapWord);
+}
+
 void ObjectStartArray::initialize(MemRegion reserved_region) {
   // We're based on the assumption that we use the same
   // size blocks as the card table.
   assert((int)block_size == (int)CardTable::card_size, "Sanity");
-  assert((int)block_size <= 512, "block_size must be less than or equal to 512");
+  assert(block_size <= MaxBlockSize, "block_size must be less than or equal to " UINT32_FORMAT, MaxBlockSize);
 
   // Calculate how much space must be reserved
   _reserved_region = reserved_region;

src/hotspot/share/gc/parallel/objectStartArray.hpp

@@ -52,11 +52,16 @@ class ObjectStartArray : public CHeapObj<mtGC> {
     clean_block                  = -1
   };
 
-  enum BlockSizeConstants {
-    block_shift                  = 9,
-    block_size                   = 1 << block_shift,
-    block_size_in_words          = block_size / sizeof(HeapWord)
-  };
+  static uint block_shift;
+  static uint block_size;
+  static uint block_size_in_words;
+
+  // This maximum is derived from that we need an extra bit for possible
+  // offsets in the byte for backskip values (this is a hard limit)
+  static const uint MaxBlockSize = 1024;
+
+  // Initialize block size based on card size
+  static void initialize_block_size(uint card_shift);
 
  protected:
 

src/hotspot/share/gc/parallel/parallelArguments.cpp

@@ -97,6 +97,8 @@ static size_t default_gen_alignment() {
 }
 
 void ParallelArguments::initialize_alignments() {
+  // Initialize card size before initializing alignments
+  CardTable::initialize_card_size();
   SpaceAlignment = GenAlignment = default_gen_alignment();
   HeapAlignment = compute_heap_alignment();
 }

src/hotspot/share/gc/shared/blockOffsetTable.cpp

@@ -33,6 +33,18 @@
 #include "runtime/java.hpp"
 #include "services/memTracker.hpp"
 
+uint BOTConstants::LogN = 0;
+uint BOTConstants::LogN_words = 0;
+uint BOTConstants::N_bytes = 0;
+uint BOTConstants::N_words = 0;
+
+void BOTConstants::initialize_bot_size(uint card_shift) {
+  LogN =  card_shift;
+  LogN_words = LogN - LogHeapWordSize;
+  N_bytes = 1 << LogN;
+  N_words = 1 << LogN_words;
+}
+
 //////////////////////////////////////////////////////////////////////
 // BlockOffsetSharedArray
 //////////////////////////////////////////////////////////////////////

src/hotspot/share/gc/shared/blockOffsetTable.hpp

@@ -27,6 +27,7 @@
 
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/memset_with_concurrent_readers.hpp"
+#include "gc/shared/cardTable.hpp"
 #include "memory/allocation.hpp"
 #include "memory/memRegion.hpp"
 #include "memory/virtualspace.hpp"
@@ -49,16 +50,20 @@ class ContiguousSpace;
 
 class BOTConstants : public AllStatic {
 public:
-  static const uint LogN = 9;
-  static const uint LogN_words = LogN - LogHeapWordSize;
-  static const uint N_bytes = 1 << LogN;
-  static const uint N_words = 1 << LogN_words;
+  static uint LogN;
+  static uint LogN_words;
+  static uint N_bytes;
+  static uint N_words;
+
   // entries "e" of at least N_words mean "go back by Base^(e-N_words)."
   // All entries are less than "N_words + N_powers".
   static const uint LogBase = 4;
   static const uint Base = (1 << LogBase);
   static const uint N_powers = 14;
 
+  // Initialize bot size based on card size
+  static void initialize_bot_size(uint card_shift);
+
   static size_t power_to_cards_back(uint i) {
     return (size_t)1 << (LogBase * i);
   }
@@ -93,6 +98,7 @@ class BlockOffsetTable {
   BlockOffsetTable(HeapWord* bottom, HeapWord* end):
     _bottom(bottom), _end(end) {
     assert(_bottom <= _end, "arguments out of order");
+    assert(BOTConstants::N_bytes == CardTable::card_size, "sanity");
   }
 
   // Note that the committed size of the covered space may have changed,

src/hotspot/share/gc/shared/cardTable.cpp

@@ -26,13 +26,48 @@
 #include "gc/shared/cardTable.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/space.inline.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/parallel/objectStartArray.hpp"
 #include "logging/log.hpp"
 #include "memory/virtualspace.hpp"
 #include "runtime/java.hpp"
 #include "runtime/os.hpp"
+#include "runtime/globals_extension.hpp"
 #include "services/memTracker.hpp"
 #include "utilities/align.hpp"
 
+uint CardTable::card_shift = 0;
+uint CardTable::card_size = 0;
+uint CardTable::card_size_in_words = 0;
+
+void CardTable::initialize_card_size(uint min_card_size) {
+  assert(UseG1GC || UseParallelGC || UseSerialGC,
+         "Initialize card size should only be called by card based collectors.");
+
+  // Card size is the max. of minimum permissible value and GCCardSizeInBytes
+  card_size = MAX2(min_card_size, GCCardSizeInBytes);
+  card_shift = log2i_exact(card_size);
+  card_size_in_words = card_size / sizeof(HeapWord);
+
+  // Set blockOffsetTable size based on card table entry size
+  BOTConstants::initialize_bot_size(card_shift);
+
+#if INCLUDE_PARALLELGC
+  // Set ObjectStartArray block size based on card table entry size
+  ObjectStartArray::initialize_block_size(card_shift);
+#endif
+
+  if (GCCardSizeInBytes != card_size) {
+    FLAG_SET_ERGO(GCCardSizeInBytes, card_size);
+  }
+
+  log_info_p(gc, init)("CardTable entry size: " UINT32_FORMAT,  card_size);
+}
+
+void CardTable::initialize_card_size() {
+  initialize_card_size(CardSizeMin);
+}
+
 size_t CardTable::compute_byte_map_size() {
   assert(_guard_index == cards_required(_whole_heap.word_size()) - 1,
                                         "uninitialized, check declaration order");
@@ -41,6 +76,7 @@ size_t CardTable::compute_byte_map_size() {
   return align_up(_guard_index + 1, MAX2(_page_size, granularity));
 }
 
+
 CardTable::CardTable(MemRegion whole_heap) :
   _whole_heap(whole_heap),
   _guard_index(0),
@@ -56,8 +92,6 @@ CardTable::CardTable(MemRegion whole_heap) :
 {
   assert((uintptr_t(_whole_heap.start())  & (card_size - 1))  == 0, "heap must start at card boundary");
   assert((uintptr_t(_whole_heap.end()) & (card_size - 1))  == 0, "heap must end at card boundary");
-
-  assert(card_size <= 512, "card_size must be less than 512"); // why?
 }
 
 CardTable::~CardTable() {
@@ -428,7 +462,8 @@ MemRegion CardTable::dirty_card_range_after_reset(MemRegion mr,
 }
 
 uintx CardTable::ct_max_alignment_constraint() {
-  return card_size * os::vm_page_size();
+  // CardTable max alignment is computed with card_size_max
+  return CardSizeMax * os::vm_page_size();
 }
 
 void CardTable::verify_guard() {

src/hotspot/share/gc/shared/cardTable.hpp

@@ -228,17 +228,25 @@ class CardTable: public CHeapObj<mtGC> {
   MemRegion dirty_card_range_after_reset(MemRegion mr, bool reset,
                                          int reset_val);
 
-  // Constants
-  enum SomePublicConstants {
-    card_shift                  = 9,
-    card_size                   = 1 << card_shift,
-    card_size_in_words          = card_size / sizeof(HeapWord)
-  };
+  // CardTable entry size
+  static uint card_shift;
+  static uint card_size;
+  static uint card_size_in_words;
+
+  // min and max permissible card sizes
+  static const uint CardSizeMin = 512;
+  static const uint CardSizeMax = 1024;
 
   static constexpr CardValue clean_card_val()          { return clean_card; }
   static constexpr CardValue dirty_card_val()          { return dirty_card; }
   static intptr_t clean_card_row_val()   { return clean_card_row; }
 
+  // Initialize card size based on the min_card_size value
+  static void initialize_card_size(uint min_card_size);
+
+  //Initialize card size based on CardTable::card_size_min
+  static void initialize_card_size();
+
   // Card marking array base (adjusted for heap low boundary)
   // This would be the 0th element of _byte_map, if the heap started at 0x0.
   // But since the heap starts at some higher address, this points to somewhere

src/hotspot/share/gc/shared/gc_globals.hpp

@@ -692,9 +692,13 @@
   product(uintx, GCDrainStackTargetSize, 64,                                \
           "Number of entries we will try to leave on the stack "            \
           "during parallel gc")                                             \
-          range(0, max_juint)
-
-// end of GC_FLAGS
+          range(0, max_juint)                                               \
+                                                                            \
+  product(uint, GCCardSizeInBytes, 512,                                     \
+          "Card table entry size (in bytes) for card based collectors")     \
+          range(512, 1024)                                                  \
+          constraint(GCCardSizeInBytesConstraintFunc, AfterErgo)            \
+  // end of GC_FLAGS
 
 DECLARE_FLAGS(GC_FLAGS)
 

src/hotspot/share/gc/shared/genArguments.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/genArguments.hpp"
 #include "gc/shared/generation.hpp"
+#include "gc/shared/cardTable.hpp"
 #include "logging/log.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/java.hpp"
@@ -61,6 +62,8 @@ static size_t bound_minus_alignment(size_t desired_size,
 }
 
 void GenArguments::initialize_alignments() {
+  // Initialize card size before initializing alignments
+  CardTable::initialize_card_size();
   SpaceAlignment = GenAlignment = (size_t)Generation::GenGrain;
   HeapAlignment = compute_heap_alignment();
 }

src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp

@@ -424,3 +424,17 @@ JVMFlag::Error MaxMetaspaceSizeConstraintFunc(size_t value, bool verbose) {
   }
 }
 
+JVMFlag::Error GCCardSizeInBytesConstraintFunc(uint value, bool verbose) {
+  if(!(UseG1GC || UseParallelGC || UseSerialGC))
+    return JVMFlag::SUCCESS;
+  if (!is_power_of_2(value)) {
+    JVMFlag::printError(verbose,
+                        "GCCardSizeInBytes (" UINT32_FORMAT ") must be "
+                        "a power of 2\n",
+                        value);
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  } else {
+    return JVMFlag::SUCCESS;
+  }
+}
+

src/hotspot/share/gc/shared/jvmFlagConstraintsGC.hpp

@@ -66,7 +66,8 @@
  f(uintx,  TLABWasteIncrementConstraintFunc)                   \
  f(uintx,  SurvivorRatioConstraintFunc)                        \
  f(size_t, MetaspaceSizeConstraintFunc)                        \
- f(size_t, MaxMetaspaceSizeConstraintFunc)
+ f(size_t, MaxMetaspaceSizeConstraintFunc)                     \
+ f(uint, GCCardSizeInBytesConstraintFunc)
 
 SHARED_GC_CONSTRAINTS(DECLARE_CONSTRAINT)