Skip to content
Permalink
Browse files
8275056: Virtualize G1CardSet containers over heap region
Reviewed-by: sjohanss, ayang
  • Loading branch information
Thomas Schatzl committed Nov 15, 2021
1 parent fe45835 commit 1830b8da9028af430ee4791f310b5fc9cb1ff37d
@@ -44,36 +44,54 @@

G1CardSet::CardSetPtr G1CardSet::FullCardSet = (G1CardSet::CardSetPtr)-1;

static uint default_log2_card_region_per_region() {
uint log2_card_region_per_heap_region = 0;

const uint card_container_limit = G1CardSetContainer::LogCardsPerRegionLimit;
if (card_container_limit < (uint)HeapRegion::LogCardsPerRegion) {
log2_card_region_per_heap_region = (uint)HeapRegion::LogCardsPerRegion - card_container_limit;
}

return log2_card_region_per_heap_region;
}

G1CardSetConfiguration::G1CardSetConfiguration() :
G1CardSetConfiguration(HeapRegion::LogCardsPerRegion, /* inline_ptr_bits_per_card */
G1RemSetArrayOfCardsEntries, /* num_cards_in_array */
(double)G1RemSetCoarsenHowlBitmapToHowlFullPercent / 100, /* cards_in_bitmap_threshold_percent */
G1RemSetHowlNumBuckets, /* num_buckets_in_howl */
(double)G1RemSetCoarsenHowlToFullPercent / 100, /* cards_in_howl_threshold_percent */
(uint)HeapRegion::CardsPerRegion) /* max_cards_in_cardset */
{ }
(uint)HeapRegion::CardsPerRegion, /* max_cards_in_cardset */
default_log2_card_region_per_region()) /* log2_card_region_per_region */
{
assert((_log2_card_region_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion,
"inconsistent heap region virtualization setup");
}

G1CardSetConfiguration::G1CardSetConfiguration(uint num_cards_in_array,
double cards_in_bitmap_threshold_percent,
uint max_buckets_in_howl,
double cards_in_howl_threshold_percent,
uint max_cards_in_card_set) :
uint max_cards_in_card_set,
uint log2_card_region_per_region) :
G1CardSetConfiguration(log2i_exact(max_cards_in_card_set), /* inline_ptr_bits_per_card */
num_cards_in_array, /* num_cards_in_array */
cards_in_bitmap_threshold_percent, /* cards_in_bitmap_threshold_percent */
G1CardSetHowl::num_buckets(max_cards_in_card_set, /* num_buckets_in_howl */
num_cards_in_array,
max_buckets_in_howl),
cards_in_howl_threshold_percent, /* cards_in_howl_threshold_percent */
max_cards_in_card_set) /* max_cards_in_cardset */
{ }
max_cards_in_card_set, /* max_cards_in_cardset */
log2_card_region_per_region)
{ }

G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
uint num_cards_in_array,
double cards_in_bitmap_threshold_percent,
uint num_buckets_in_howl,
double cards_in_howl_threshold_percent,
uint max_cards_in_card_set) :
uint max_cards_in_card_set,
uint log2_card_region_per_heap_region) :
_inline_ptr_bits_per_card(inline_ptr_bits_per_card),
_num_cards_in_array(num_cards_in_array),
_num_buckets_in_howl(num_buckets_in_howl),
@@ -82,13 +100,14 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
_num_cards_in_howl_bitmap(G1CardSetHowl::bitmap_size(_max_cards_in_card_set, _num_buckets_in_howl)),
_cards_in_howl_bitmap_threshold(_num_cards_in_howl_bitmap * cards_in_bitmap_threshold_percent),
_log2_num_cards_in_howl_bitmap(log2i_exact(_num_cards_in_howl_bitmap)),
_bitmap_hash_mask(~(~(0) << _log2_num_cards_in_howl_bitmap)) {
_bitmap_hash_mask(~(~(0) << _log2_num_cards_in_howl_bitmap)),
_log2_card_region_per_heap_region(log2_card_region_per_heap_region),
_log2_cards_per_card_region(log2i_exact(_max_cards_in_card_set) - _log2_card_region_per_heap_region) {

assert(is_power_of_2(_max_cards_in_card_set),
"max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set);

init_card_set_alloc_options();

log_configuration();
}

@@ -109,11 +128,14 @@ void G1CardSetConfiguration::log_configuration() {
"InlinePtr #elems %u size %zu "
"Array Of Cards #elems %u size %zu "
"Howl #buckets %u coarsen threshold %u "
"Howl Bitmap #elems %u size %zu coarsen threshold %u",
"Howl Bitmap #elems %u size %zu coarsen threshold %u "
"Card regions per heap region %u cards per card region %u",
num_cards_in_inline_ptr(), sizeof(void*),
num_cards_in_array(), G1CardSetArray::size_in_bytes(num_cards_in_array()),
num_buckets_in_howl(), cards_in_howl_threshold(),
num_cards_in_howl_bitmap(), G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), cards_in_howl_bitmap_threshold());
num_cards_in_howl_bitmap(), G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), cards_in_howl_bitmap_threshold(),
(uint)1 << log2_card_region_per_heap_region(),
(uint)1 << log2_cards_per_card_region());
}

uint G1CardSetConfiguration::num_cards_in_inline_ptr() const {
@@ -828,8 +850,26 @@ class G1ContainerCards {
}
};

template <typename Closure, template <typename> class CardOrRanges>
class G1CardSetIterateCardsIterator : public G1CardSet::G1CardSetPtrIterator {
G1CardSet* _card_set;
Closure& _iter;

public:

G1CardSetIterateCardsIterator(G1CardSet* card_set,
Closure& iter) :
_card_set(card_set),
_iter(iter) { }

void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
CardOrRanges<Closure> cl(_iter, region_idx);
_card_set->iterate_cards_or_ranges_in_container(card_set, cl);
}
};

void G1CardSet::iterate_cards(G1CardSetCardIterator& iter) {
G1CardSetMergeCardIterator<G1CardSetCardIterator, G1ContainerCards> cl(this, iter);
G1CardSetIterateCardsIterator<G1CardSetCardIterator, G1ContainerCards> cl(this, iter);
iterate_containers(&cl);
}

@@ -58,6 +58,8 @@ class G1CardSetConfiguration {
uint _cards_in_howl_bitmap_threshold;
uint _log2_num_cards_in_howl_bitmap;
size_t _bitmap_hash_mask;
uint _log2_card_region_per_heap_region;
uint _log2_cards_per_card_region;

G1CardSetAllocOptions* _card_set_alloc_options;

@@ -66,7 +68,8 @@ class G1CardSetConfiguration {
double cards_in_bitmap_threshold_percent,
uint num_buckets_in_howl,
double cards_in_howl_threshold_percent,
uint max_cards_in_card_set);
uint max_cards_in_card_set,
uint log2_card_region_per_heap_region);
void init_card_set_alloc_options();

void log_configuration();
@@ -75,12 +78,13 @@ class G1CardSetConfiguration {
// Initialize card set configuration from globals.
G1CardSetConfiguration();
// Initialize card set configuration from parameters.
// Only for test
// Testing only.
G1CardSetConfiguration(uint num_cards_in_array,
double cards_in_bitmap_threshold_percent,
uint max_buckets_in_howl,
double cards_in_howl_threshold_percent,
uint max_cards_in_card_set);
uint max_cards_in_cardset,
uint log2_card_region_per_region);

~G1CardSetConfiguration();

@@ -115,6 +119,20 @@ class G1CardSetConfiguration {
// with more entries per region are coarsened to Full.
uint max_cards_in_region() const { return _max_cards_in_card_set; }

// Heap region virtualization: there are some limitations to how many cards the
// containers can cover to save memory for the common case. Heap region virtualization
// allows to use multiple entries in the G1CardSet hash table per area covered
// by the remembered set (e.g. heap region); each such entry is called "card_region".
//
// The next two members give information about how many card regions are there
// per area (heap region) and how many cards each card region has.

// The log2 of the amount of card regions per heap region configured.
uint log2_card_region_per_heap_region() const { return _log2_card_region_per_heap_region; }
// The log2 of the number of cards per card region. This is calculated from max_cards_in_region()
// and above.
uint log2_cards_per_card_region() const { return _log2_cards_per_card_region; }

// Memory object types configuration
// Number of distinctly sized memory objects on the card set heap.
// Currently contains CHT-Nodes, ArrayOfCards, BitMaps, Howl
@@ -171,9 +189,6 @@ class G1CardSet : public CHeapObj<mtGCCardSet> {
friend class G1CardSetTest;
friend class G1CardSetMtTestTask;

template <typename Closure, template <typename> class CardorRanges>
friend class G1CardSetMergeCardIterator;

friend class G1TransferCard;

friend class G1ReleaseCardsets;
@@ -278,23 +293,6 @@ class G1CardSet : public CHeapObj<mtGCCardSet> {
template <class CardVisitor>
void iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& found);

// Iterate over the container, calling a method on every card or card range contained
// in the card container.
// For every container, first calls
//
// void start_iterate(uint tag, uint region_idx);
//
// Then for every card or card range it calls
//
// void do_card(uint card_idx);
// void do_card_range(uint card_idx, uint length);
//
// where card_idx is the card index within that region_idx passed before in
// start_iterate().
//
template <class CardOrRangeVisitor>
void iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& found);

uint card_set_type_to_mem_object_type(uintptr_t type) const;
uint8_t* allocate_mem_object(uintptr_t type);
void free_mem_object(CardSetPtr card_set);
@@ -340,7 +338,23 @@ class G1CardSet : public CHeapObj<mtGCCardSet> {

void print(outputStream* os);

// Various iterators - should be made inlineable somehow.
// Iterate over the container, calling a method on every card or card range contained
// in the card container.
// For every container, first calls
//
// void start_iterate(uint tag, uint region_idx);
//
// Then for every card or card range it calls
//
// void do_card(uint card_idx);
// void do_card_range(uint card_idx, uint length);
//
// where card_idx is the card index within that region_idx passed before in
// start_iterate().
//
template <class CardOrRangeVisitor>
void iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& found);

class G1CardSetPtrIterator {
public:
virtual void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) = 0;
@@ -354,11 +368,6 @@ class G1CardSet : public CHeapObj<mtGCCardSet> {
};

void iterate_cards(G1CardSetCardIterator& iter);

// Iterate all cards for card set merging. Must be a CardOrRangeVisitor as
// explained above.
template <class CardOrRangeVisitor>
void iterate_for_merge(CardOrRangeVisitor& cl);
};

class G1CardSetHashTableValue {
@@ -80,46 +80,4 @@ inline void G1CardSet::iterate_cards_or_ranges_in_container(CardSetPtr const car
ShouldNotReachHere();
}

template <typename Closure>
class G1ContainerCardsOrRanges {
Closure& _iter;
uint _region_idx;

public:
G1ContainerCardsOrRanges(Closure& iter, uint region_idx) : _iter(iter), _region_idx(region_idx) { }

bool start_iterate(uint tag) {
return _iter.start_iterate(tag, _region_idx);
}

void operator()(uint card_idx) {
_iter.do_card(card_idx);
}

void operator()(uint card_idx, uint length) {
_iter.do_card_range(card_idx, length);
}
};

template <typename Closure, template <typename> class CardOrRanges>
class G1CardSetMergeCardIterator : public G1CardSet::G1CardSetPtrIterator {
G1CardSet* _card_set;
Closure& _iter;

public:

G1CardSetMergeCardIterator(G1CardSet* card_set, Closure& iter) : _card_set(card_set), _iter(iter) { }

void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
CardOrRanges<Closure> cl(_iter, region_idx);
_card_set->iterate_cards_or_ranges_in_container(card_set, cl);
}
};

template <class CardOrRangeVisitor>
inline void G1CardSet::iterate_for_merge(CardOrRangeVisitor& cl) {
G1CardSetMergeCardIterator<CardOrRangeVisitor, G1ContainerCardsOrRanges> cl2(this, cl);
iterate_containers(&cl2, true /* at_safepoint */);
}

#endif // SHARE_GC_G1_G1CARDSET_INLINE_HPP
@@ -106,14 +106,14 @@ class G1RemSetScanState : public CHeapObj<mtGC> {
// within a region to claim. Dependent on the region size as proxy for the heap
// size, we limit the total number of chunks to limit memory usage and maintenance
// effort of that table vs. granularity of distributing scanning work.
// Testing showed that 8 for 1M/2M region, 16 for 4M/8M regions, 32 for 16/32M regions
// seems to be such a good trade-off.
// Testing showed that 8 for 1M/2M region, 16 for 4M/8M regions, 32 for 16/32M regions,
// and so on seems to be such a good trade-off.
static uint get_chunks_per_region(uint log_region_size) {
// Limit the expected input values to current known possible values of the
// (log) region size. Adjust as necessary after testing if changing the permissible
// values for region size.
assert(log_region_size >= 20 && log_region_size <= 25,
"expected value in [20,25], but got %u", log_region_size);
assert(log_region_size >= 20 && log_region_size <= 29,
"expected value in [20,29], but got %u", log_region_size);
return 1u << (log_region_size / 2 - 7);
}

@@ -257,7 +257,7 @@
\
product(size_t, G1HeapRegionSize, 0, \
"Size of the G1 regions.") \
range(0, 32*M) \
range(0, NOT_LP64(32*M) LP64_ONLY(512*M)) \
constraint(G1HeapRegionSizeConstraintFunc,AfterMemoryInit) \
\
product(uint, G1ConcRefinementThreads, 0, \
@@ -65,8 +65,9 @@ void HeapRegion::setup_heap_region_size(size_t max_heap_size) {
size_t region_size = G1HeapRegionSize;
// G1HeapRegionSize = 0 means decide ergonomically.
if (region_size == 0) {
region_size = MAX2(max_heap_size / HeapRegionBounds::target_number(),
HeapRegionBounds::min_size());
region_size = clamp(max_heap_size / HeapRegionBounds::target_number(),
HeapRegionBounds::min_size(),
HeapRegionBounds::max_ergonomics_size());
}

// Make sure region size is a power of 2. Rounding up since this
@@ -34,19 +34,22 @@ class HeapRegionBounds : public AllStatic {
// heaps a bit more efficiently.
static const size_t MIN_REGION_SIZE = 1024 * 1024;

// Maximum region size determined ergonomically.
static const size_t MAX_ERGONOMICS_SIZE = 32 * 1024 * 1024;
// Maximum region size; we don't go higher than that. There's a good
// reason for having an upper bound. We don't want regions to get too
// large, otherwise cleanup's effectiveness would decrease as there
// will be fewer opportunities to find totally empty regions after
// marking.
static const size_t MAX_REGION_SIZE = 32 * 1024 * 1024;
static const size_t MAX_REGION_SIZE = 512 * 1024 * 1024;

// The automatic region size calculation will try to have around this
// many regions in the heap.
static const size_t TARGET_REGION_NUMBER = 2048;

public:
static inline size_t min_size();
static inline size_t max_ergonomics_size();
static inline size_t max_size();
static inline size_t target_number();
};
@@ -31,6 +31,10 @@ size_t HeapRegionBounds::min_size() {
return MIN_REGION_SIZE;
}

size_t HeapRegionBounds::max_ergonomics_size() {
return MAX_ERGONOMICS_SIZE;
}

size_t HeapRegionBounds::max_size() {
return MAX_REGION_SIZE;
}
@@ -26,6 +26,7 @@

#include "precompiled.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/heapRegionManager.inline.hpp"

1 comment on commit 1830b8d

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on 1830b8d Nov 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.