Skip to content
Permalink
Browse files
8276299: G1: Unify the wording buffer/node/element in G1SegmentedArra…
…yXxx, G1CardSetXxx and related classes

Reviewed-by: tschatzl, ayang, iwalulya
  • Loading branch information
Hamlin Li committed Dec 2, 2021
1 parent 51d6d7a commit 67745fa749e5075b37aeca7db9d446bc287da835
Showing 14 changed files with 367 additions and 365 deletions.
@@ -143,7 +143,7 @@ void G1Arguments::initialize_card_set_configuration() {
// Round to next 8 byte boundary for array to maximize space usage.
size_t const cur_size = G1CardSetArray::size_in_bytes(G1RemSetArrayOfCardsEntries);
FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries,
G1RemSetArrayOfCardsEntries + (uint)(align_up(cur_size, G1CardSetAllocOptions::BufferAlignment) - cur_size) / sizeof(G1CardSetArray::EntryDataType));
G1RemSetArrayOfCardsEntries + (uint)(align_up(cur_size, G1CardSetAllocOptions::SlotAlignment) - cur_size) / sizeof(G1CardSetArray::EntryDataType));

// Howl card set container globals.
if (FLAG_IS_DEFAULT(G1RemSetHowlNumBuckets)) {
@@ -125,10 +125,10 @@ void G1CardSetConfiguration::init_card_set_alloc_options() {

void G1CardSetConfiguration::log_configuration() {
log_debug_p(gc, remset)("Card Set container configuration: "
"InlinePtr #elems %u size %zu "
"Array Of Cards #elems %u size %zu "
"InlinePtr #cards %u size %zu "
"Array Of Cards #cards %u size %zu "
"Howl #buckets %u coarsen threshold %u "
"Howl Bitmap #elems %u size %zu coarsen threshold %u "
"Howl Bitmap #cards %u size %zu coarsen threshold %u "
"Card regions per heap region %u cards per card region %u",
max_cards_in_inline_ptr(), sizeof(void*),
max_cards_in_array(), G1CardSetArray::size_in_bytes(max_cards_in_array()),
@@ -199,8 +199,8 @@ void G1CardSetCoarsenStats::print_on(outputStream* out) {
class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
using CardSetPtr = G1CardSet::CardSetPtr;

// Did we insert at least one element in the table?
bool volatile _inserted_elem;
// Did we insert at least one card in the table?
bool volatile _inserted_card;

G1CardSetMemoryManager* _mm;
CardSetHash _table;
@@ -247,7 +247,7 @@ class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {

G1CardSetHashTable(G1CardSetMemoryManager* mm,
size_t initial_log_table_size = InitialLogTableSize) :
_inserted_elem(false),
_inserted_card(false),
_mm(mm),
_table(mm, initial_log_table_size) {
}
@@ -267,10 +267,10 @@ class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr());
bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow);

if (!_inserted_elem && inserted) {
if (!_inserted_card && inserted) {
// It does not matter to us who is setting the flag so a regular atomic store
// is sufficient.
Atomic::store(&_inserted_elem, true);
Atomic::store(&_inserted_card, true);
}

return found.value();
@@ -295,9 +295,9 @@ class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
}

void reset() {
if (Atomic::load(&_inserted_elem)) {
if (Atomic::load(&_inserted_card)) {
_table.unsafe_reset(InitialLogTableSize);
Atomic::store(&_inserted_elem, false);
Atomic::store(&_inserted_card, false);
}
}

@@ -630,7 +630,7 @@ void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,

// Need to correct for that the Full remembered set occupies more cards than the
// bitmap before.
// We add 1 element less because the values will be incremented
// We add 1 card less because the values will be incremented
// in G1CardSet::add_card for the current addition or where already incremented in
// G1CardSet::add_to_howl after coarsening.
diff -= 1;
@@ -95,7 +95,7 @@ class G1CardSetInlinePtr : public StackObj {
return result;
}

uint find(uint const card_idx, uint const bits_per_card, uint start_at, uint num_elems);
uint find(uint const card_idx, uint const bits_per_card, uint start_at, uint num_cards);

public:
G1CardSetInlinePtr() : _value_addr(nullptr), _value((CardSetPtr)G1CardSet::CardSetInlinePtr) { }
@@ -218,7 +218,7 @@ class G1CardSetArray : public G1CardSetContainer {
}

public:
G1CardSetArray(uint const card_in_region, EntryCountType num_elems);
G1CardSetArray(uint const card_in_region, EntryCountType num_cards);

G1AddCardResult add(uint card_idx);

@@ -50,19 +50,19 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card

uint cur_idx = 0;
while (true) {
uint num_elems = num_cards_in(_value);
if (num_elems > 0) {
cur_idx = find(card_idx, bits_per_card, cur_idx, num_elems);
uint num_cards = num_cards_in(_value);
if (num_cards > 0) {
cur_idx = find(card_idx, bits_per_card, cur_idx, num_cards);
}
// Check if the card is already stored in the pointer.
if (cur_idx < num_elems) {
if (cur_idx < num_cards) {
return Found;
}
// Check if there is actually enough space.
if (num_elems >= max_cards_in_inline_ptr) {
if (num_cards >= max_cards_in_inline_ptr) {
return Overflow;
}
CardSetPtr new_value = merge(_value, card_idx, num_elems, bits_per_card);
CardSetPtr new_value = merge(_value, card_idx, num_cards, bits_per_card);
CardSetPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
if (_value == old_value) {
return Added;
@@ -77,38 +77,38 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card
}
}

inline uint G1CardSetInlinePtr::find(uint card_idx, uint bits_per_card, uint start_at, uint num_elems) {
assert(start_at < num_elems, "Precondition!");
inline uint G1CardSetInlinePtr::find(uint card_idx, uint bits_per_card, uint start_at, uint num_cards) {
assert(start_at < num_cards, "Precondition!");

uintptr_t const card_mask = (1 << bits_per_card) - 1;
uintptr_t value = ((uintptr_t)_value) >> card_pos_for(start_at, bits_per_card);

// Check if the card is already stored in the pointer.
for (uint cur_idx = start_at; cur_idx < num_elems; cur_idx++) {
for (uint cur_idx = start_at; cur_idx < num_cards; cur_idx++) {
if ((value & card_mask) == card_idx) {
return cur_idx;
}
value >>= bits_per_card;
}
return num_elems;
return num_cards;
}

inline bool G1CardSetInlinePtr::contains(uint card_idx, uint bits_per_card) {
uint num_elems = num_cards_in(_value);
if (num_elems == 0) {
uint num_cards = num_cards_in(_value);
if (num_cards == 0) {
return false;
}
uint cur_idx = find(card_idx, bits_per_card, 0, num_elems);
return cur_idx < num_elems;
uint cur_idx = find(card_idx, bits_per_card, 0, num_cards);
return cur_idx < num_cards;
}

template <class CardVisitor>
inline void G1CardSetInlinePtr::iterate(CardVisitor& found, uint bits_per_card) {
uint const num_elems = num_cards_in(_value);
uint const num_cards = num_cards_in(_value);
uintptr_t const card_mask = (1 << bits_per_card) - 1;

uintptr_t value = ((uintptr_t)_value) >> card_pos_for(0, bits_per_card);
for (uint cur_idx = 0; cur_idx < num_elems; cur_idx++) {
for (uint cur_idx = 0; cur_idx < num_cards; cur_idx++) {
found(value & card_mask);
value >>= bits_per_card;
}
@@ -136,9 +136,9 @@ inline uintptr_t G1CardSetContainer::decrement_refcount() {
return Atomic::sub(&_ref_count, 2u);
}

inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_elems) :
inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_cards) :
G1CardSetContainer(),
_size(num_elems),
_size(num_cards),
_num_entries(1) {
assert(_size > 0, "CardSetArray of size 0 not supported.");
assert(_size < LockBitMask, "Only support CardSetArray of size %u or smaller.", LockBitMask - 1);
@@ -166,7 +166,7 @@ inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(EntryCountType

inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
assert(card_idx < (1u << (sizeof(_data[0]) * BitsPerByte)),
"Card index %u does not fit card element.", card_idx);
"Card index %u does not fit allowed card value range.", card_idx);
EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
EntryCountType idx = 0;
for (; idx < num_entries; idx++) {
@@ -181,7 +181,7 @@ inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
// Reload number of entries from the G1CardSetArrayLocker as it might have changed.
// It already read the actual value with the necessary synchronization.
num_entries = x.num_entries();
// Look if the elements added while waiting for the lock are the same as our card.
// Look if the cards added while waiting for the lock are the same as our card.
for (; idx < num_entries; idx++) {
if (_data[idx] == card_idx) {
return Found;
@@ -30,113 +30,113 @@
#include "runtime/atomic.hpp"
#include "utilities/ostream.hpp"

template <class Elem>
G1CardSetAllocator<Elem>::G1CardSetAllocator(const char* name,
const G1CardSetAllocOptions* buffer_options,
G1CardSetBufferList* free_buffer_list) :
_segmented_array(buffer_options, free_buffer_list),
template <class Slot>
G1CardSetAllocator<Slot>::G1CardSetAllocator(const char* name,
const G1CardSetAllocOptions* alloc_options,
G1CardSetFreeList* free_segment_list) :
_segmented_array(alloc_options, free_segment_list),
_transfer_lock(false),
_free_nodes_list(),
_pending_nodes_list(),
_num_pending_nodes(0),
_num_free_nodes(0)
_free_slots_list(),
_pending_slots_list(),
_num_pending_slots(0),
_num_free_slots(0)
{
uint elem_size = _segmented_array.elem_size();
assert(elem_size >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small", elem_size, name);
uint slot_size = _segmented_array.slot_size();
assert(slot_size >= sizeof(G1CardSetContainer), "Slot instance size %u for allocator %s too small", slot_size, name);
}

template <class Elem>
G1CardSetAllocator<Elem>::~G1CardSetAllocator() {
template <class Slot>
G1CardSetAllocator<Slot>::~G1CardSetAllocator() {
drop_all();
}

template <class Elem>
bool G1CardSetAllocator<Elem>::try_transfer_pending() {
template <class Slot>
bool G1CardSetAllocator<Slot>::try_transfer_pending() {
// Attempt to claim the lock.
if (Atomic::load_acquire(&_transfer_lock) || // Skip CAS if likely to fail.
Atomic::cmpxchg(&_transfer_lock, false, true)) {
return false;
}
// Have the lock; perform the transfer.

// Claim all the pending nodes.
G1CardSetContainer* first = _pending_nodes_list.pop_all();
// Claim all the pending slots.
G1CardSetContainer* first = _pending_slots_list.pop_all();

if (first != nullptr) {
// Prepare to add the claimed nodes, and update _num_pending_nodes.
// Prepare to add the claimed slots, and update _num_pending_slots.
G1CardSetContainer* last = first;
Atomic::load_acquire(&_num_pending_nodes);
Atomic::load_acquire(&_num_pending_slots);

uint count = 1;
for (G1CardSetContainer* next = first->next(); next != nullptr; next = next->next()) {
last = next;
++count;
}

Atomic::sub(&_num_pending_nodes, count);
Atomic::sub(&_num_pending_slots, count);

// Wait for any in-progress pops to avoid ABA for them.
GlobalCounter::write_synchronize();
// Add synchronized nodes to _free_node_list.
// Add synchronized slots to _free_slots_list.
// Update count first so there can be no underflow in allocate().
Atomic::add(&_num_free_nodes, count);
_free_nodes_list.prepend(*first, *last);
Atomic::add(&_num_free_slots, count);
_free_slots_list.prepend(*first, *last);
}
Atomic::release_store(&_transfer_lock, false);
return true;
}

template <class Elem>
void G1CardSetAllocator<Elem>::free(Elem* elem) {
assert(elem != nullptr, "precondition");
template <class Slot>
void G1CardSetAllocator<Slot>::free(Slot* slot) {
assert(slot != nullptr, "precondition");
// Desired minimum transfer batch size. There is relatively little
// importance to the specific number. It shouldn't be too big, else
// we're wasting space when the release rate is low. If the release
// rate is high, we might accumulate more than this before being
// able to start a new transfer, but that's okay. Also note that
// the allocation rate and the release rate are going to be fairly
// similar, due to how the buffers are used. - kbarret
// similar, due to how the slots are used. - kbarret
uint const trigger_transfer = 10;

uint pending_count = Atomic::add(&_num_pending_nodes, 1u, memory_order_relaxed);
uint pending_count = Atomic::add(&_num_pending_slots, 1u, memory_order_relaxed);

G1CardSetContainer* node = reinterpret_cast<G1CardSetContainer*>(reinterpret_cast<char*>(elem));
G1CardSetContainer* container = reinterpret_cast<G1CardSetContainer*>(reinterpret_cast<char*>(slot));

node->set_next(nullptr);
assert(node->next() == nullptr, "precondition");
container->set_next(nullptr);
assert(container->next() == nullptr, "precondition");

_pending_nodes_list.push(*node);
_pending_slots_list.push(*container);

if (pending_count > trigger_transfer) {
try_transfer_pending();
}
}

template <class Elem>
void G1CardSetAllocator<Elem>::drop_all() {
_free_nodes_list.pop_all();
_pending_nodes_list.pop_all();
_num_pending_nodes = 0;
_num_free_nodes = 0;
template <class Slot>
void G1CardSetAllocator<Slot>::drop_all() {
_free_slots_list.pop_all();
_pending_slots_list.pop_all();
_num_pending_slots = 0;
_num_free_slots = 0;
_segmented_array.drop_all();
}

template <class Elem>
void G1CardSetAllocator<Elem>::print(outputStream* os) {
uint num_allocated_nodes = _segmented_array.num_allocated_nodes();
uint num_available_nodes = _segmented_array.num_available_nodes();
uint highest = _segmented_array.first_array_buffer() != nullptr
? _segmented_array.first_array_buffer()->num_elems()
template <class Slot>
void G1CardSetAllocator<Slot>::print(outputStream* os) {
uint num_allocated_slots = _segmented_array.num_allocated_slots();
uint num_available_slots = _segmented_array.num_available_slots();
uint highest = _segmented_array.first_array_segment() != nullptr
? _segmented_array.first_array_segment()->num_slots()
: 0;
uint num_buffers = _segmented_array.num_buffers();
os->print("MA " PTR_FORMAT ": %u elems pending (allocated %u available %u) used %.3f highest %u buffers %u size %zu ",
uint num_segments = _segmented_array.num_segments();
os->print("MA " PTR_FORMAT ": %u slots pending (allocated %u available %u) used %.3f highest %u segments %u size %zu ",
p2i(this),
_num_pending_nodes,
num_allocated_nodes,
num_available_nodes,
percent_of(num_allocated_nodes - _num_pending_nodes, num_available_nodes),
_num_pending_slots,
num_allocated_slots,
num_available_slots,
percent_of(num_allocated_slots - _num_pending_slots, num_available_slots),
highest,
num_buffers,
num_segments,
mem_size());
}

@@ -205,7 +205,7 @@ G1SegmentedArrayMemoryStats G1CardSetMemoryManager::memory_stats() const {
G1SegmentedArrayMemoryStats result;
for (uint i = 0; i < num_mem_object_types(); i++) {
result._num_mem_sizes[i] += _allocators[i].mem_size();
result._num_segments[i] += _allocators[i].num_buffers();
result._num_segments[i] += _allocators[i].num_segments();
}
return result;
}

1 comment on commit 67745fa

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on 67745fa Dec 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.