Skip to content
Permalink
Browse files
8273626: G1: Factor out concurrent segmented array from G1CardSetAllo…
…cator

Reviewed-by: tschatzl, ayang
  • Loading branch information
Hamlin Li committed Oct 19, 2021
1 parent a4491f2 commit d17d81a8b2a6336f37e17f31413a62c7adf49936
Show file tree
Hide file tree
Showing 8 changed files with 520 additions and 325 deletions.
@@ -65,6 +65,8 @@ G1CardSetConfiguration::G1CardSetConfiguration() :
_cards_in_howl_bitmap_threshold = _num_cards_in_howl_bitmap * (double)G1RemSetCoarsenHowlBitmapToHowlFullPercent / 100;
_bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);

init_card_set_alloc_options();

log_configuration();
}

@@ -89,9 +91,23 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
_log2_num_cards_in_howl_bitmap = log2i_exact(_num_cards_in_howl_bitmap);
_bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);

init_card_set_alloc_options();

log_configuration();
}

G1CardSetConfiguration::~G1CardSetConfiguration() {
FREE_C_HEAP_ARRAY(size_t, _card_set_alloc_options);
}

void G1CardSetConfiguration::init_card_set_alloc_options() {
_card_set_alloc_options = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC);
new (&_card_set_alloc_options[0]) G1CardSetAllocOptions((uint)CardSetHash::get_node_size());
new (&_card_set_alloc_options[1]) G1CardSetAllocOptions((uint)G1CardSetArray::size_in_bytes(_num_cards_in_array), 2, 256);
new (&_card_set_alloc_options[2]) G1CardSetAllocOptions((uint)G1CardSetBitMap::size_in_bytes(_num_cards_in_howl_bitmap), 2, 256);
new (&_card_set_alloc_options[3]) G1CardSetAllocOptions((uint)G1CardSetHowl::size_in_bytes(_num_buckets_in_howl), 2, 256);
}

void G1CardSetConfiguration::log_configuration() {
log_debug_p(gc, remset)("Card Set container configuration: "
"InlinePtr #elems %u size %zu "
@@ -112,15 +128,8 @@ uint G1CardSetConfiguration::num_cards_in_inline_ptr(uint bits_per_card) {
return G1CardSetInlinePtr::max_cards_in_inline_ptr(bits_per_card);
}

G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options() {
G1CardSetAllocOptions* result = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC);

result[0] = { (uint)CardSetHash::get_node_size() };
result[1] = { (uint)G1CardSetArray::size_in_bytes(num_cards_in_array()), 2, 256 };
result[2] = { (uint)G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), 2, 256 };
result[3] = { (uint)G1CardSetHowl::size_in_bytes(num_buckets_in_howl()), 2, 256 };

return result;
const G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options(uint idx) {
return &_card_set_alloc_options[idx];
}

const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) {
@@ -32,7 +32,6 @@
#include "utilities/lockFreeStack.hpp"

class G1CardSetAllocOptions;
class G1CardSetBufferList;
class G1CardSetHashTable;
class G1CardSetHashTableValue;
class G1CardSetMemoryManager;
@@ -60,6 +59,10 @@ class G1CardSetConfiguration {
uint _log2_num_cards_in_howl_bitmap;
size_t _bitmap_hash_mask;

G1CardSetAllocOptions* _card_set_alloc_options;

void init_card_set_alloc_options();

void log_configuration();
public:

@@ -73,6 +76,8 @@ class G1CardSetConfiguration {
double cards_in_howl_threshold,
uint max_cards_in_cardset);

~G1CardSetConfiguration();

// Inline pointer configuration
uint inline_ptr_bits_per_card() const { return _inline_ptr_bits_per_card; }
uint num_cards_in_inline_ptr() const;
@@ -108,9 +113,8 @@ class G1CardSetConfiguration {
// Number of distinctly sized memory objects on the card set heap.
// Currently contains CHT-Nodes, ArrayOfCards, BitMaps, Howl
static constexpr uint num_mem_object_types() { return 4; }
// Returns the memory allocation options for the memory objects on the card set heap. The returned
// array must be freed by the caller.
G1CardSetAllocOptions* mem_object_alloc_options();
// Returns the memory allocation options for the memory objects on the card set heap.
const G1CardSetAllocOptions* mem_object_alloc_options(uint idx);

// For a given memory object, get a descriptive name.
static const char* mem_object_type_name_str(uint index);
@@ -31,8 +31,6 @@
#include "utilities/growableArray.hpp"
#include "utilities/ticks.hpp"

class G1CardSetBuffer;

// Task handling deallocation of free card set memory.
class G1CardSetFreeMemoryTask : public G1ServiceTask {

@@ -30,99 +30,20 @@
#include "utilities/formatBuffer.hpp"
#include "utilities/ostream.hpp"

G1CardSetBuffer::G1CardSetBuffer(uint elem_size, uint num_instances, G1CardSetBuffer* next) :
_elem_size(elem_size), _num_elems(num_instances), _next(next), _next_allocate(0) {

_buffer = NEW_C_HEAP_ARRAY(char, (size_t)_num_elems * elem_size, mtGCCardSet);
}

G1CardSetBuffer::~G1CardSetBuffer() {
FREE_C_HEAP_ARRAY(mtGCCardSet, _buffer);
}

void* G1CardSetBuffer::get_new_buffer_elem() {
if (_next_allocate >= _num_elems) {
return nullptr;
}
uint result = Atomic::fetch_and_add(&_next_allocate, 1u, memory_order_relaxed);
if (result >= _num_elems) {
return nullptr;
}
void* r = _buffer + (uint)result * _elem_size;
return r;
}

void G1CardSetBufferList::bulk_add(G1CardSetBuffer& first, G1CardSetBuffer& last, size_t num, size_t mem_size) {
_list.prepend(first, last);
Atomic::add(&_num_buffers, num, memory_order_relaxed);
Atomic::add(&_mem_size, mem_size, memory_order_relaxed);
}

void G1CardSetBufferList::print_on(outputStream* out, const char* prefix) {
out->print_cr("%s: buffers %zu size %zu", prefix, Atomic::load(&_num_buffers), Atomic::load(&_mem_size));
}

G1CardSetBuffer* G1CardSetBufferList::get() {
GlobalCounter::CriticalSection cs(Thread::current());

G1CardSetBuffer* result = _list.pop();
if (result != nullptr) {
Atomic::dec(&_num_buffers, memory_order_relaxed);
Atomic::sub(&_mem_size, result->mem_size(), memory_order_relaxed);
}
return result;
}

G1CardSetBuffer* G1CardSetBufferList::get_all(size_t& num_buffers, size_t& mem_size) {
GlobalCounter::CriticalSection cs(Thread::current());

G1CardSetBuffer* result = _list.pop_all();
num_buffers = Atomic::load(&_num_buffers);
mem_size = Atomic::load(&_mem_size);

if (result != nullptr) {
Atomic::sub(&_num_buffers, num_buffers, memory_order_relaxed);
Atomic::sub(&_mem_size, mem_size, memory_order_relaxed);
}
return result;
}

void G1CardSetBufferList::free_all() {
size_t num_freed = 0;
size_t mem_size_freed = 0;
G1CardSetBuffer* cur;

while ((cur = _list.pop()) != nullptr) {
mem_size_freed += cur->mem_size();
num_freed++;
delete cur;
}

Atomic::sub(&_num_buffers, num_freed, memory_order_relaxed);
Atomic::sub(&_mem_size, mem_size_freed, memory_order_relaxed);
}

template <class Elem>
G1CardSetAllocator<Elem>::G1CardSetAllocator(const char* name,
const G1CardSetAllocOptions& buffer_options,
const G1CardSetAllocOptions* buffer_options,
G1CardSetBufferList* free_buffer_list) :
_alloc_options(buffer_options),
_first(nullptr),
_last(nullptr),
_num_buffers(0),
_mem_size(0),
_free_buffer_list(free_buffer_list),
_segmented_array(name, buffer_options, free_buffer_list),
_transfer_lock(false),
_free_nodes_list(),
_pending_nodes_list(),
_num_pending_nodes(0),
_num_free_nodes(0),
_num_allocated_nodes(0),
_num_available_nodes(0)
_num_free_nodes(0)
{
assert(elem_size() >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small",
elem_size(), name);
assert(_free_buffer_list != nullptr, "precondition!");
uint elem_size = _segmented_array.elem_size();
assert(elem_size >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small", elem_size, name);
}

template <class Elem>
@@ -164,7 +85,6 @@ bool G1CardSetAllocator<Elem>::try_transfer_pending() {
template <class Elem>
void G1CardSetAllocator<Elem>::free(Elem* elem) {
assert(elem != nullptr, "precondition");
assert(elem_size() >= sizeof(G1CardSetContainer), "size mismatch");
// Desired minimum transfer batch size. There is relatively little
// importance to the specific number. It shouldn't be too big, else
// we're wasting space when the release rate is low. If the release
@@ -192,47 +112,27 @@ template <class Elem>
void G1CardSetAllocator<Elem>::drop_all() {
_free_nodes_list.pop_all();
_pending_nodes_list.pop_all();
G1CardSetBuffer* cur = Atomic::load_acquire(&_first);

if (cur != nullptr) {
assert(_last != nullptr, "If there is at least one element, there must be a last one.");

G1CardSetBuffer* first = cur;
#ifdef ASSERT
// Check list consistency.
G1CardSetBuffer* last = cur;
uint num_buffers = 0;
size_t mem_size = 0;
while (cur != nullptr) {
mem_size += cur->mem_size();
num_buffers++;

G1CardSetBuffer* next = cur->next();
last = cur;
cur = next;
}
#endif
assert(num_buffers == _num_buffers, "Buffer count inconsistent %u %u", num_buffers, _num_buffers);
assert(mem_size == _mem_size, "Memory size inconsistent");
assert(last == _last, "Inconsistent last element");

_free_buffer_list->bulk_add(*first, *_last, _num_buffers, _mem_size);
}

_first = nullptr;
_last = nullptr;
_num_available_nodes = 0;
_num_allocated_nodes = 0;
_num_pending_nodes = 0;
_num_buffers = 0;
_mem_size = 0;
_num_free_nodes = 0;
}

template <class Elem>
void G1CardSetAllocator<Elem>::print(outputStream* os) {
uint num_allocated_nodes = _segmented_array.num_allocated_nodes();
uint num_available_nodes = _segmented_array.num_available_nodes();
uint highest = _segmented_array.first_array_buffer() != nullptr
? _segmented_array.first_array_buffer()->num_elems()
: 0;
uint num_buffers = _segmented_array.num_buffers();
os->print("MA " PTR_FORMAT ": %u elems pending (allocated %u available %u) used %.3f highest %u buffers %u size %zu ",
p2i(this), _num_pending_nodes, _num_allocated_nodes, _num_available_nodes, percent_of(_num_allocated_nodes - _num_pending_nodes, _num_available_nodes), _first != nullptr ? _first->num_elems() : 0, _num_buffers, mem_size());
p2i(this),
_num_pending_nodes,
num_allocated_nodes,
num_available_nodes,
percent_of(num_allocated_nodes - _num_pending_nodes, num_available_nodes),
highest,
num_buffers,
mem_size());
}

G1CardSetMemoryStats::G1CardSetMemoryStats() {
@@ -411,13 +311,11 @@ G1CardSetMemoryManager::G1CardSetMemoryManager(G1CardSetConfiguration* config,
_allocators = NEW_C_HEAP_ARRAY(G1CardSetAllocator<G1CardSetContainer>,
_config->num_mem_object_types(),
mtGC);
G1CardSetAllocOptions* alloc_options = _config->mem_object_alloc_options();
for (uint i = 0; i < num_mem_object_types(); i++) {
new (&_allocators[i]) G1CardSetAllocator<G1CardSetContainer>(_config->mem_object_type_name_str(i),
alloc_options[i],
_config->mem_object_alloc_options(i),
free_list_pool->free_list(i));
}
FREE_C_HEAP_ARRAY(size_t, alloc_options);
}

uint G1CardSetMemoryManager::num_mem_object_types() const {

1 comment on commit d17d81a

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on d17d81a Oct 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.