Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8273626: G1: Factor out concurrent segmented array from G1CardSetAllocator #5478

@@ -65,6 +65,8 @@ G1CardSetConfiguration::G1CardSetConfiguration() :
_cards_in_howl_bitmap_threshold = _num_cards_in_howl_bitmap * (double)G1RemSetCoarsenHowlBitmapToHowlFullPercent / 100;
_bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);

init_card_set_alloc_options();

log_configuration();
}

@@ -89,9 +91,23 @@ G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
_log2_num_cards_in_howl_bitmap = log2i_exact(_num_cards_in_howl_bitmap);
_bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);

init_card_set_alloc_options();
Hamlin-Li marked this conversation as resolved.
Show resolved Hide resolved

log_configuration();
}

G1CardSetConfiguration::~G1CardSetConfiguration() {
FREE_C_HEAP_ARRAY(size_t, _card_set_alloc_options);
}

void G1CardSetConfiguration::init_card_set_alloc_options() {
_card_set_alloc_options = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC);
new (&_card_set_alloc_options[0]) G1CardSetAllocOptions((uint)CardSetHash::get_node_size());
new (&_card_set_alloc_options[1]) G1CardSetAllocOptions((uint)G1CardSetArray::size_in_bytes(_num_cards_in_array), 2, 256);
new (&_card_set_alloc_options[2]) G1CardSetAllocOptions((uint)G1CardSetBitMap::size_in_bytes(_num_cards_in_howl_bitmap), 2, 256);
new (&_card_set_alloc_options[3]) G1CardSetAllocOptions((uint)G1CardSetHowl::size_in_bytes(_num_buckets_in_howl), 2, 256);
}

void G1CardSetConfiguration::log_configuration() {
log_debug_p(gc, remset)("Card Set container configuration: "
"InlinePtr #elems %u size %zu "
@@ -112,15 +128,8 @@ uint G1CardSetConfiguration::num_cards_in_inline_ptr(uint bits_per_card) {
return G1CardSetInlinePtr::max_cards_in_inline_ptr(bits_per_card);
}

G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options() {
G1CardSetAllocOptions* result = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC);

result[0] = { (uint)CardSetHash::get_node_size() };
result[1] = { (uint)G1CardSetArray::size_in_bytes(num_cards_in_array()), 2, 256 };
result[2] = { (uint)G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), 2, 256 };
result[3] = { (uint)G1CardSetHowl::size_in_bytes(num_buckets_in_howl()), 2, 256 };

return result;
const G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options(uint idx) {
return &_card_set_alloc_options[idx];
}

const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) {
@@ -32,7 +32,6 @@
#include "utilities/lockFreeStack.hpp"

class G1CardSetAllocOptions;
class G1CardSetBufferList;
class G1CardSetHashTable;
class G1CardSetHashTableValue;
class G1CardSetMemoryManager;
@@ -60,6 +59,10 @@ class G1CardSetConfiguration {
uint _log2_num_cards_in_howl_bitmap;
size_t _bitmap_hash_mask;

G1CardSetAllocOptions* _card_set_alloc_options;

void init_card_set_alloc_options();

void log_configuration();
public:

@@ -73,6 +76,8 @@ class G1CardSetConfiguration {
double cards_in_howl_threshold,
uint max_cards_in_cardset);

~G1CardSetConfiguration();

// Inline pointer configuration
uint inline_ptr_bits_per_card() const { return _inline_ptr_bits_per_card; }
uint num_cards_in_inline_ptr() const;
@@ -108,9 +113,8 @@ class G1CardSetConfiguration {
// Number of distinctly sized memory objects on the card set heap.
// Currently contains CHT-Nodes, ArrayOfCards, BitMaps, Howl
static constexpr uint num_mem_object_types() { return 4; }
// Returns the memory allocation options for the memory objects on the card set heap. The returned
// array must be freed by the caller.
G1CardSetAllocOptions* mem_object_alloc_options();
// Returns the memory allocation options for the memory objects on the card set heap.
const G1CardSetAllocOptions* mem_object_alloc_options(uint idx);
Hamlin-Li marked this conversation as resolved.
Show resolved Hide resolved

// For a given memory object, get a descriptive name.
static const char* mem_object_type_name_str(uint index);
@@ -31,8 +31,6 @@
#include "utilities/growableArray.hpp"
#include "utilities/ticks.hpp"

class G1CardSetBuffer;

// Task handling deallocation of free card set memory.
class G1CardSetFreeMemoryTask : public G1ServiceTask {

@@ -30,99 +30,20 @@
#include "utilities/formatBuffer.hpp"
#include "utilities/ostream.hpp"

G1CardSetBuffer::G1CardSetBuffer(uint elem_size, uint num_instances, G1CardSetBuffer* next) :
_elem_size(elem_size), _num_elems(num_instances), _next(next), _next_allocate(0) {

_buffer = NEW_C_HEAP_ARRAY(char, (size_t)_num_elems * elem_size, mtGCCardSet);
}

G1CardSetBuffer::~G1CardSetBuffer() {
FREE_C_HEAP_ARRAY(mtGCCardSet, _buffer);
}

void* G1CardSetBuffer::get_new_buffer_elem() {
if (_next_allocate >= _num_elems) {
return nullptr;
}
uint result = Atomic::fetch_and_add(&_next_allocate, 1u, memory_order_relaxed);
if (result >= _num_elems) {
return nullptr;
}
void* r = _buffer + (uint)result * _elem_size;
return r;
}

void G1CardSetBufferList::bulk_add(G1CardSetBuffer& first, G1CardSetBuffer& last, size_t num, size_t mem_size) {
_list.prepend(first, last);
Atomic::add(&_num_buffers, num, memory_order_relaxed);
Atomic::add(&_mem_size, mem_size, memory_order_relaxed);
}

void G1CardSetBufferList::print_on(outputStream* out, const char* prefix) {
out->print_cr("%s: buffers %zu size %zu", prefix, Atomic::load(&_num_buffers), Atomic::load(&_mem_size));
}

G1CardSetBuffer* G1CardSetBufferList::get() {
GlobalCounter::CriticalSection cs(Thread::current());

G1CardSetBuffer* result = _list.pop();
if (result != nullptr) {
Atomic::dec(&_num_buffers, memory_order_relaxed);
Atomic::sub(&_mem_size, result->mem_size(), memory_order_relaxed);
}
return result;
}

G1CardSetBuffer* G1CardSetBufferList::get_all(size_t& num_buffers, size_t& mem_size) {
GlobalCounter::CriticalSection cs(Thread::current());

G1CardSetBuffer* result = _list.pop_all();
num_buffers = Atomic::load(&_num_buffers);
mem_size = Atomic::load(&_mem_size);

if (result != nullptr) {
Atomic::sub(&_num_buffers, num_buffers, memory_order_relaxed);
Atomic::sub(&_mem_size, mem_size, memory_order_relaxed);
}
return result;
}

void G1CardSetBufferList::free_all() {
size_t num_freed = 0;
size_t mem_size_freed = 0;
G1CardSetBuffer* cur;

while ((cur = _list.pop()) != nullptr) {
mem_size_freed += cur->mem_size();
num_freed++;
delete cur;
}

Atomic::sub(&_num_buffers, num_freed, memory_order_relaxed);
Atomic::sub(&_mem_size, mem_size_freed, memory_order_relaxed);
}

template <class Elem>
G1CardSetAllocator<Elem>::G1CardSetAllocator(const char* name,
const G1CardSetAllocOptions& buffer_options,
const G1CardSetAllocOptions* buffer_options,
G1CardSetBufferList* free_buffer_list) :
_alloc_options(buffer_options),
_first(nullptr),
_last(nullptr),
_num_buffers(0),
_mem_size(0),
_free_buffer_list(free_buffer_list),
_segmented_array(name, buffer_options, free_buffer_list),
_transfer_lock(false),
_free_nodes_list(),
_pending_nodes_list(),
_num_pending_nodes(0),
_num_free_nodes(0),
_num_allocated_nodes(0),
_num_available_nodes(0)
_num_free_nodes(0)
{
assert(elem_size() >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small",
elem_size(), name);
assert(_free_buffer_list != nullptr, "precondition!");
uint elem_size = _segmented_array.elem_size();
assert(elem_size >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small", elem_size, name);
}

template <class Elem>
@@ -164,7 +85,6 @@ bool G1CardSetAllocator<Elem>::try_transfer_pending() {
template <class Elem>
void G1CardSetAllocator<Elem>::free(Elem* elem) {
assert(elem != nullptr, "precondition");
assert(elem_size() >= sizeof(G1CardSetContainer), "size mismatch");
// Desired minimum transfer batch size. There is relatively little
// importance to the specific number. It shouldn't be too big, else
// we're wasting space when the release rate is low. If the release
@@ -192,47 +112,27 @@ template <class Elem>
void G1CardSetAllocator<Elem>::drop_all() {
_free_nodes_list.pop_all();
_pending_nodes_list.pop_all();
G1CardSetBuffer* cur = Atomic::load_acquire(&_first);

if (cur != nullptr) {
assert(_last != nullptr, "If there is at least one element, there must be a last one.");

G1CardSetBuffer* first = cur;
#ifdef ASSERT
// Check list consistency.
G1CardSetBuffer* last = cur;
uint num_buffers = 0;
size_t mem_size = 0;
while (cur != nullptr) {
mem_size += cur->mem_size();
num_buffers++;

G1CardSetBuffer* next = cur->next();
last = cur;
cur = next;
}
#endif
assert(num_buffers == _num_buffers, "Buffer count inconsistent %u %u", num_buffers, _num_buffers);
assert(mem_size == _mem_size, "Memory size inconsistent");
assert(last == _last, "Inconsistent last element");

_free_buffer_list->bulk_add(*first, *_last, _num_buffers, _mem_size);
}

_first = nullptr;
_last = nullptr;
_num_available_nodes = 0;
_num_allocated_nodes = 0;
_num_pending_nodes = 0;
_num_buffers = 0;
_mem_size = 0;
_num_free_nodes = 0;
}

template <class Elem>
void G1CardSetAllocator<Elem>::print(outputStream* os) {
uint num_allocated_nodes = _segmented_array.num_allocated_nodes();
uint num_available_nodes = _segmented_array.num_available_nodes();
uint highest = _segmented_array.first_array_buffer() != nullptr
? _segmented_array.first_array_buffer()->num_elems()
: 0;
uint num_buffers = _segmented_array.num_buffers();
os->print("MA " PTR_FORMAT ": %u elems pending (allocated %u available %u) used %.3f highest %u buffers %u size %zu ",
p2i(this), _num_pending_nodes, _num_allocated_nodes, _num_available_nodes, percent_of(_num_allocated_nodes - _num_pending_nodes, _num_available_nodes), _first != nullptr ? _first->num_elems() : 0, _num_buffers, mem_size());
p2i(this),
_num_pending_nodes,
num_allocated_nodes,
num_available_nodes,
percent_of(num_allocated_nodes - _num_pending_nodes, num_available_nodes),
highest,
num_buffers,
mem_size());
}

G1CardSetMemoryStats::G1CardSetMemoryStats() {
@@ -411,13 +311,11 @@ G1CardSetMemoryManager::G1CardSetMemoryManager(G1CardSetConfiguration* config,
_allocators = NEW_C_HEAP_ARRAY(G1CardSetAllocator<G1CardSetContainer>,
_config->num_mem_object_types(),
mtGC);
G1CardSetAllocOptions* alloc_options = _config->mem_object_alloc_options();
for (uint i = 0; i < num_mem_object_types(); i++) {
new (&_allocators[i]) G1CardSetAllocator<G1CardSetContainer>(_config->mem_object_type_name_str(i),
alloc_options[i],
_config->mem_object_alloc_options(i),
free_list_pool->free_list(i));
}
FREE_C_HEAP_ARRAY(size_t, alloc_options);
}

uint G1CardSetMemoryManager::num_mem_object_types() const {