Skip to content

8321013: Parallel: Refactor ObjectStartArray #16884

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

8321013: Parallel: Refactor ObjectStartArray #16884

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4c36f5e
pgc-bot
albertnetymk Nov 3, 2023
1cdec09
Merge branch 'master' into pgc-bot
albertnetymk Dec 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 68 additions & 74 deletions src/hotspot/share/gc/parallel/objectStartArray.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,21 @@
#include "precompiled.hpp"
#include "gc/parallel/objectStartArray.inline.hpp"
#include "gc/shared/cardTableBarrierSet.hpp"
#include "memory/allocation.inline.hpp"
#include "nmt/memTracker.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/java.hpp"
#include "utilities/align.hpp"

uint ObjectStartArray::_card_shift = 0;
uint ObjectStartArray::_card_size = 0;
uint ObjectStartArray::_card_size_in_words = 0;
static size_t num_bytes_required(MemRegion mr) {
assert(CardTable::is_card_aligned(mr.start()), "precondition");
assert(CardTable::is_card_aligned(mr.end()), "precondition");

void ObjectStartArray::initialize_block_size(uint card_shift) {
_card_shift = card_shift;
_card_size = 1 << _card_shift;
_card_size_in_words = _card_size / sizeof(HeapWord);
return mr.word_size() / BOTConstants::card_size_in_words();
}

void ObjectStartArray::initialize(MemRegion reserved_region) {
// We're based on the assumption that we use the same
// size blocks as the card table.
assert(_card_size == CardTable::card_size(), "Sanity");
assert(_card_size <= MaxBlockSize, "block_size must be less than or equal to " UINT32_FORMAT, MaxBlockSize);

// Calculate how much space must be reserved
_reserved_region = reserved_region;

size_t bytes_to_reserve = reserved_region.word_size() / _card_size_in_words;
size_t bytes_to_reserve = num_bytes_required(reserved_region);
assert(bytes_to_reserve > 0, "Sanity");

bytes_to_reserve =
Expand All @@ -62,91 +51,96 @@ void ObjectStartArray::initialize(MemRegion reserved_region) {
if (!backing_store.is_reserved()) {
vm_exit_during_initialization("Could not reserve space for ObjectStartArray");
}
MemTracker::record_virtual_memory_type((address)backing_store.base(), mtGC);
MemTracker::record_virtual_memory_type(backing_store.base(), mtGC);

// We do not commit any memory initially
_virtual_space.initialize(backing_store);

_raw_base = (jbyte*)_virtual_space.low_boundary();
assert(_raw_base != nullptr, "set from the backing_store");

_offset_base = _raw_base - (size_t(reserved_region.start()) >> _card_shift);

_covered_region.set_start(reserved_region.start());
_covered_region.set_word_size(0);
assert(_virtual_space.low_boundary() != nullptr, "set from the backing_store");

_blocks_region.set_start((HeapWord*)_raw_base);
_blocks_region.set_word_size(0);
_offset_base = (uint8_t*)(_virtual_space.low_boundary() - (uintptr_t(reserved_region.start()) >> BOTConstants::log_card_size()));
}

void ObjectStartArray::set_covered_region(MemRegion mr) {
assert(_reserved_region.contains(mr), "MemRegion outside of reserved space");
assert(_reserved_region.start() == mr.start(), "Attempt to move covered region");

HeapWord* low_bound = mr.start();
HeapWord* high_bound = mr.end();
assert((uintptr_t(low_bound) & (_card_size - 1)) == 0, "heap must start at block boundary");
assert((uintptr_t(high_bound) & (_card_size - 1)) == 0, "heap must end at block boundary");

size_t requested_blocks_size_in_bytes = mr.word_size() / _card_size_in_words;
DEBUG_ONLY(_covered_region = mr;)

size_t requested_size = num_bytes_required(mr);
// Only commit memory in page sized chunks
requested_blocks_size_in_bytes =
align_up(requested_blocks_size_in_bytes, os::vm_page_size());
requested_size = align_up(requested_size, os::vm_page_size());

_covered_region = mr;
size_t current_size = _virtual_space.committed_size();

size_t current_blocks_size_in_bytes = _blocks_region.byte_size();
if (requested_size == current_size) {
return;
}

if (requested_blocks_size_in_bytes > current_blocks_size_in_bytes) {
if (requested_size > current_size) {
// Expand
size_t expand_by = requested_blocks_size_in_bytes - current_blocks_size_in_bytes;
size_t expand_by = requested_size - current_size;
if (!_virtual_space.expand_by(expand_by)) {
vm_exit_out_of_memory(expand_by, OOM_MMAP_ERROR, "object start array expansion");
}
// Clear *only* the newly allocated region
memset(_blocks_region.end(), clean_block, expand_by);
}

if (requested_blocks_size_in_bytes < current_blocks_size_in_bytes) {
} else {
// Shrink
size_t shrink_by = current_blocks_size_in_bytes - requested_blocks_size_in_bytes;
size_t shrink_by = current_size - requested_size;
_virtual_space.shrink_by(shrink_by);
}

_blocks_region.set_word_size(requested_blocks_size_in_bytes / sizeof(HeapWord));

assert(requested_blocks_size_in_bytes % sizeof(HeapWord) == 0, "Block table not expanded in word sized increment");
assert(requested_blocks_size_in_bytes == _blocks_region.byte_size(), "Sanity");
assert(block_for_addr(low_bound) == &_raw_base[0], "Checking start of map");
assert(block_for_addr(high_bound-1) <= &_raw_base[_blocks_region.byte_size()-1], "Checking end of map");
}

void ObjectStartArray::reset() {
memset(_blocks_region.start(), clean_block, _blocks_region.byte_size());
static void fill_range(uint8_t* start, uint8_t* end, uint8_t v) {
// + 1 for inclusive
memset(start, v, pointer_delta(end, start, sizeof(uint8_t)) + 1);
}

bool ObjectStartArray::object_starts_in_range(HeapWord* start_addr,
HeapWord* end_addr) const {
assert(start_addr <= end_addr,
"Range is wrong. start_addr (" PTR_FORMAT ") is after end_addr (" PTR_FORMAT ")",
p2i(start_addr), p2i(end_addr));

assert(is_aligned(start_addr, _card_size), "precondition");

if (start_addr == end_addr) {
// No objects in empty range.
return false;
void ObjectStartArray::update_for_block_work(HeapWord* blk_start,
HeapWord* blk_end) {
HeapWord* const cur_card_boundary = align_up_by_card_size(blk_start);
uint8_t* const offset_entry = entry_for_addr(cur_card_boundary);

// The first card holds the actual offset.
*offset_entry = checked_cast<uint8_t>(pointer_delta(cur_card_boundary, blk_start));

// Check if this block spans over other cards.
uint8_t* const end_entry = entry_for_addr(blk_end - 1);
assert(offset_entry <= end_entry, "inv");

if (offset_entry != end_entry) {
// Handling remaining entries.
uint8_t* start_entry_for_region = offset_entry + 1;
for (uint i = 0; i < BOTConstants::N_powers; i++) {
// -1 so that the reach ends in this region and not at the start
// of the next.
uint8_t* reach = offset_entry + BOTConstants::power_to_cards_back(i + 1) - 1;
uint8_t value = checked_cast<uint8_t>(BOTConstants::card_size_in_words() + i);

fill_range(start_entry_for_region, MIN2(reach, end_entry), value);
start_entry_for_region = reach + 1;

if (reach >= end_entry) {
break;
}
}
assert(start_entry_for_region > end_entry, "Sanity check");
}

jbyte* start_block = block_for_addr(start_addr);
jbyte* end_block = block_for_addr(end_addr - 1);
debug_only(verify_for_block(blk_start, blk_end);)
}

for (jbyte* block = start_block; block <= end_block; block++) {
if (*block != clean_block) {
return true;
void ObjectStartArray::verify_for_block(HeapWord* blk_start, HeapWord* blk_end) const {
assert(is_crossing_card_boundary(blk_start, blk_end), "precondition");

const uint8_t* const start_entry = entry_for_addr(align_up_by_card_size(blk_start));
const uint8_t* const end_entry = entry_for_addr(blk_end - 1);
// Check entries in [start_entry, end_entry]
assert(*start_entry < BOTConstants::card_size_in_words(), "offset entry");

for (const uint8_t* i = start_entry + 1; i <= end_entry; ++i) {
const uint8_t prev = *(i-1);
const uint8_t value = *i;
if (prev != value) {
assert(value >= prev, "monotonic");
size_t n_cards_back = BOTConstants::entry_to_cards_back(value);
assert(start_entry == (i - n_cards_back), "inv");
}
}

return false;
}
137 changes: 31 additions & 106 deletions src/hotspot/share/gc/parallel/objectStartArray.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#define SHARE_GC_PARALLEL_OBJECTSTARTARRAY_HPP

#include "gc/parallel/psVirtualspace.hpp"
#include "gc/shared/blockOffsetTable.hpp"
#include "memory/allocation.hpp"
#include "memory/memRegion.hpp"
#include "oops/oop.hpp"
Expand All @@ -36,141 +37,65 @@
//

class ObjectStartArray : public CHeapObj<mtGC> {
friend class VerifyObjectStartArrayClosure;

private:
PSVirtualSpace _virtual_space;
MemRegion _reserved_region;
// The committed (old-gen heap) virtual space this object-start-array covers.
MemRegion _covered_region;
MemRegion _blocks_region;
jbyte* _raw_base;
jbyte* _offset_base;

static uint _card_shift;
static uint _card_size;
static uint _card_size_in_words;

public:

enum BlockValueConstants {
clean_block = -1
};

// Maximum size an offset table entry can cover. This maximum is derived from that
// we need an extra bit for possible offsets in the byte for backskip values, leaving 2^7 possible offsets.
// Minimum object alignment is 8 bytes (2^3), so we can at most represent 2^10 offsets within a BOT value.
static const uint MaxBlockSize = 1024;

// Initialize block size based on card size
static void initialize_block_size(uint card_shift);
DEBUG_ONLY(MemRegion _covered_region;)

static uint card_shift() {
return _card_shift;
}

static uint card_size() {
return _card_size;
}
static uint card_size_in_words() {
return _card_size_in_words;
}
// BOT array
PSVirtualSpace _virtual_space;

protected:
// Biased array-start of BOT array for fast heap-addr / BOT entry translation
uint8_t* _offset_base;

// Mapping from address to object start array entry
jbyte* block_for_addr(void* p) const {
uint8_t* entry_for_addr(const void* const p) const {
assert(_covered_region.contains(p),
"out of bounds access to object start array");
jbyte* result = &_offset_base[uintptr_t(p) >> _card_shift];
assert(_blocks_region.contains(result),
"out of bounds result in byte_for");
uint8_t* result = &_offset_base[uintptr_t(p) >> BOTConstants::log_card_size()];
return result;
}

// Mapping from object start array entry to address of first word
HeapWord* addr_for_block(jbyte* p) {
assert(_blocks_region.contains(p),
"out of bounds access to object start array");
size_t delta = pointer_delta(p, _offset_base, sizeof(jbyte));
HeapWord* result = (HeapWord*) (delta << _card_shift);
HeapWord* addr_for_entry(const uint8_t* const p) const {
size_t delta = pointer_delta(p, _offset_base, sizeof(uint8_t));
HeapWord* result = (HeapWord*) (delta << BOTConstants::log_card_size());
assert(_covered_region.contains(result),
"out of bounds accessor from card marking array");
return result;
}

// Mapping that includes the derived offset.
// If the block is clean, returns the last address in the covered region.
// If the block is < index 0, returns the start of the covered region.
HeapWord* offset_addr_for_block(jbyte* p) const {
// We have to do this before the assert
if (p < _raw_base) {
return _covered_region.start();
}

assert(_blocks_region.contains(p),
"out of bounds access to object start array");

if (*p == clean_block) {
return _covered_region.end();
}
static HeapWord* align_up_by_card_size(HeapWord* const addr) {
return align_up(addr, BOTConstants::card_size());
}

size_t delta = pointer_delta(p, _offset_base, sizeof(jbyte));
HeapWord* result = (HeapWord*) (delta << _card_shift);
result += *p;
void update_for_block_work(HeapWord* blk_start, HeapWord* blk_end);

assert(_covered_region.contains(result),
"out of bounds accessor from card marking array");

return result;
}
void verify_for_block(HeapWord* blk_start, HeapWord* blk_end) const;

public:

// This method is in lieu of a constructor, so that this class can be
// embedded inline in other classes.
void initialize(MemRegion reserved_region);

// Heap old-gen resizing
void set_covered_region(MemRegion mr);

void reset();

MemRegion covered_region() { return _covered_region; }

#define assert_covered_region_contains(addr) \
assert(_covered_region.contains(addr), \
#addr " (" PTR_FORMAT ") is not in covered region [" PTR_FORMAT ", " PTR_FORMAT "]", \
p2i(addr), p2i(_covered_region.start()), p2i(_covered_region.end()))

void allocate_block(HeapWord* p) {
assert_covered_region_contains(p);
jbyte* block = block_for_addr(p);
HeapWord* block_base = addr_for_block(block);
size_t offset = pointer_delta(p, block_base, sizeof(HeapWord*));
assert(offset < 128, "Sanity");
// When doing MT offsets, we can't assert this.
//assert(offset > *block, "Found backwards allocation");
*block = (jbyte)offset;
static bool is_crossing_card_boundary(HeapWord* const blk_start,
HeapWord* const blk_end) {
HeapWord* cur_card_boundary = align_up_by_card_size(blk_start);
// Strictly greater-than, since we check if this block *crosses* card boundary.
return blk_end > cur_card_boundary;
}

// Optimized for finding the first object that crosses into
// a given block. The blocks contain the offset of the last
// object in that block. Scroll backwards by one, and the first
// object hit should be at the beginning of the block
inline HeapWord* object_start(HeapWord* addr) const;
// Returns the address of the start of the block reaching into the card containing
// "addr".
inline HeapWord* block_start_reaching_into_card(HeapWord* const addr) const;

bool is_block_allocated(HeapWord* addr) {
assert_covered_region_contains(addr);
jbyte* block = block_for_addr(addr);
return *block != clean_block;
// [blk_start, blk_end) representing a block of memory in the heap.
void update_for_block(HeapWord* blk_start, HeapWord* blk_end) {
if (is_crossing_card_boundary(blk_start, blk_end)) {
update_for_block_work(blk_start, blk_end);
}
}

// Return true iff an object starts in
// [start_addr, end_addr_aligned_up)
// where
// end_addr_aligned_up = align_up(end_addr, _card_size)
// Precondition: start_addr is card-size aligned
bool object_starts_in_range(HeapWord* start_addr, HeapWord* end_addr) const;
inline HeapWord* object_start(HeapWord* const addr) const;
};

#endif // SHARE_GC_PARALLEL_OBJECTSTARTARRAY_HPP
Loading