Skip to content
Permalink
Browse files

8241141: Restructure humongous object allocation in G1

Reviewed-by: tschatzl, kbarrett
  • Loading branch information
kstefanj committed Apr 8, 2020
1 parent dd4e04d commit 52ea4802f1aef656ffed7ff117d565742441cfe5
@@ -209,14 +209,15 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size,
}

HeapWord*
G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,
G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
uint num_regions,
size_t word_size) {
assert(first != G1_NO_HRM_INDEX, "pre-condition");
assert(first_hr != NULL, "pre-condition");
assert(is_humongous(word_size), "word_size should be humongous");
assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");

// Index of last region in the series.
uint first = first_hr->hrm_index();
uint last = first + num_regions - 1;

// We need to initialize the region(s) we just discovered. This is
@@ -231,10 +232,8 @@ G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,
size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
assert(word_size <= word_size_sum, "sanity");

// This will be the "starts humongous" region.
HeapRegion* first_hr = region_at(first);
// The header of the new object will be placed at the bottom of
// the first region.
// The passed in hr will be the "starts humongous" region. The header
// of the new object will be placed at the bottom of this region.
HeapWord* new_obj = first_hr->bottom();
// This will be the new top of the new object.
HeapWord* obj_top = new_obj + word_size;
@@ -340,57 +339,28 @@ HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) {

_verifier->verify_region_sets_optional();

uint first = G1_NO_HRM_INDEX;
uint obj_regions = (uint) humongous_obj_size_in_regions(word_size);

if (obj_regions == 1) {
// Only one region to allocate, try to use a fast path by directly allocating
// from the free lists. Do not try to expand here, we will potentially do that
// later.
HeapRegion* hr = new_region(word_size, HeapRegionType::Humongous, false /* do_expand */);
if (hr != NULL) {
first = hr->hrm_index();
}
} else {
// Policy: Try only empty regions (i.e. already committed first). Maybe we
// are lucky enough to find some.
first = _hrm->find_contiguous_only_empty(obj_regions);
if (first != G1_NO_HRM_INDEX) {
_hrm->allocate_free_regions_starting_at(first, obj_regions);
}
}

if (first == G1_NO_HRM_INDEX) {
// Policy: First try to allocate a humongous object in the free list.
HeapRegion* humongous_start = _hrm->allocate_humongous(obj_regions);
if (humongous_start == NULL) {
// Policy: We could not find enough regions for the humongous object in the
// free list. Look through the heap to find a mix of free and uncommitted regions.
// If so, try expansion.
first = _hrm->find_contiguous_empty_or_unavailable(obj_regions);
if (first != G1_NO_HRM_INDEX) {
// We found something. Make sure these regions are committed, i.e. expand
// the heap. Alternatively we could do a defragmentation GC.
log_debug(gc, ergo, heap)("Attempt heap expansion (humongous allocation request failed). Allocation request: " SIZE_FORMAT "B",
word_size * HeapWordSize);

_hrm->expand_at(first, obj_regions, workers());
// If so, expand the heap and allocate the humongous object.
humongous_start = _hrm->expand_and_allocate_humongous(obj_regions);
if (humongous_start != NULL) {
// We managed to find a region by expanding the heap.
log_debug(gc, ergo, heap)("Heap expansion (humongous allocation request). Allocation request: " SIZE_FORMAT "B",
word_size * HeapWordSize);
policy()->record_new_heap_size(num_regions());

#ifdef ASSERT
for (uint i = first; i < first + obj_regions; ++i) {
HeapRegion* hr = region_at(i);
assert(hr->is_free(), "sanity");
assert(hr->is_empty(), "sanity");
assert(is_on_master_free_list(hr), "sanity");
}
#endif
_hrm->allocate_free_regions_starting_at(first, obj_regions);
} else {
// Policy: Potentially trigger a defragmentation GC.
}
}

HeapWord* result = NULL;
if (first != G1_NO_HRM_INDEX) {
result = humongous_obj_allocate_initialize_regions(first, obj_regions, word_size);
if (humongous_start != NULL) {
result = humongous_obj_allocate_initialize_regions(humongous_start, obj_regions, word_size);
assert(result != NULL, "it should always return a valid result");

// A successful humongous object allocation changes the used space
@@ -4892,8 +4862,7 @@ HeapRegion* G1CollectedHeap::alloc_highest_free_region() {
log_debug(gc, ergo, heap)("Attempt heap expansion (requested address range outside heap bounds). region size: " SIZE_FORMAT "B",
HeapRegion::GrainWords * HeapWordSize);
}
_hrm->allocate_free_regions_starting_at(index, 1);
return region_at(index);
return _hrm->allocate_free_regions_starting_at(index, 1);
}
return NULL;
}
@@ -409,7 +409,7 @@ class G1CollectedHeap : public CollectedHeap {
// Initialize a contiguous set of free regions of length num_regions
// and starting at index first so that they appear as a single
// humongous region.
HeapWord* humongous_obj_allocate_initialize_regions(uint first,
HeapWord* humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
uint num_regions,
size_t word_size);

@@ -135,6 +135,35 @@ HeapRegion* HeapRegionManager::allocate_free_region(HeapRegionType type, uint re
return hr;
}

HeapRegion* HeapRegionManager::allocate_humongous_from_free_list(uint num_regions) {
uint candidate = find_contiguous_in_free_list(num_regions);
if (candidate == G1_NO_HRM_INDEX) {
return NULL;
}
return allocate_free_regions_starting_at(candidate, num_regions);
}

HeapRegion* HeapRegionManager::allocate_humongous_allow_expand(uint num_regions) {
uint candidate = find_contiguous_allow_expand(num_regions);
if (candidate == G1_NO_HRM_INDEX) {
return NULL;
}
expand_exact(candidate, num_regions, G1CollectedHeap::heap()->workers());
return allocate_free_regions_starting_at(candidate, num_regions);
}

HeapRegion* HeapRegionManager::allocate_humongous(uint num_regions) {
// Special case a single region to avoid expensive search.
if (num_regions == 1) {
return allocate_free_region(HeapRegionType::Humongous, G1NUMA::AnyNodeIndex);
}
return allocate_humongous_from_free_list(num_regions);
}

HeapRegion* HeapRegionManager::expand_and_allocate_humongous(uint num_regions) {
return allocate_humongous_allow_expand(num_regions);
}

#ifdef ASSERT
bool HeapRegionManager::is_free(HeapRegion* hr) const {
return _free_list.contains(hr);
@@ -271,6 +300,19 @@ uint HeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretou
return expanded;
}

void HeapRegionManager::expand_exact(uint start, uint num_regions, WorkGang* pretouch_workers) {
assert(num_regions != 0, "Need to request at least one region");
uint end = start + num_regions;

for (uint i = start; i < end; i++) {
if (!is_available(i)) {
make_regions_available(i, 1, pretouch_workers);
}
}

verify_optional();
}

uint HeapRegionManager::expand_on_preferred_node(uint preferred_index) {
uint expand_candidate = UINT_MAX;
for (uint i = 0; i < max_length(); i++) {
@@ -291,7 +333,7 @@ uint HeapRegionManager::expand_on_preferred_node(uint preferred_index) {
return 0;
}

make_regions_available(expand_candidate, 1, NULL);
expand_exact(expand_candidate, 1, NULL);
return 1;
}

@@ -300,36 +342,61 @@ bool HeapRegionManager::is_on_preferred_index(uint region_index, uint preferred_
return region_node_index == preferred_node_index;
}

uint HeapRegionManager::find_contiguous(size_t num, bool empty_only) {
uint found = 0;
size_t length_found = 0;
uint cur = 0;

while (length_found < num && cur < max_length()) {
HeapRegion* hr = _regions.get_by_index(cur);
if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) {
// This region is a potential candidate for allocation into.
length_found++;
} else {
// This region is not a candidate. The next region is the next possible one.
found = cur + 1;
length_found = 0;
void HeapRegionManager::guarantee_contiguous_range(uint start, uint num_regions) {
// General sanity check, regions found should either be available and empty
// or not available so that we can make them available and use them.
for (uint i = start; i < (start + num_regions); i++) {
HeapRegion* hr = _regions.get_by_index(i);
guarantee(!is_available(i) || hr->is_free(),
"Found region sequence starting at " UINT32_FORMAT ", length " UINT32_FORMAT
" that is not free at " UINT32_FORMAT ". Hr is " PTR_FORMAT ", type is %s",
start, num_regions, i, p2i(hr), hr->get_type_str());
}
}

uint HeapRegionManager::find_contiguous_in_range(uint start, uint end, uint num_regions) {
assert(start <= end, "precondition");
assert(num_regions >= 1, "precondition");
uint candidate = start; // First region in candidate sequence.
uint unchecked = candidate; // First unchecked region in candidate.
// While the candidate sequence fits in the range...
while (num_regions <= (end - candidate)) {
// Walk backward over the regions for the current candidate.
for (uint i = candidate + num_regions - 1; true; --i) {
if (is_available(i) && !at(i)->is_free()) {
// Region i can't be used, so restart with i+1 as the start
// of a new candidate sequence, and with the region after the
// old candidate sequence being the first unchecked region.
unchecked = candidate + num_regions;
candidate = i + 1;
break;
} else if (i == unchecked) {
// All regions of candidate sequence have passed check.
guarantee_contiguous_range(candidate, num_regions);
return candidate;
}
}
cur++;
}
return G1_NO_HRM_INDEX;
}

if (length_found == num) {
for (uint i = found; i < (found + num); i++) {
HeapRegion* hr = _regions.get_by_index(i);
// sanity check
guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()),
"Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT
" that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr));
}
return found;
} else {
return G1_NO_HRM_INDEX;
}
uint HeapRegionManager::find_contiguous_in_free_list(uint num_regions) {
BitMap::idx_t range_start = 0;
BitMap::idx_t range_end = range_start;
uint candidate = G1_NO_HRM_INDEX;

do {
range_start = _available_map.get_next_one_offset(range_end);
range_end = _available_map.get_next_zero_offset(range_start);
candidate = find_contiguous_in_range((uint) range_start, (uint) range_end, num_regions);
} while (candidate == G1_NO_HRM_INDEX && range_end < max_length());

return candidate;
}

uint HeapRegionManager::find_contiguous_allow_expand(uint num_regions) {
// Find any candidate.
return find_contiguous_in_range(0, max_length(), num_regions);
}

HeapRegion* HeapRegionManager::next_region_in_heap(const HeapRegion* r) const {
@@ -94,11 +94,19 @@ class HeapRegionManager: public CHeapObj<mtGC> {
// Notify other data structures about change in the heap layout.
void update_committed_space(HeapWord* old_end, HeapWord* new_end);

// Find a contiguous set of empty or uncommitted regions of length num and return
// Find a contiguous set of empty or uncommitted regions of length num_regions and return
// the index of the first region or G1_NO_HRM_INDEX if the search was unsuccessful.
// If only_empty is true, only empty regions are considered.
// Searches from bottom to top of the heap, doing a first-fit.
uint find_contiguous(size_t num, bool only_empty);
// Start and end defines the range to seek in, policy is first-fit.
uint find_contiguous_in_range(uint start, uint end, uint num_regions);
// Find a contiguous set of empty regions of length num_regions. Returns the start index
// of that set, or G1_NO_HRM_INDEX.
uint find_contiguous_in_free_list(uint num_regions);
// Find a contiguous set of empty or unavailable regions of length num_regions. Returns the
// start index of that set, or G1_NO_HRM_INDEX.
uint find_contiguous_allow_expand(uint num_regions);

void guarantee_contiguous_range(uint start, uint num_regions) ;

// Finds the next sequence of unavailable regions starting from start_idx. Returns the
// length of the sequence found. If this result is zero, no such sequence could be found,
// otherwise res_idx indicates the start index of these regions.
@@ -122,6 +130,14 @@ class HeapRegionManager: public CHeapObj<mtGC> {
void uncommit_regions(uint index, size_t num_regions = 1);
// Allocate a new HeapRegion for the given index.
HeapRegion* new_heap_region(uint hrm_index);

// Humongous allocation helpers
virtual HeapRegion* allocate_humongous_from_free_list(uint num_regions);
virtual HeapRegion* allocate_humongous_allow_expand(uint num_regions);

// Expand helper for cases when the regions to expand are well defined.
void expand_exact(uint start, uint num_regions, WorkGang* pretouch_workers);

#ifdef ASSERT
public:
bool is_free(HeapRegion* hr) const;
@@ -183,7 +199,13 @@ class HeapRegionManager: public CHeapObj<mtGC> {
// Allocate a free region with specific node index. If fails allocate with next node index.
virtual HeapRegion* allocate_free_region(HeapRegionType type, uint requested_node_index);

inline void allocate_free_regions_starting_at(uint first, uint num_regions);
// Allocate a humongous object from the free list
HeapRegion* allocate_humongous(uint num_regions);

// Allocate a humongous object by expanding the heap
HeapRegion* expand_and_allocate_humongous(uint num_regions);

inline HeapRegion* allocate_free_regions_starting_at(uint first, uint num_regions);

// Remove all regions from the free list.
void remove_all_free_regions() {
@@ -233,13 +255,6 @@ class HeapRegionManager: public CHeapObj<mtGC> {
// Try to expand on the given node index.
virtual uint expand_on_preferred_node(uint node_index);

// Find a contiguous set of empty regions of length num. Returns the start index of
// that set, or G1_NO_HRM_INDEX.
virtual uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); }
// Find a contiguous set of empty or unavailable regions of length num. Returns the
// start index of that set, or G1_NO_HRM_INDEX.
virtual uint find_contiguous_empty_or_unavailable(size_t num) { return find_contiguous(num, false); }

HeapRegion* next_region_in_heap(const HeapRegion* r) const;

// Find the highest free or uncommitted region in the reserved heap,
@@ -73,8 +73,10 @@ inline void HeapRegionManager::insert_into_free_list(HeapRegion* hr) {
_free_list.add_ordered(hr);
}

inline void HeapRegionManager::allocate_free_regions_starting_at(uint first, uint num_regions) {
_free_list.remove_starting_at(at(first), num_regions);
inline HeapRegion* HeapRegionManager::allocate_free_regions_starting_at(uint first, uint num_regions) {
HeapRegion* start = at(first);
_free_list.remove_starting_at(start, num_regions);
return start;
}

#endif // SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
@@ -229,7 +229,7 @@ class FreeRegionList : public HeapRegionSetBase {

// Remove all (contiguous) regions from first to first + num_regions -1 from
// this list.
// Num_regions must be > 1.
// Num_regions must be >= 1.
void remove_starting_at(HeapRegion* first, uint num_regions);

virtual void verify();
@@ -325,18 +325,28 @@ HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType
return hr;
}

uint HeterogeneousHeapRegionManager::find_contiguous_only_empty(size_t num) {
HeapRegion* HeterogeneousHeapRegionManager::allocate_humongous_from_free_list(uint num_regions) {
if (has_borrowed_regions()) {
return G1_NO_HRM_INDEX;
return NULL;
}
return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, true);
uint candidate = find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, true);
if (candidate == G1_NO_HRM_INDEX) {
return NULL;
}
return allocate_free_regions_starting_at(candidate, num_regions);
}

uint HeterogeneousHeapRegionManager::find_contiguous_empty_or_unavailable(size_t num) {
HeapRegion* HeterogeneousHeapRegionManager::allocate_humongous_allow_expand(uint num_regions) {
if (has_borrowed_regions()) {
return G1_NO_HRM_INDEX;
return NULL;
}
uint candidate = find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, false);
if (candidate == G1_NO_HRM_INDEX) {
return NULL;
}
return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, false);

expand_exact(candidate, num_regions, NULL);
return allocate_free_regions_starting_at(candidate, num_regions);
}

uint HeterogeneousHeapRegionManager::find_contiguous(size_t start, size_t end, size_t num, bool empty_only) {

0 comments on commit 52ea480

Please sign in to comment.