Skip to content

Commit

Permalink
8191565: Last-ditch Full GC should also move humongous objects
Browse files Browse the repository at this point in the history
Reviewed-by: tschatzl, sjohanss
  • Loading branch information
Ivan Walulya committed Mar 16, 2023
1 parent f629152 commit 96889bf
Show file tree
Hide file tree
Showing 15 changed files with 354 additions and 79 deletions.
142 changes: 82 additions & 60 deletions src/hotspot/share/gc/g1/g1CollectedHeap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -193,80 +193,56 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size,
return res;
}

HeapWord*
G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
uint num_regions,
size_t word_size) {
assert(first_hr != NULL, "pre-condition");
assert(is_humongous(word_size), "word_size should be humongous");
assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");

// Index of last region in the series.
uint first = first_hr->hrm_index();
uint last = first + num_regions - 1;

// We need to initialize the region(s) we just discovered. This is
// a bit tricky given that it can happen concurrently with
// refinement threads refining cards on these regions and
// potentially wanting to refine the BOT as they are scanning
// those cards (this can happen shortly after a cleanup; see CR
// 6991377). So we have to set up the region(s) carefully and in
// a specific order.

// The word size sum of all the regions we will allocate.
size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
void G1CollectedHeap::set_humongous_metadata(HeapRegion* first_hr,
uint num_regions,
size_t word_size,
bool update_remsets) {
// Calculate the new top of the humongous object.
HeapWord* obj_top = first_hr->bottom() + word_size;
// The word size sum of all the regions used
size_t word_size_sum = num_regions * HeapRegion::GrainWords;
assert(word_size <= word_size_sum, "sanity");

// The passed in hr will be the "starts humongous" region. The header
// of the new object will be placed at the bottom of this region.
HeapWord* new_obj = first_hr->bottom();
// This will be the new top of the new object.
HeapWord* obj_top = new_obj + word_size;

// First, we need to zero the header of the space that we will be
// allocating. When we update top further down, some refinement
// threads might try to scan the region. By zeroing the header we
// ensure that any thread that will try to scan the region will
// come across the zero klass word and bail out.
//
// NOTE: It would not have been correct to have used
// CollectedHeap::fill_with_object() and make the space look like
// an int array. The thread that is doing the allocation will
// later update the object header to a potentially different array
// type and, for a very short period of time, the klass and length
// fields will be inconsistent. This could cause a refinement
// thread to calculate the object size incorrectly.
Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
// How many words memory we "waste" which cannot hold a filler object.
size_t words_not_fillable = 0;

// Next, pad out the unused tail of the last region with filler
// Pad out the unused tail of the last region with filler
// objects, for improved usage accounting.
// How many words we use for filler objects.
size_t word_fill_size = word_size_sum - word_size;

// How many words memory we "waste" which cannot hold a filler object.
size_t words_not_fillable = 0;
// How many words can we use for filler objects.
size_t words_fillable = word_size_sum - word_size;

if (word_fill_size >= min_fill_size()) {
fill_with_objects(obj_top, word_fill_size);
} else if (word_fill_size > 0) {
if (words_fillable >= G1CollectedHeap::min_fill_size()) {
G1CollectedHeap::fill_with_objects(obj_top, words_fillable);
} else {
// We have space to fill, but we cannot fit an object there.
words_not_fillable = word_fill_size;
word_fill_size = 0;
words_not_fillable = words_fillable;
words_fillable = 0;
}

// We will set up the first region as "starts humongous". This
// will also update the BOT covering all the regions to reflect
// that there is a single object that starts at the bottom of the
// first region.
first_hr->set_starts_humongous(obj_top, word_fill_size);
_policy->remset_tracker()->update_at_allocate(first_hr);
// Then, if there are any, we will set up the "continues
// humongous" regions.
HeapRegion* hr = NULL;
first_hr->hr_clear(false /* clear_space */);
first_hr->set_starts_humongous(obj_top, words_fillable);

if (update_remsets) {
_policy->remset_tracker()->update_at_allocate(first_hr);
}

// Indices of first and last regions in the series.
uint first = first_hr->hrm_index();
uint last = first + num_regions - 1;

HeapRegion* hr = nullptr;
for (uint i = first + 1; i <= last; ++i) {
hr = region_at(i);
hr->hr_clear(false /* clear_space */);
hr->set_continues_humongous(first_hr);
_policy->remset_tracker()->update_at_allocate(hr);
if (update_remsets) {
_policy->remset_tracker()->update_at_allocate(hr);
}
}

// Up to this point no concurrent thread would have been able to
Expand Down Expand Up @@ -297,11 +273,57 @@ G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
assert(words_not_fillable == 0 ||
first_hr->bottom() + word_size_sum - words_not_fillable == hr->top(),
"Miscalculation in humongous allocation");
}

HeapWord*
G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
uint num_regions,
size_t word_size) {
assert(first_hr != NULL, "pre-condition");
assert(is_humongous(word_size), "word_size should be humongous");
assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");

// Index of last region in the series.
uint first = first_hr->hrm_index();
uint last = first + num_regions - 1;

// We need to initialize the region(s) we just discovered. This is
// a bit tricky given that it can happen concurrently with
// refinement threads refining cards on these regions and
// potentially wanting to refine the BOT as they are scanning
// those cards (this can happen shortly after a cleanup; see CR
// 6991377). So we have to set up the region(s) carefully and in
// a specific order.

increase_used((word_size_sum - words_not_fillable) * HeapWordSize);
// The passed in hr will be the "starts humongous" region. The header
// of the new object will be placed at the bottom of this region.
HeapWord* new_obj = first_hr->bottom();

// First, we need to zero the header of the space that we will be
// allocating. When we update top further down, some refinement
// threads might try to scan the region. By zeroing the header we
// ensure that any thread that will try to scan the region will
// come across the zero klass word and bail out.
//
// NOTE: It would not have been correct to have used
// CollectedHeap::fill_with_object() and make the space look like
// an int array. The thread that is doing the allocation will
// later update the object header to a potentially different array
// type and, for a very short period of time, the klass and length
// fields will be inconsistent. This could cause a refinement
// thread to calculate the object size incorrectly.
Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);

// Next, update the metadata for the regions.
set_humongous_metadata(first_hr, num_regions, word_size, true);

HeapRegion* last_hr = region_at(last);
size_t used = byte_size(first_hr->bottom(), last_hr->top());

increase_used(used);

for (uint i = first; i <= last; ++i) {
hr = region_at(i);
HeapRegion *hr = region_at(i);
_humongous_set.add(hr);
_hr_printer.alloc(hr);
}
Expand Down
5 changes: 5 additions & 0 deletions src/hotspot/share/gc/g1/g1CollectedHeap.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,11 @@ class G1CollectedHeap : public CollectedHeap {
// Register the given region to be part of the collection set.
inline void register_humongous_candidate_region_with_region_attr(uint index);

void set_humongous_metadata(HeapRegion* first_hr,
uint num_regions,
size_t word_size,
bool update_remsets);

// We register a region with the fast "in collection set" test. We
// simply set to true the array slot corresponding to this region.
void register_young_region_with_region_attr(HeapRegion* r) {
Expand Down
48 changes: 47 additions & 1 deletion src/hotspot/share/gc/g1/g1FullCollector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/verifyOption.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
Expand Down Expand Up @@ -119,12 +119,15 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
_scope(heap->monitoring_support(), explicit_gc, clear_soft_refs, do_maximal_compaction, tracer),
_num_workers(calc_active_workers()),
_has_compaction_targets(false),
_has_humongous(false),
_oop_queue_set(_num_workers),
_array_queue_set(_num_workers),
_preserved_marks_set(true),
_serial_compaction_point(this),
_humongous_compaction_point(this),
_is_alive(this, heap->concurrent_mark()->mark_bitmap()),
_is_alive_mutator(heap->ref_processor_stw(), &_is_alive),
_humongous_compaction_regions(8),
_always_subject_to_discovery(),
_is_subject_mutator(heap->ref_processor_stw(), &_always_subject_to_discovery),
_region_attr_table() {
Expand Down Expand Up @@ -155,6 +158,7 @@ G1FullCollector::~G1FullCollector() {
delete _markers[i];
delete _compaction_points[i];
}

FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers);
FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points);
FREE_C_HEAP_ARRAY(HeapWord*, _compaction_tops);
Expand Down Expand Up @@ -246,6 +250,8 @@ void G1FullCollector::complete_collection() {
_heap->gc_epilogue(true);

_heap->verify_after_full_collection();

_heap->print_heap_after_full_collection();
}

void G1FullCollector::before_marking_update_attribute_table(HeapRegion* hr) {
Expand Down Expand Up @@ -343,6 +349,12 @@ void G1FullCollector::phase2_prepare_compaction() {
// maximally compact the tail regions of the compaction queues serially.
if (scope()->do_maximal_compaction() || !has_free_compaction_targets) {
phase2c_prepare_serial_compaction();

if (scope()->do_maximal_compaction() &&
has_humongous() &&
serial_compaction_point()->has_regions()) {
phase2d_prepare_humongous_compaction();
}
}
}

Expand Down Expand Up @@ -418,6 +430,35 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
serial_cp->update();
}

void G1FullCollector::phase2d_prepare_humongous_compaction() {
GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare humongous compaction", scope()->timer());
G1FullGCCompactionPoint* serial_cp = serial_compaction_point();
assert(serial_cp->has_regions(), "Sanity!" );

uint last_serial_target = serial_cp->current_region()->hrm_index();
uint region_index = last_serial_target + 1;
uint max_reserved_regions = _heap->max_reserved_regions();

G1FullGCCompactionPoint* humongous_cp = humongous_compaction_point();

while (region_index < max_reserved_regions) {
HeapRegion* hr = _heap->region_at_or_null(region_index);

if (hr == nullptr) {
region_index++;
continue;
} else if (hr->is_starts_humongous()) {
uint num_regions = humongous_cp->forward_humongous(hr);
region_index += num_regions; // Skip over the continues humongous regions.
continue;
} else if (is_compaction_target(region_index)) {
// Add the region to the humongous compaction point.
humongous_cp->add(hr);
}
region_index++;
}
}

void G1FullCollector::phase3_adjust_pointers() {
// Adjust the pointers to reflect the new locations
GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer());
Expand All @@ -436,6 +477,11 @@ void G1FullCollector::phase4_do_compaction() {
if (serial_compaction_point()->has_regions()) {
task.serial_compaction();
}

if (!_humongous_compaction_regions.is_empty()) {
assert(scope()->do_maximal_compaction(), "Only compact humongous during maximal compaction");
task.humongous_compaction();
}
}

void G1FullCollector::phase5_reset_metadata() {
Expand Down
12 changes: 12 additions & 0 deletions src/hotspot/share/gc/g1/g1FullCollector.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,18 @@ class G1FullCollector : StackObj {
G1FullGCScope _scope;
uint _num_workers;
bool _has_compaction_targets;
bool _has_humongous;
G1FullGCMarker** _markers;
G1FullGCCompactionPoint** _compaction_points;
OopQueueSet _oop_queue_set;
ObjArrayTaskQueueSet _array_queue_set;
PreservedMarksSet _preserved_marks_set;
G1FullGCCompactionPoint _serial_compaction_point;
G1FullGCCompactionPoint _humongous_compaction_point;
G1IsAliveClosure _is_alive;
ReferenceProcessorIsAliveMutator _is_alive_mutator;
G1RegionMarkStats* _live_stats;
GrowableArrayCHeap<HeapRegion*, mtGC> _humongous_compaction_regions;

static uint calc_active_workers();

Expand Down Expand Up @@ -115,6 +118,7 @@ class G1FullCollector : StackObj {
ObjArrayTaskQueueSet* array_queue_set() { return &_array_queue_set; }
PreservedMarksSet* preserved_mark_set() { return &_preserved_marks_set; }
G1FullGCCompactionPoint* serial_compaction_point() { return &_serial_compaction_point; }
G1FullGCCompactionPoint* humongous_compaction_point() { return &_humongous_compaction_point; }
G1CMBitMap* mark_bitmap();
ReferenceProcessor* reference_processor();
size_t live_words(uint region_index) const {
Expand All @@ -134,22 +138,30 @@ class G1FullCollector : StackObj {
inline void set_free(uint region_idx);
inline bool is_free(uint region_idx) const;
inline void update_from_compacting_to_skip_compacting(uint region_idx);
inline void update_from_skip_compacting_to_compacting(uint region_idx);

inline void set_compaction_top(HeapRegion* r, HeapWord* value);
inline HeapWord* compaction_top(HeapRegion* r) const;

inline void set_has_compaction_targets();
inline bool has_compaction_targets() const;

inline void add_humongous_region(HeapRegion* hr);
inline GrowableArrayCHeap<HeapRegion*, mtGC>& humongous_compaction_regions();

uint truncate_parallel_cps();

inline void set_has_humongous();
inline bool has_humongous();

private:
void phase1_mark_live_objects();
void phase2_prepare_compaction();

void phase2a_determine_worklists();
bool phase2b_forward_oops();
void phase2c_prepare_serial_compaction();
void phase2d_prepare_humongous_compaction();

void phase3_adjust_pointers();
void phase4_do_compaction();
Expand Down
23 changes: 23 additions & 0 deletions src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ void G1FullCollector::update_from_compacting_to_skip_compacting(uint region_idx)
_region_attr_table.set_skip_compacting(region_idx);
}

void G1FullCollector::update_from_skip_compacting_to_compacting(uint region_idx) {
DEBUG_ONLY(_region_attr_table.verify_is_skip_compacting(region_idx);)
_region_attr_table.set_compacting(region_idx);
}

void G1FullCollector::set_compaction_top(HeapRegion* r, HeapWord* value) {
Atomic::store(&_compaction_tops[r->hrm_index()], value);
}
Expand All @@ -79,5 +84,23 @@ bool G1FullCollector::has_compaction_targets() const {
return _has_compaction_targets;
}

void G1FullCollector::set_has_humongous() {
if (!_has_humongous) {
_has_humongous = true;
}
}

bool G1FullCollector::has_humongous() {
return _has_humongous;
}

void G1FullCollector::add_humongous_region(HeapRegion* hr) {
_humongous_compaction_regions.append(hr);
}

GrowableArrayCHeap<HeapRegion*, mtGC>& G1FullCollector::humongous_compaction_regions() {
return _humongous_compaction_regions;
}

#endif // SHARE_GC_G1_G1FULLCOLLECTOR_INLINE_HPP

1 comment on commit 96889bf

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.