Skip to content
Permalink
Browse files

8165443: Free Collection Set serial phase takes very long on large heaps

Reviewed-by: tschatzl, kbarrett
  • Loading branch information
kstefanj committed Dec 9, 2019
1 parent 4c4d6cd commit 9cabfa82ffa4dec1e96644e853e2fb4edfa26759

Large diffs are not rendered by default.

@@ -1201,6 +1201,11 @@ class G1CollectedHeap : public CollectedHeap {
void heap_region_par_iterate_from_start(HeapRegionClosure* cl,
HeapRegionClaimer* hrclaimer) const;

// Iterate over all regions in the collection set in parallel.
void collection_set_par_iterate_all(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id);

// Iterate over all regions currently in the current collection set.
void collection_set_iterate_all(HeapRegionClosure* blk);

@@ -201,6 +201,13 @@ void G1CollectionSet::iterate(HeapRegionClosure* cl) const {
}
}

void G1CollectionSet::par_iterate(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id,
uint total_workers) const {
iterate_part_from(cl, hr_claimer, 0, cur_length(), worker_id, total_workers);
}

void G1CollectionSet::iterate_optional(HeapRegionClosure* cl) const {
assert_at_safepoint();

@@ -215,26 +222,33 @@ void G1CollectionSet::iterate_incremental_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id,
uint total_workers) const {
assert_at_safepoint();
iterate_part_from(cl, hr_claimer, _inc_part_start, increment_length(), worker_id, total_workers);
}

size_t len = increment_length();
if (len == 0) {
void G1CollectionSet::iterate_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
size_t offset,
size_t length,
uint worker_id,
uint total_workers) const {
assert_at_safepoint();
if (length == 0) {
return;
}

size_t start_pos = (worker_id * len) / total_workers;
size_t start_pos = (worker_id * length) / total_workers;
size_t cur_pos = start_pos;

do {
uint region_idx = _collection_set_regions[cur_pos + _inc_part_start];
uint region_idx = _collection_set_regions[cur_pos + offset];
if (hr_claimer == NULL || hr_claimer->claim_region(region_idx)) {
HeapRegion* r = _g1h->region_at(region_idx);
bool result = cl->do_heap_region(r);
guarantee(!result, "Must not cancel iteration");
}

cur_pos++;
if (cur_pos == len) {
if (cur_pos == length) {
cur_pos = 0;
}
} while (cur_pos != start_pos);
@@ -254,6 +254,16 @@ class G1CollectionSet {
// Select the old regions of the initial collection set and determine how many optional
// regions we might be able to evacuate in this pause.
void finalize_old_part(double time_remaining_ms);

// Iterate the part of the collection set given by the offset and length applying the given
// HeapRegionClosure. The worker_id will determine where in the part to start the iteration
// to allow for more efficient parallel iteration.
void iterate_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
size_t offset,
size_t length,
uint worker_id,
uint total_workers) const;
public:
G1CollectionSet(G1CollectedHeap* g1h, G1Policy* policy);
~G1CollectionSet();
@@ -306,6 +316,10 @@ class G1CollectionSet {
// Iterate over the entire collection set (all increments calculated so far), applying
// the given HeapRegionClosure on all of them.
void iterate(HeapRegionClosure* cl) const;
void par_iterate(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id,
uint total_workers) const;

void iterate_optional(HeapRegionClosure* cl) const;

@@ -131,8 +131,10 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
_gc_par_phases[RedirtyCards] = new WorkerDataArray<double>("Parallel Redirty (ms):", max_gc_threads);
_gc_par_phases[RedirtyCards]->create_thread_work_items("Redirtied Cards:");

_gc_par_phases[ParFreeCSet] = new WorkerDataArray<double>("Parallel Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[YoungFreeCSet] = new WorkerDataArray<double>("Young Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[NonYoungFreeCSet] = new WorkerDataArray<double>("Non-Young Free Collection Set (ms):", max_gc_threads);
_gc_par_phases[RebuildFreeList] = new WorkerDataArray<double>("Parallel Rebuild Free List (ms):", max_gc_threads);

reset();
}
@@ -167,6 +169,8 @@ void G1GCPhaseTimes::reset() {
_recorded_start_new_cset_time_ms = 0.0;
_recorded_total_free_cset_time_ms = 0.0;
_recorded_serial_free_cset_time_ms = 0.0;
_recorded_total_rebuild_freelist_time_ms = 0.0;
_recorded_serial_rebuild_freelist_time_ms = 0.0;
_cur_fast_reclaim_humongous_time_ms = 0.0;
_cur_region_register_time = 0.0;
_cur_fast_reclaim_humongous_total = 0;
@@ -328,11 +332,11 @@ void G1GCPhaseTimes::debug_phase(WorkerDataArray<double>* phase, uint extra_inde
}
}

void G1GCPhaseTimes::trace_phase(WorkerDataArray<double>* phase, bool print_sum) const {
void G1GCPhaseTimes::trace_phase(WorkerDataArray<double>* phase, bool print_sum, uint extra_indent) const {
LogTarget(Trace, gc, phases) lt;
if (lt.is_enabled()) {
LogStream ls(lt);
log_phase(phase, 3, &ls, print_sum);
log_phase(phase, 3 + extra_indent, &ls, print_sum);
}
}

@@ -456,6 +460,7 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
_cur_strong_code_root_purge_time_ms +
_recorded_redirty_logged_cards_time_ms +
_recorded_total_free_cset_time_ms +
_recorded_total_rebuild_freelist_time_ms +
_cur_fast_reclaim_humongous_time_ms +
_cur_expand_heap_time_ms +
_cur_string_deduplication_time_ms;
@@ -492,9 +497,14 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
#endif

debug_time("Free Collection Set", _recorded_total_free_cset_time_ms);
trace_time("Free Collection Set Serial", _recorded_serial_free_cset_time_ms);
trace_phase(_gc_par_phases[YoungFreeCSet]);
trace_phase(_gc_par_phases[NonYoungFreeCSet]);
trace_time("Serial Free Collection Set", _recorded_serial_free_cset_time_ms);
trace_phase(_gc_par_phases[ParFreeCSet]);
trace_phase(_gc_par_phases[YoungFreeCSet], true, 1);
trace_phase(_gc_par_phases[NonYoungFreeCSet], true, 1);

debug_time("Rebuild Free List", _recorded_total_rebuild_freelist_time_ms);
trace_time("Serial Rebuild Free List ", _recorded_serial_rebuild_freelist_time_ms);
trace_phase(_gc_par_phases[RebuildFreeList]);

if (G1EagerReclaimHumongousObjects) {
debug_time("Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms);
@@ -566,8 +576,10 @@ const char* G1GCPhaseTimes::phase_name(GCParPhases phase) {
"StringDedupQueueFixup",
"StringDedupTableFixup",
"RedirtyCards",
"ParFreeCSet",
"YoungFreeCSet",
"NonYoungFreeCSet",
"RebuildFreeList",
"MergePSS"
//GCParPhasesSentinel only used to tell end of enum
};
@@ -76,8 +76,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
StringDedupQueueFixup,
StringDedupTableFixup,
RedirtyCards,
ParFreeCSet,
YoungFreeCSet,
NonYoungFreeCSet,
RebuildFreeList,
MergePSS,
GCParPhasesSentinel
};
@@ -171,6 +173,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {

double _recorded_serial_free_cset_time_ms;

double _recorded_total_rebuild_freelist_time_ms;

double _recorded_serial_rebuild_freelist_time_ms;

double _cur_region_register_time;

double _cur_fast_reclaim_humongous_time_ms;
@@ -195,7 +201,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
void log_phase(WorkerDataArray<double>* phase, uint indent, outputStream* out, bool print_sum) const;
void debug_serial_phase(WorkerDataArray<double>* phase, uint extra_indent = 0) const;
void debug_phase(WorkerDataArray<double>* phase, uint extra_indent = 0) const;
void trace_phase(WorkerDataArray<double>* phase, bool print_sum = true) const;
void trace_phase(WorkerDataArray<double>* phase, bool print_sum = true, uint extra_indent = 0) const;

void info_time(const char* name, double value) const;
void debug_time(const char* name, double value) const;
@@ -318,6 +324,14 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
_recorded_serial_free_cset_time_ms = time_ms;
}

void record_total_rebuild_freelist_time_ms(double time_ms) {
_recorded_total_rebuild_freelist_time_ms = time_ms;
}

void record_serial_rebuild_freelist_time_ms(double time_ms) {
_recorded_serial_rebuild_freelist_time_ms = time_ms;
}

void record_register_regions(double time_ms, size_t total, size_t candidates) {
_cur_region_register_time = time_ms;
_cur_fast_reclaim_humongous_total = total;
@@ -401,6 +415,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
return _recorded_total_free_cset_time_ms;
}

double total_rebuild_freelist_time_ms() {
return _recorded_total_rebuild_freelist_time_ms;
}

double non_young_cset_choice_time_ms() {
return _recorded_non_young_cset_choice_time_ms;
}
@@ -587,7 +587,7 @@ double G1Policy::other_time_ms(double pause_time_ms) const {
}

double G1Policy::constant_other_time_ms(double pause_time_ms) const {
return other_time_ms(pause_time_ms) - phase_times()->total_free_cset_time_ms();
return other_time_ms(pause_time_ms) - phase_times()->total_free_cset_time_ms() - phase_times()->total_rebuild_freelist_time_ms();
}

bool G1Policy::about_to_start_mixed_phase() const {
@@ -110,6 +110,19 @@ void HeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_hea
}
}

void HeapRegion::handle_evacuation_failure() {
uninstall_surv_rate_group();
clear_young_index_in_cset();
set_evacuation_failed(false);
set_old();
}

void HeapRegion::unlink_from_list() {
set_next(NULL);
set_prev(NULL);
set_containing_set(NULL);
}

void HeapRegion::hr_clear(bool keep_remset, bool clear_space, bool locked) {
assert(_humongous_start_region == NULL,
"we should have already filtered out humongous regions");
@@ -464,14 +464,16 @@ class HeapRegion : public CHeapObj<mtGC> {
void set_prev(HeapRegion* prev) { _prev = prev; }
HeapRegion* prev() { return _prev; }

void unlink_from_list();

// Every region added to a set is tagged with a reference to that
// set. This is used for doing consistency checking to make sure that
// the contents of a set are as they should be and it's only
// available in non-product builds.
#ifdef ASSERT
void set_containing_set(HeapRegionSetBase* containing_set) {
assert((containing_set == NULL && _containing_set != NULL) ||
(containing_set != NULL && _containing_set == NULL),
assert((containing_set != NULL && _containing_set == NULL) ||
containing_set == NULL,
"containing_set: " PTR_FORMAT " "
"_containing_set: " PTR_FORMAT,
p2i(containing_set), p2i(_containing_set));
@@ -559,6 +561,9 @@ class HeapRegion : public CHeapObj<mtGC> {
return (HeapWord *) obj >= next_top_at_mark_start();
}

// Update the region state after a failed evacuation.
void handle_evacuation_failure();

// Iterate over the objects overlapping the given memory region, applying cl
// to all references in the region. This is a helper for
// G1RemSet::refine_card*, and is tightly coupled with them.
@@ -614,3 +614,80 @@ bool HeapRegionClaimer::claim_region(uint region_index) {
uint old_val = Atomic::cmpxchg(&_claims[region_index], Unclaimed, Claimed);
return old_val == Unclaimed;
}

class G1RebuildFreeListTask : public AbstractGangTask {
HeapRegionManager* _hrm;
FreeRegionList* _worker_freelists;
uint _worker_chunk_size;
uint _num_workers;

public:
G1RebuildFreeListTask(HeapRegionManager* hrm, uint num_workers) :
AbstractGangTask("G1 Rebuild Free List Task"),
_hrm(hrm),
_worker_freelists(NEW_C_HEAP_ARRAY(FreeRegionList, num_workers, mtGC)),
_worker_chunk_size((_hrm->max_length() + num_workers - 1) / num_workers),
_num_workers(num_workers) {
for (uint worker = 0; worker < _num_workers; worker++) {
::new (&_worker_freelists[worker]) FreeRegionList("Appendable Worker Free List");
}
}

~G1RebuildFreeListTask() {
for (uint worker = 0; worker < _num_workers; worker++) {
_worker_freelists[worker].~FreeRegionList();
}
FREE_C_HEAP_ARRAY(FreeRegionList, _worker_freelists);
}

FreeRegionList* worker_freelist(uint worker) {
return &_worker_freelists[worker];
}

// Each worker creates a free list for a chunk of the heap. The chunks won't
// be overlapping so we don't need to do any claiming.
void work(uint worker_id) {
Ticks start_time = Ticks::now();
EventGCPhaseParallel event;

uint start = worker_id * _worker_chunk_size;
uint end = MIN2(start + _worker_chunk_size, _hrm->max_length());

// If start is outside the heap, this worker has nothing to do.
if (start > end) {
return;
}

FreeRegionList *free_list = worker_freelist(worker_id);
for (uint i = start; i < end; i++) {
HeapRegion *region = _hrm->at_or_null(i);
if (region != NULL && region->is_free()) {
// Need to clear old links to allow to be added to new freelist.
region->unlink_from_list();
free_list->add_to_tail(region);
}
}

event.commit(GCId::current(), worker_id, G1GCPhaseTimes::phase_name(G1GCPhaseTimes::RebuildFreeList));
G1CollectedHeap::heap()->phase_times()->record_time_secs(G1GCPhaseTimes::RebuildFreeList, worker_id, (Ticks::now() - start_time).seconds());
}
};

void HeapRegionManager::rebuild_free_list(WorkGang* workers) {
// Abandon current free list to allow a rebuild.
_free_list.abandon();

uint const num_workers = clamp(max_length(), 1u, workers->active_workers());
G1RebuildFreeListTask task(this, num_workers);

log_debug(gc, ergo)("Running %s using %u workers for rebuilding free list of %u (%u) regions",
task.name(), num_workers, num_free_regions(), max_length());
workers->run_task(&task, num_workers);

// Link the partial free lists together.
Ticks serial_time = Ticks::now();
for (uint worker = 0; worker < num_workers; worker++) {
_free_list.append_ordered(task.worker_freelist(worker));
}
G1CollectedHeap::heap()->phase_times()->record_serial_rebuild_freelist_time_ms((Ticks::now() - serial_time).seconds() * 1000.0);
}
@@ -172,6 +172,9 @@ class HeapRegionManager: public CHeapObj<mtGC> {
// Insert the given region into the free region list.
inline void insert_into_free_list(HeapRegion* hr);

// Rebuild the free region list from scratch.
void rebuild_free_list(WorkGang* workers);

// Insert the given region list into the global free region list.
void insert_list_into_free_list(FreeRegionList* list) {
_free_list.add_ordered(list);

0 comments on commit 9cabfa8

Please sign in to comment.
You can’t perform that action at this time.