Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
8247845: Shenandoah: refactor TLAB/GCLAB retirement code
Reviewed-by: rkennke
  • Loading branch information
shipilev committed Jun 29, 2020
1 parent 5ad963c commit a793293464d2a457b92784bc2c1033c199044d12
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 58 deletions.
@@ -1044,28 +1044,79 @@ void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
}
}

class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
public:
ShenandoahCheckCleanGCLABClosure() {}
void do_thread(Thread* thread) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
}
};

class ShenandoahRetireGCLABClosure : public ThreadClosure {
private:
bool const _resize;
public:
ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
void do_thread(Thread* thread) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
gclab->retire();
if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
ShenandoahThreadLocalData::set_gclab_size(thread, 0);
}
}
};

void ShenandoahHeap::make_parsable(bool retire_tlabs) {
if (UseTLAB) {
CollectedHeap::ensure_parsability(retire_tlabs);
void ShenandoahHeap::labs_make_parsable() {
assert(UseTLAB, "Only call with UseTLAB");

ShenandoahRetireGCLABClosure cl(false);

for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
ThreadLocalAllocBuffer& tlab = t->tlab();
tlab.make_parsable();
cl.do_thread(t);
}
ShenandoahRetireGCLABClosure cl;

workers()->threads_do(&cl);
}

void ShenandoahHeap::tlabs_retire(bool resize) {
assert(UseTLAB, "Only call with UseTLAB");
assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

ThreadLocalAllocStats stats;

for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
ThreadLocalAllocBuffer& tlab = t->tlab();
tlab.retire(&stats);
if (resize) {
tlab.resize();
}
}

stats.publish();

#ifdef ASSERT
ShenandoahCheckCleanGCLABClosure cl;
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
cl.do_thread(t);
}
workers()->threads_do(&cl);
#endif
}

void ShenandoahHeap::resize_tlabs() {
CollectedHeap::resize_all_tlabs();
void ShenandoahHeap::gclabs_retire(bool resize) {
assert(UseTLAB, "Only call with UseTLAB");
assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");

ShenandoahRetireGCLABClosure cl(resize);
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
cl.do_thread(t);
}
workers()->threads_do(&cl);
}

class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
@@ -1122,25 +1173,6 @@ size_t ShenandoahHeap::max_tlab_size() const {
return ShenandoahHeapRegion::max_tlab_size_words();
}

class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
public:
void do_thread(Thread* thread) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
gclab->retire();
if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
ShenandoahThreadLocalData::set_gclab_size(thread, 0);
}
}
};

void ShenandoahHeap::retire_and_reset_gclabs() {
ShenandoahRetireAndResetGCLABClosure cl;
for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
cl.do_thread(t);
}
workers()->threads_do(&cl);
}

void ShenandoahHeap::collect(GCCause::Cause cause) {
control_thread()->request_gc(cause);
}
@@ -1173,8 +1205,8 @@ jlong ShenandoahHeap::millis_since_last_gc() {
}

void ShenandoahHeap::prepare_for_verify() {
if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
make_parsable(false);
if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
labs_make_parsable();
}
}

@@ -1264,7 +1296,7 @@ class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
* This is public API, used in preparation of object_iterate().
* Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
* need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
* control, we call SH::make_tlabs_parsable().
* control, we call SH::tlabs_retire, SH::gclabs_retire.
*/
void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
// No-op.
@@ -1420,10 +1452,13 @@ void ShenandoahHeap::op_init_mark() {
}

set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
make_parsable(true);

// We need to reset all TLABs because they might be below the TAMS, and we need to mark
// the objects in them. Do not let mutators allocate any new objects in their current TLABs.
// It is also a good place to resize the TLAB sizes for future allocations.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_manage_tlabs);
tlabs_retire(ResizeTLAB);
}

{
@@ -1437,11 +1472,6 @@ void ShenandoahHeap::op_init_mark() {

concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);

if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
resize_tlabs();
}

if (ShenandoahPacing) {
pacer()->setup_for_mark();
}
@@ -1539,9 +1569,9 @@ void ShenandoahHeap::op_final_mark() {
// which would be outside the collection set, so no cset writes would happen there.
// Weaker one: new allocations would happen past update watermark, and so less work would
// be needed for reference updates (would update the large filler instead).
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
make_parsable(true);
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_manage_labs);
tlabs_retire(false);
}

{
@@ -1917,10 +1947,6 @@ void ShenandoahHeap::op_full(GCCause::Cause cause) {
metrics.snap_before();

full_gc()->do_it(cause);
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
resize_all_tlabs();
}

metrics.snap_after();

@@ -2511,9 +2537,12 @@ void ShenandoahHeap::op_init_updaterefs() {

set_evacuation_in_progress(false);

{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
retire_and_reset_gclabs();
// Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
// make them parsable for update code to work correctly. Plus, we can compute new sizes
// for future GCLABs here.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_manage_gclabs);
gclabs_retire(true);
}

if (ShenandoahVerify) {
@@ -595,7 +595,6 @@ class ShenandoahHeap : public CollectedHeap {
inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
void retire_and_reset_gclabs();

public:
HeapWord* allocate_memory(ShenandoahAllocRequest& request);
@@ -615,10 +614,11 @@ class ShenandoahHeap : public CollectedHeap {
size_t max_tlab_size() const;
size_t tlab_used(Thread* ignored) const;

void resize_tlabs();
void ensure_parsability(bool retire_labs);

void ensure_parsability(bool retire_tlabs);
void make_parsable(bool retire_tlabs);
void labs_make_parsable();
void tlabs_retire(bool resize);
void gclabs_retire(bool resize);

// ---------- Marking support
//
@@ -145,7 +145,10 @@ void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
_preserved_marks->init(heap->workers()->active_workers());
}

heap->make_parsable(true);
if (UseTLAB) {
heap->gclabs_retire(true);
heap->tlabs_retire(true);
}

OrderAccess::fence();

@@ -60,11 +60,10 @@ class outputStream;
\
f(init_mark_gross, "Pause Init Mark (G)") \
f(init_mark, "Pause Init Mark (N)") \
f(make_parsable, " Make Parsable") \
f(init_manage_tlabs, " Manage TLABs") \
f(init_update_region_states, " Update Region States") \
f(scan_roots, " Scan Roots") \
SHENANDOAH_PAR_PHASE_DO(scan_, " S: ", f) \
f(resize_tlabs, " Resize TLABs") \
\
f(conc_mark, "Concurrent Marking") \
f(conc_mark_roots, " Roots ") \
@@ -86,7 +85,7 @@ class outputStream;
SHENANDOAH_PAR_PHASE_DO(purge_weak_par_, " WR: ", f) \
f(purge_cldg, " CLDG") \
f(final_update_region_states, " Update Region States") \
f(retire_tlabs, " Retire TLABs") \
f(final_manage_labs, " Manage GC/TLABs") \
f(choose_cset, " Choose Collection Set") \
f(final_rebuild_freeset, " Rebuild Free Set") \
f(init_evac, " Initial Evacuation") \
@@ -113,7 +112,7 @@ class outputStream;
\
f(init_update_refs_gross, "Pause Init Update Refs (G)") \
f(init_update_refs, "Pause Init Update Refs (N)") \
f(init_update_refs_retire_gclabs, " Retire GCLABs") \
f(init_update_refs_manage_gclabs, " Manage GCLABs") \
\
f(conc_update_refs, "Concurrent Update Refs") \
\
@@ -166,7 +165,6 @@ class outputStream;
f(full_gc_copy_objects_humong, " Humongous Objects") \
f(full_gc_copy_objects_reset_complete, " Reset Complete Bitmap") \
f(full_gc_copy_objects_rebuild, " Rebuild Region Sets") \
f(full_gc_resize_tlabs, " Resize TLABs") \
f(full_gc_heapdump_post, " Post Heap Dump") \
\
f(conc_uncommit, "Concurrent Uncommit") \
@@ -690,7 +690,10 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label,
}

OrderAccess::fence();
_heap->make_parsable(false);

if (UseTLAB) {
_heap->labs_make_parsable();
}

// Allocate temporary bitmap for storing marking wavefront:
_verification_bit_map->clear();

0 comments on commit a793293

Please sign in to comment.