Skip to content
Permalink
Browse files

8241139: Shenandoah: distribute mark-compact work exactly to minimize…

… fragmentation

Reviewed-by: rkennke
  • Loading branch information
shipilev committed Mar 23, 2020
1 parent fe53372 commit b6783320ca615a0407f364f6f2bfecf5f0f63527
@@ -332,57 +332,46 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
PreservedMarksSet* const _preserved_marks;
ShenandoahHeap* const _heap;
ShenandoahHeapRegionSet** const _worker_slices;
ShenandoahRegionIterator _heap_regions;

ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
ShenandoahHeapRegion* from_region = _heap_regions.next();

// Look for next candidate for this slice:
while (from_region != NULL) {
// Empty region: get it into the slice to defragment the slice itself.
// We could have skipped this without violating correctness, but we really
// want to compact all live regions to the start of the heap, which sometimes
// means moving them into the fully empty regions.
if (from_region->is_empty()) break;

// Can move the region, and this is not the humongous region. Humongous
// moves are special cased here, because their moves are handled separately.
if (from_region->is_stw_move_allowed() && !from_region->is_humongous()) break;

from_region = _heap_regions.next();
}

if (from_region != NULL) {
assert(slice != NULL, "sanity");
assert(!from_region->is_humongous(), "this path cannot handle humongous regions");
assert(from_region->is_empty() || from_region->is_stw_move_allowed(), "only regions that can be moved in mark-compact");
slice->add_region(from_region);
}

return from_region;
}

public:
ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) :
AbstractGangTask("Shenandoah Prepare For Compaction Task"),
_preserved_marks(preserved_marks),
_heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
}

static bool is_candidate_region(ShenandoahHeapRegion* r) {
// Empty region: get it into the slice to defragment the slice itself.
// We could have skipped this without violating correctness, but we really
// want to compact all live regions to the start of the heap, which sometimes
// means moving them into the fully empty regions.
if (r->is_empty()) return true;

// Can move the region, and this is not the humongous region. Humongous
// moves are special cased here, because their moves are handled separately.
return r->is_stw_move_allowed() && !r->is_humongous();
}

void work(uint worker_id) {
ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
ShenandoahHeapRegion* from_region = next_from_region(slice);
ShenandoahHeapRegionSetIterator it(slice);
ShenandoahHeapRegion* from_region = it.next();
// No work?
if (from_region == NULL) {
return;
return;
}

// Sliding compaction. Walk all regions in the slice, and compact them.
// Remember empty regions and reuse them as needed.
ResourceMark rm;

GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());

ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);

while (from_region != NULL) {
assert(is_candidate_region(from_region), "Sanity");

cl.set_from_region(from_region);
if (from_region->has_live()) {
_heap->marked_object_iterate(from_region, &cl);
@@ -392,7 +381,7 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
if (!cl.is_compact_same_region()) {
empty_regions.append(from_region);
}
from_region = next_from_region(slice);
from_region = it.next();
}
cl.finish_region();

@@ -509,6 +498,148 @@ class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure
}
};

void ShenandoahMarkCompact::distribute_slices(ShenandoahHeapRegionSet** worker_slices) {
ShenandoahHeap* heap = ShenandoahHeap::heap();

uint n_workers = heap->workers()->active_workers();
size_t n_regions = heap->num_regions();

// What we want to accomplish: have the dense prefix of data, while still balancing
// out the parallel work.
//
// Assuming the amount of work is driven by the live data that needs moving, we can slice
// the entire heap into equal-live-sized prefix slices, and compact into them. So, each
// thread takes all regions in its prefix subset, and then it takes some regions from
// the tail.
//
// Tail region selection becomes interesting.
//
// First, we want to distribute the regions fairly between the workers, and those regions
// might have different amount of live data. So, until we sure no workers need live data,
// we need to only take what the worker needs.
//
// Second, since we slide everything to the left in each slice, the most busy regions
// would be the ones on the left. Which means we want to have all workers have their after-tail
// regions as close to the left as possible.
//
// The easiest way to do this is to distribute after-tail regions in round-robin between
// workers that still need live data.
//
// Consider parallel workers A, B, C, then the target slice layout would be:
//
// AAAAAAAABBBBBBBBCCCCCCCC|ABCABCABCABCABCABCABCABABABABABABABABABABAAAAA
//
// (.....dense-prefix.....) (.....................tail...................)
// [all regions fully live] [left-most regions are fuller that right-most]
//

// Compute how much live data is there. This would approximate the size of dense prefix
// we target to create.
size_t total_live = 0;
for (size_t idx = 0; idx < n_regions; idx++) {
ShenandoahHeapRegion *r = heap->get_region(idx);
if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
total_live += r->get_live_data_words();
}
}

// Estimate the size for the dense prefix. Note that we specifically count only the
// "full" regions, so there would be some non-full regions in the slice tail.
size_t live_per_worker = total_live / n_workers;
size_t prefix_regions_per_worker = live_per_worker / ShenandoahHeapRegion::region_size_words();
size_t prefix_regions_total = prefix_regions_per_worker * n_workers;
prefix_regions_total = MIN2(prefix_regions_total, n_regions);
assert(prefix_regions_total <= n_regions, "Sanity");

// There might be non-candidate regions in the prefix. To compute where the tail actually
// ends up being, we need to account those as well.
size_t prefix_end = prefix_regions_total;
for (size_t idx = 0; idx < prefix_regions_total; idx++) {
ShenandoahHeapRegion *r = heap->get_region(idx);
if (!ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
prefix_end++;
}
}
prefix_end = MIN2(prefix_end, n_regions);
assert(prefix_end <= n_regions, "Sanity");

// Distribute prefix regions per worker: each thread definitely gets its own same-sized
// subset of dense prefix.
size_t prefix_idx = 0;

size_t* live = NEW_C_HEAP_ARRAY(size_t, n_workers, mtGC);

for (size_t wid = 0; wid < n_workers; wid++) {
ShenandoahHeapRegionSet* slice = worker_slices[wid];

live[wid] = 0;
size_t regs = 0;

// Add all prefix regions for this worker
while (prefix_idx < prefix_end && regs < prefix_regions_per_worker) {
ShenandoahHeapRegion *r = heap->get_region(prefix_idx);
if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
slice->add_region(r);
live[wid] += r->get_live_data_words();
regs++;
}
prefix_idx++;
}
}

// Distribute the tail among workers in round-robin fashion.
size_t wid = n_workers - 1;

for (size_t tail_idx = prefix_end; tail_idx < n_regions; tail_idx++) {
ShenandoahHeapRegion *r = heap->get_region(tail_idx);
if (ShenandoahPrepareForCompactionTask::is_candidate_region(r)) {
assert(wid < n_workers, "Sanity");

size_t live_region = r->get_live_data_words();

// Select next worker that still needs live data.
size_t old_wid = wid;
do {
wid++;
if (wid == n_workers) wid = 0;
} while (live[wid] + live_region >= live_per_worker && old_wid != wid);

if (old_wid == wid) {
// Circled back to the same worker? This means liveness data was
// miscalculated. Bump the live_per_worker limit so that
// everyone gets a piece of the leftover work.
live_per_worker += ShenandoahHeapRegion::region_size_words();
}

worker_slices[wid]->add_region(r);
live[wid] += live_region;
}
}

FREE_C_HEAP_ARRAY(size_t, live);

#ifdef ASSERT
ResourceBitMap map(n_regions);
for (size_t wid = 0; wid < n_workers; wid++) {
ShenandoahHeapRegionSetIterator it(worker_slices[wid]);
ShenandoahHeapRegion* r = it.next();
while (r != NULL) {
size_t num = r->region_number();
assert(ShenandoahPrepareForCompactionTask::is_candidate_region(r), "Sanity: " SIZE_FORMAT, num);
assert(!map.at(num), "No region distributed twice: " SIZE_FORMAT, num);
map.at_put(num, true);
r = it.next();
}
}

for (size_t rid = 0; rid < n_regions; rid++) {
bool is_candidate = ShenandoahPrepareForCompactionTask::is_candidate_region(heap->get_region(rid));
bool is_distributed = map.at(rid);
assert(is_distributed || !is_candidate, "All candidates are distributed: " SIZE_FORMAT, rid);
}
#endif
}

void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
@@ -533,8 +664,11 @@ void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegi
// Compute the new addresses for regular objects
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
heap->workers()->run_task(&prepare_task);

distribute_slices(worker_slices);

ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices);
heap->workers()->run_task(&task);
}

// Compute the new addresses for humongous objects
@@ -71,6 +71,7 @@ class ShenandoahMarkCompact : public CHeapObj<mtGC> {
void phase3_update_references();
void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices);

void distribute_slices(ShenandoahHeapRegionSet** worker_slices);
void calculate_target_humongous_objects();
void compact_humongous_objects();
};

0 comments on commit b678332

Please sign in to comment.