Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8254739: G1: Optimize evacuation failure for regions with few failed objects #5181

Closed
Changes from 11 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
954b5ee
speed up iterate evac failure objs in one region.
Hamlin-Li Aug 18, 2021
4d88f42
fix: reset state after iteration
Hamlin-Li Aug 18, 2021
74bc4d9
Use a segmented array rather than a linked list to record evacuation …
Hamlin-Li Aug 25, 2021
05f026a
Fix compilation issues on some platform.
Hamlin-Li Aug 25, 2021
ded8275
Fix test failures; Fix compilation failures on some platforms
Hamlin-Li Aug 26, 2021
51d19eb
Fix test failures; Fix compilation failures on some platforms
Hamlin-Li Aug 26, 2021
f454925
Fix compilation error on windows
Hamlin-Li Aug 26, 2021
34aed3f
Fix compilation error on windows
Hamlin-Li Aug 26, 2021
43a8b59
Merge branch 'master' into speedup-iterate-evac-failure-objs-in-one-r…
Hamlin-Li Oct 25, 2021
6132372
Fix wrong merge
Hamlin-Li Oct 25, 2021
44e3562
Use refactored G1SegmentedArray rather than home-made Array+Node
Hamlin-Li Oct 25, 2021
e779c3a
Add asserts, comments
Hamlin-Li Oct 26, 2021
924fec5
Rename from g1EvacuationFailureObjsInHR to g1EvacFailureObjsInHR
Hamlin-Li Oct 26, 2021
0070826
Refactor as Thomas suggested
Hamlin-Li Oct 29, 2021
c4ca77c
Fix compilation error
Hamlin-Li Oct 29, 2021
ab04f1c
Fix compilation error
Hamlin-Li Oct 29, 2021
3712037
Merge branch 'openjdk:master' into speedup-iterate-evac-failure-objs-…
Hamlin-Li Oct 29, 2021
82c172a
Refine code based on Thomas' suggestion
Hamlin-Li Oct 29, 2021
d33f87b
Merge branch 'openjdk:master' into speedup-iterate-evac-failure-objs-…
Hamlin-Li Nov 3, 2021
e588cad
Move allocation/deallocation in one place
Hamlin-Li Nov 3, 2021
3efa90b
Fix typo
Hamlin-Li Nov 5, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -70,38 +70,37 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
assert(_hr->is_in(obj_addr), "sanity");

if (obj->is_forwarded() && obj->forwardee() == obj) {
// The object failed to move.

zap_dead_objects(_last_forwarded_object_end, obj_addr);
// We consider all objects that we find self-forwarded to be
// live. What we'll do is that we'll update the prev marking
// info so that they are all under PTAMS and explicitly marked.
if (!_cm->is_marked_in_prev_bitmap(obj)) {
_cm->mark_in_prev_bitmap(obj);
}
if (_during_concurrent_start) {
// For the next marking info we'll only mark the
// self-forwarded objects explicitly if we are during
// concurrent start (since, normally, we only mark objects pointed
// to by roots if we succeed in copying them). By marking all
// self-forwarded objects we ensure that we mark any that are
// still pointed to be roots. During concurrent marking, and
// after concurrent start, we don't need to mark any objects
// explicitly and all objects in the CSet are considered
// (implicitly) live. So, we won't mark them explicitly and
// we'll leave them over NTAMS.
_cm->mark_in_next_bitmap(_worker_id, _hr, obj);
}
size_t obj_size = obj->size();
// The object failed to move.
assert(obj->is_forwarded() && obj->forwardee() == obj, "sanity");

zap_dead_objects(_last_forwarded_object_end, obj_addr);
// We consider all objects that we find self-forwarded to be
// live. What we'll do is that we'll update the prev marking
// info so that they are all under PTAMS and explicitly marked.
if (!_cm->is_marked_in_prev_bitmap(obj)) {
_cm->mark_in_prev_bitmap(obj);
}
if (_during_concurrent_start) {
// For the next marking info we'll only mark the
// self-forwarded objects explicitly if we are during
// concurrent start (since, normally, we only mark objects pointed
// to by roots if we succeed in copying them). By marking all
// self-forwarded objects we ensure that we mark any that are
// still pointed to be roots. During concurrent marking, and
// after concurrent start, we don't need to mark any objects
// explicitly and all objects in the CSet are considered
// (implicitly) live. So, we won't mark them explicitly and
// we'll leave them over NTAMS.
_cm->mark_in_next_bitmap(_worker_id, _hr, obj);
}
size_t obj_size = obj->size();

_marked_bytes += (obj_size * HeapWordSize);
PreservedMarks::init_forwarded_mark(obj);
_marked_bytes += (obj_size * HeapWordSize);
PreservedMarks::init_forwarded_mark(obj);

HeapWord* obj_end = obj_addr + obj_size;
_last_forwarded_object_end = obj_end;
_hr->alloc_block_in_bot(obj_addr, obj_end);
}
HeapWord* obj_end = obj_addr + obj_size;
_last_forwarded_object_end = obj_end;
_hr->alloc_block_in_bot(obj_addr, obj_end);
}

// Fill the memory area from start to end with filler objects, and update the BOT
@@ -164,7 +163,8 @@ class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
RemoveSelfForwardPtrObjClosure rspc(hr,
during_concurrent_start,
_worker_id);
hr->object_iterate(&rspc);
// Iterates evac failure objs which are recorded during evacuation.
hr->iterate_evac_failure_objs(&rspc);
// Need to zap the remainder area of the processed region.
rspc.zap_remainder();

@@ -0,0 +1,117 @@
/*
* Copyright (c) 2021, Huawei and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

#include "precompiled.hpp"
#include "gc/g1/g1EvacuationFailureObjsInHR.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
#include "gc/g1/g1SegmentedArray.inline.hpp"
#include "gc/g1/heapRegion.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "utilities/quickSort.hpp"


const uint G1EvacuationFailureObjsInHR::MaxBufferLength =
static_cast<uint>(1u << (HeapRegion::LogOfHRGrainBytes-LogHeapWordSize));

const G1SegmentedArrayAllocOptions G1EvacuationFailureObjsInHR::_alloc_options =
G1SegmentedArrayAllocOptions(uint(sizeof (Elem)), BufferLength, UINT_MAX, Alignment);
G1SegmentedArrayBufferList<mtGC> G1EvacuationFailureObjsInHR::_free_buffer_list;

void G1EvacuationFailureObjsInHR::visit_buffer(G1SegmentedArrayBuffer<mtGC>* node, uint limit) {
node->copy_to(&_offset_array[_objs_num]);
_objs_num += limit;
// Verify elements in the buffer
DEBUG_ONLY(node->iterate_elems(*this));
}

void G1EvacuationFailureObjsInHR::visit_elem(void* elem) {
uint* ptr = (uint*)elem;
assert(*ptr < _max_offset, "must be, %u", *ptr);
}

void G1EvacuationFailureObjsInHR::compact() {
assert(_offset_array == NULL, "must be");
uint num = _nodes_array.num_allocated_nodes();
_offset_array = NEW_C_HEAP_ARRAY(Elem, num, mtGC);
// Copy buffers' data to local array
_nodes_array.iterate_nodes(*this);
assert(_objs_num == num, "must be %u, %u", _objs_num, num);
_nodes_array.drop_all();
}

static int order_oop(G1EvacuationFailureObjsInHR::Elem a,
G1EvacuationFailureObjsInHR::Elem b) {
return static_cast<int>(a-b);
}

void G1EvacuationFailureObjsInHR::sort() {
QuickSort::sort(_offset_array, _objs_num, order_oop, true);
}

void G1EvacuationFailureObjsInHR::clear_array() {
FREE_C_HEAP_ARRAY(Elem, _offset_array);
_offset_array = NULL;
_objs_num = 0;
}

void G1EvacuationFailureObjsInHR::iterate_internal(ObjectClosure* closure) {
Elem prev = 0;
for (uint i = 0; i < _objs_num; i++) {
assert(i == 0 ? (prev <= _offset_array[i]) : (prev < _offset_array[i]),
"must be, %u, %u, %u", i, prev, _offset_array[i]);
assert(prev < _max_offset, "must be, %u", prev);
closure->do_object(cast_from_offset(prev = _offset_array[i]));
}
clear_array();
}

G1EvacuationFailureObjsInHR::G1EvacuationFailureObjsInHR(uint region_idx, HeapWord* bottom) :
_max_offset(static_cast<Elem>(1u << (HeapRegion::LogOfHRGrainBytes-LogHeapWordSize))),
_region_idx(region_idx),
_bottom(bottom),
_nodes_array("", &_alloc_options, &_free_buffer_list),
_offset_array(NULL),
_objs_num(0) {
assert(HeapRegion::LogOfHRGrainBytes < 32, "must be");
}

G1EvacuationFailureObjsInHR::~G1EvacuationFailureObjsInHR() {
assert(_offset_array == NULL, "must be");
}

void G1EvacuationFailureObjsInHR::record(oop obj) {
assert(obj != NULL, "must be");
assert(_region_idx == G1CollectedHeap::heap()->heap_region_containing(obj)->hrm_index(), "must be");
Elem offset = cast_from_oop_addr(obj);
assert(obj == cast_from_offset(offset), "must be");
assert(offset < _max_offset, "must be, %u", offset);
Elem* e = _nodes_array.allocate();
*e = offset;
}

void G1EvacuationFailureObjsInHR::iterate(ObjectClosure* closure) {
compact();
sort();
iterate_internal(closure);
}
@@ -0,0 +1,95 @@
/*
* Copyright (c) 2021, Huawei and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

#ifndef SHARE_GC_G1_G1EVACUATIONFAILUREOBJSINHR_HPP
#define SHARE_GC_G1_G1EVACUATIONFAILUREOBJSINHR_HPP

#include "gc/g1/g1SegmentedArray.hpp"
#include "memory/iterator.hpp"
#include "oops/oop.hpp"

// This class
// 1. records the objects per region which have failed to evacuate.
// 2. speeds up removing self forwarded ptrs in post evacuation phase.
//
class G1EvacuationFailureObjsInHR {

public:
typedef uint Elem;

private:
static const uint BufferLength = 256;
static const uint MaxBufferLength;
static const uint Alignment = 4;

static const G1SegmentedArrayAllocOptions _alloc_options;
// This free list is shared among evacuation failure process in all regions.
static G1SegmentedArrayBufferList<mtGC> _free_buffer_list;

const Elem _max_offset;
const uint _region_idx;
const HeapWord* _bottom;

// To improve space efficiency, elements are offset rather than raw addr
G1SegmentedArray<Elem, mtGC> _nodes_array;
// Local array contains the _nodes_array data in flat layout
Elem* _offset_array;
uint _objs_num;

private:
oop cast_from_offset(Elem offset) {
return cast_to_oop(_bottom + offset);
}
Elem cast_from_oop_addr(oop obj) {
const HeapWord* o = cast_from_oop<const HeapWord*>(obj);
size_t offset = pointer_delta(o, _bottom);
return static_cast<Elem>(offset);
}

// Copy buffers' data to local array
void compact();
void sort();
void clear_array();
// Iterate through evac failure objects in local array
void iterate_internal(ObjectClosure* closure);

public:
G1EvacuationFailureObjsInHR(uint region_idx, HeapWord* bottom);
~G1EvacuationFailureObjsInHR();

// Record an evac failure object
void record(oop obj);
// Iterate through evac failure objects
void iterate(ObjectClosure* closure);

//
// Copy a buffer data to local array
void visit_buffer(G1SegmentedArrayBuffer<mtGC>* node, uint limit);

// Verify elements in the buffer
void visit_elem(void* elem);
};


#endif //SHARE_GC_G1_G1EVACUATIONFAILUREOBJSINHR_HPP
@@ -607,6 +607,9 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz
if (forward_ptr == NULL) {
// Forward-to-self succeeded. We are the "owner" of the object.
HeapRegion* r = _g1h->heap_region_containing(old);
// Records evac failure objs, this will help speed up iteration
// of these objs later in *remove self forward* phase of post evacuation.
r->record_evac_failure_obj(old);

if (_evac_failure_regions->record(r->hrm_index())) {
_g1h->hr_printer()->evac_failure(r);
@@ -73,6 +73,19 @@ class G1SegmentedArrayBuffer : public CHeapObj<flag> {

size_t mem_size() const { return sizeof(*this) + (size_t)_num_elems * _elem_size; }

uint length() const {
// _next_allocate might grow greater than _num_elems in multi-thread env,
// so, here we need to return the adjusted real length value.
return _next_allocate > _num_elems ? _num_elems : _next_allocate;
}

void copy_to(void* dest) const {
::memcpy(dest, _buffer, length() * _elem_size);
}

template<typename Visitor>
void iterate_elems(Visitor& v) const;

bool is_full() const { return _next_allocate >= _num_elems; }
};

@@ -190,11 +203,17 @@ class G1SegmentedArray {
private:
inline G1SegmentedArrayBuffer<flag>* create_new_buffer(G1SegmentedArrayBuffer<flag>* const prev);

DEBUG_ONLY(uint length() const;)

public:
const G1SegmentedArrayBuffer<flag>* first_array_buffer() const { return Atomic::load(&_first); }

uint num_available_nodes() const { return Atomic::load(&_num_available_nodes); }
uint num_allocated_nodes() const { return Atomic::load(&_num_allocated_nodes); }
uint num_allocated_nodes() const {
uint allocated = Atomic::load(&_num_allocated_nodes);
assert(length() == allocated, "Must be");
return allocated;
}

inline uint elem_size() const;

@@ -212,6 +231,9 @@ class G1SegmentedArray {
inline Elem* allocate();

inline uint num_buffers() const;

template<typename Visitor>
void iterate_nodes(Visitor& v) const;
};

#endif //SHARE_GC_G1_G1SEGMENTEDARRAY_HPP