Skip to content
Permalink
Browse files
8283566: G1: Improve G1BarrierSet::enqueue performance
Reviewed-by: tschatzl, ayang
  • Loading branch information
shipilev committed Mar 31, 2022
1 parent d276da5 commit 6ebf845ff7d758c99367488818427fcb7a5dbda9
Showing 4 changed files with 36 additions and 20 deletions.
@@ -61,13 +61,6 @@ G1BarrierSet::G1BarrierSet(G1CardTable* card_table) :
_dirty_card_queue_set(&_dirty_card_queue_buffer_allocator)
{}

void G1BarrierSet::enqueue(oop pre_val) {
// Nulls should have been already filtered.
assert(oopDesc::is_oop(pre_val, true), "Error");
SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
G1BarrierSet::satb_mark_queue_set().enqueue(queue, pre_val);
}

template <class T> void
G1BarrierSet::write_ref_array_pre_work(T* dst, size_t count) {
G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set();
@@ -56,10 +56,12 @@ class G1BarrierSet: public CardTableBarrierSet {
}

// Add "pre_val" to a set of objects that may have been disconnected from the
// pre-marking object graph.
static void enqueue(oop pre_val);
// pre-marking object graph. Prefer the version that takes location, as it
// can avoid touching the heap unnecessarily.
template <class T> static void enqueue(T* dst);
static void enqueue_preloaded(oop pre_val);

static void enqueue_if_weak(DecoratorSet decorators, oop value);
static void enqueue_preloaded_if_weak(DecoratorSet decorators, oop value);

template <class T> void write_ref_array_pre_work(T* dst, size_t count);
virtual void write_ref_array_pre(oop* dst, size_t count, bool dest_uninitialized);
@@ -28,22 +28,43 @@
#include "gc/g1/g1BarrierSet.hpp"

#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/shared/accessBarrierSupport.inline.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/oop.hpp"

inline void G1BarrierSet::enqueue_preloaded(oop pre_val) {
// Nulls should have been already filtered.
assert(oopDesc::is_oop(pre_val, true), "Error");

G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set();
if (!queue_set.is_active()) return;

SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
queue_set.enqueue_known_active(queue, pre_val);
}

template <class T>
inline void G1BarrierSet::enqueue(T* dst) {
G1SATBMarkQueueSet& queue_set = G1BarrierSet::satb_mark_queue_set();
if (!queue_set.is_active()) return;

T heap_oop = RawAccess<MO_RELAXED>::oop_load(dst);
if (!CompressedOops::is_null(heap_oop)) {
SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
queue_set.enqueue_known_active(queue, CompressedOops::decode_not_null(heap_oop));
}
}

template <DecoratorSet decorators, typename T>
inline void G1BarrierSet::write_ref_field_pre(T* field) {
if (HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value ||
HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
return;
}

T heap_oop = RawAccess<MO_RELAXED>::oop_load(field);
if (!CompressedOops::is_null(heap_oop)) {
enqueue(CompressedOops::decode_not_null(heap_oop));
}
enqueue(field);
}

template <DecoratorSet decorators, typename T>
@@ -55,7 +76,7 @@ inline void G1BarrierSet::write_ref_field_post(T* field, oop new_val) {
}
}

inline void G1BarrierSet::enqueue_if_weak(DecoratorSet decorators, oop value) {
inline void G1BarrierSet::enqueue_preloaded_if_weak(DecoratorSet decorators, oop value) {
assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
// Loading from a weak or phantom reference needs enqueueing, as
// the object may not have been reachable (part of the snapshot)
@@ -65,7 +86,7 @@ inline void G1BarrierSet::enqueue_if_weak(DecoratorSet decorators, oop value) {
const bool needs_enqueue = (!peek && !on_strong_oop_ref);

if (needs_enqueue && value != NULL) {
enqueue(value);
enqueue_preloaded(value);
}
}

@@ -74,7 +95,7 @@ template <typename T>
inline oop G1BarrierSet::AccessBarrier<decorators, BarrierSetT>::
oop_load_not_in_heap(T* addr) {
oop value = ModRef::oop_load_not_in_heap(addr);
enqueue_if_weak(decorators, value);
enqueue_preloaded_if_weak(decorators, value);
return value;
}

@@ -83,15 +104,15 @@ template <typename T>
inline oop G1BarrierSet::AccessBarrier<decorators, BarrierSetT>::
oop_load_in_heap(T* addr) {
oop value = ModRef::oop_load_in_heap(addr);
enqueue_if_weak(decorators, value);
enqueue_preloaded_if_weak(decorators, value);
return value;
}

template <DecoratorSet decorators, typename BarrierSetT>
inline oop G1BarrierSet::AccessBarrier<decorators, BarrierSetT>::
oop_load_in_heap_at(oop base, ptrdiff_t offset) {
oop value = ModRef::oop_load_in_heap_at(base, offset);
enqueue_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
enqueue_preloaded_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset), value);
return value;
}

@@ -2279,7 +2279,7 @@ void G1CollectedHeap::object_iterate_parallel(ObjectClosure* cl, uint worker_id,
}

void G1CollectedHeap::keep_alive(oop obj) {
G1BarrierSet::enqueue(obj);
G1BarrierSet::enqueue_preloaded(obj);
}

void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {

1 comment on commit 6ebf845

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on 6ebf845 Mar 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.