Skip to content
Permalink
Browse files
8258252: Move PtrQueue enqueue to PtrQueueSet subclasses
Reviewed-by: tschatzl, rkennke
  • Loading branch information
Kim Barrett committed Dec 16, 2020
1 parent 17ace83 commit cdb53422e8f57aeee82eb535b1dc77b62c9b9783
@@ -65,7 +65,8 @@ G1BarrierSet::G1BarrierSet(G1CardTable* card_table) :
void G1BarrierSet::enqueue(oop pre_val) {
// Nulls should have been already filtered.
assert(oopDesc::is_oop(pre_val, true), "Error");
G1ThreadLocalData::satb_mark_queue(Thread::current()).enqueue(pre_val);
SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
G1BarrierSet::satb_mark_queue_set().enqueue(queue, pre_val);
}

template <class T> void
@@ -99,7 +100,8 @@ void G1BarrierSet::write_ref_field_post_slow(volatile CardValue* byte) {
if (*byte != G1CardTable::dirty_card_val()) {
*byte = G1CardTable::dirty_card_val();
Thread* thr = Thread::current();
G1ThreadLocalData::dirty_card_queue(thr).enqueue(byte);
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thr);
G1BarrierSet::dirty_card_queue_set().enqueue(queue, byte);
}
}

@@ -116,13 +118,14 @@ void G1BarrierSet::invalidate(MemRegion mr) {
OrderAccess::storeload();
// Enqueue if necessary.
Thread* thr = Thread::current();
G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thr);
for (; byte <= last_byte; byte++) {
CardValue bv = *byte;
if ((bv != G1CardTable::g1_young_card_val()) &&
(bv != G1CardTable::dirty_card_val())) {
*byte = G1CardTable::dirty_card_val();
queue.enqueue(byte);
qset.enqueue(queue, byte);
}
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -52,10 +52,13 @@ JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_pre_entry(oopDesc* orig, Jav
}
assert(oopDesc::is_oop(orig, true /* ignore mark word */), "Error");
// store the original value that was in the field reference
G1ThreadLocalData::satb_mark_queue(thread).enqueue(orig);
SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(thread);
G1BarrierSet::satb_mark_queue_set().enqueue(queue, orig);
JRT_END

// G1 post write barrier slowpath
JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_post_entry(void* card_addr, JavaThread* thread))
G1ThreadLocalData::dirty_card_queue(thread).enqueue(card_addr);
JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_post_entry(volatile G1CardTable::CardValue* card_addr,
JavaThread* thread))
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thread);
G1BarrierSet::dirty_card_queue_set().enqueue(queue, card_addr);
JRT_END
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,8 @@
#ifndef SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP
#define SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP

#include "memory/allocation.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "memory/allStatic.hpp"
#include "oops/oopsHierarchy.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
@@ -35,14 +36,16 @@ class JavaThread;

class G1BarrierSetRuntime: public AllStatic {
public:
using CardValue = G1CardTable::CardValue;

// Arraycopy stub generator
static void write_ref_array_pre_oop_entry(oop* dst, size_t length);
static void write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length);
static void write_ref_array_post_entry(HeapWord* dst, size_t length);

// C2 slow-path runtime calls.
static void write_ref_field_pre_entry(oopDesc* orig, JavaThread *thread);
static void write_ref_field_post_entry(void* card_addr, JavaThread* thread);
static void write_ref_field_post_entry(volatile CardValue* card_addr, JavaThread* thread);
};

#endif // SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP
@@ -23,6 +23,7 @@
*/

#include "precompiled.hpp"
#include "gc/g1/g1BarrierSet.inline.hpp"
#include "gc/g1/g1BufferNodeList.hpp"
#include "gc/g1/g1CardTableEntryClosure.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
@@ -71,14 +72,6 @@ void G1DirtyCardQueue::on_thread_detach() {
dirty_card_qset()->record_detached_refinement_stats(_refinement_stats);
}

void G1DirtyCardQueue::handle_completed_buffer() {
assert(!is_empty(), "precondition");
_refinement_stats->inc_dirtied_cards(size());
BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
allocate_buffer();
dirty_card_qset()->handle_completed_buffer(node, _refinement_stats);
}

// Assumed to be zero by concurrent threads.
static uint par_ids_start() { return 0; }

@@ -106,8 +99,28 @@ uint G1DirtyCardQueueSet::num_par_ids() {
return (uint)os::initial_active_processor_count();
}

void G1DirtyCardQueueSet::enqueue(G1DirtyCardQueue& queue,
volatile CardValue* card_ptr) {
CardValue* value = const_cast<CardValue*>(card_ptr);
if (!try_enqueue(queue, value)) {
handle_zero_index(queue);
retry_enqueue(queue, value);
}
}

void G1DirtyCardQueueSet::handle_zero_index(G1DirtyCardQueue& queue) {
assert(queue.index() == 0, "precondition");
BufferNode* old_node = exchange_buffer_with_new(queue);
if (old_node != nullptr) {
G1ConcurrentRefineStats* stats = queue.refinement_stats();
stats->inc_dirtied_cards(buffer_size());
handle_completed_buffer(old_node, stats);
}
}

void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) {
G1ThreadLocalData::dirty_card_queue(t).handle_zero_index();
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(t);
G1BarrierSet::dirty_card_queue_set().handle_zero_index(queue);
}

#ifdef ASSERT
@@ -27,6 +27,7 @@

#include "gc/g1/g1BufferNodeList.hpp"
#include "gc/g1/g1FreeIdSet.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1ConcurrentRefineStats.hpp"
#include "gc/shared/ptrQueue.hpp"
#include "memory/allocation.hpp"
@@ -41,9 +42,6 @@ class Thread;
class G1DirtyCardQueue: public PtrQueue {
G1ConcurrentRefineStats* _refinement_stats;

protected:
virtual void handle_completed_buffer();

public:
G1DirtyCardQueue(G1DirtyCardQueueSet* qset);

@@ -263,6 +261,19 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
// if none available.
BufferNode* get_completed_buffer();

// Called when queue is full or has no buffer.
void handle_zero_index(G1DirtyCardQueue& queue);

// Enqueue the buffer, and optionally perform refinement by the mutator.
// Mutator refinement is only done by Java threads, and only if there
// are more than max_cards (possibly padded) cards in the completed
// buffers. Updates stats.
//
// Mutator refinement, if performed, stops processing a buffer if
// SuspendibleThreadSet::should_yield(), recording the incompletely
// processed buffer for later processing of the remainder.
void handle_completed_buffer(BufferNode* node, G1ConcurrentRefineStats* stats);

public:
G1DirtyCardQueueSet(BufferNode::Allocator* allocator);
~G1DirtyCardQueueSet();
@@ -302,16 +313,8 @@ class G1DirtyCardQueueSet: public PtrQueueSet {

G1BufferNodeList take_all_completed_buffers();

// Helper for G1DirtyCardQueue::handle_completed_buffer().
// Enqueue the buffer, and optionally perform refinement by the mutator.
// Mutator refinement is only done by Java threads, and only if there
// are more than max_cards (possibly padded) cards in the completed
// buffers. Updates stats.
//
// Mutator refinement, if performed, stops processing a buffer if
// SuspendibleThreadSet::should_yield(), recording the incompletely
// processed buffer for later processing of the remainder.
void handle_completed_buffer(BufferNode* node, G1ConcurrentRefineStats* stats);
using CardValue = G1CardTable::CardValue;
void enqueue(G1DirtyCardQueue& queue, volatile CardValue* card_ptr);

// If there are more than stop_at cards in the completed buffers, pop
// a buffer, refine its contents, and return true. Otherwise return
@@ -40,6 +40,7 @@
class UpdateLogBuffersDeferred : public BasicOopIterateClosure {
private:
G1CollectedHeap* _g1h;
G1RedirtyCardsLocalQueueSet* _rdc_local_qset;
G1RedirtyCardsQueue* _rdcq;
G1CardTable* _ct;

@@ -48,8 +49,13 @@ class UpdateLogBuffersDeferred : public BasicOopIterateClosure {
size_t _last_enqueued_card;

public:
UpdateLogBuffersDeferred(G1RedirtyCardsQueue* rdcq) :
_g1h(G1CollectedHeap::heap()), _rdcq(rdcq), _ct(_g1h->card_table()), _last_enqueued_card(SIZE_MAX) {}
UpdateLogBuffersDeferred(G1RedirtyCardsLocalQueueSet* rdc_local_qset,
G1RedirtyCardsQueue* rdcq) :
_g1h(G1CollectedHeap::heap()),
_rdc_local_qset(rdc_local_qset),
_rdcq(rdcq),
_ct(_g1h->card_table()),
_last_enqueued_card(SIZE_MAX) {}

virtual void do_oop(narrowOop* p) { do_oop_work(p); }
virtual void do_oop( oop* p) { do_oop_work(p); }
@@ -67,7 +73,7 @@ class UpdateLogBuffersDeferred : public BasicOopIterateClosure {
}
size_t card_index = _ct->index_for(p);
if (card_index != _last_enqueued_card) {
_rdcq->enqueue(_ct->byte_for_index(card_index));
_rdc_local_qset->enqueue(*_rdcq, _ct->byte_for_index(card_index));
_last_enqueued_card = card_index;
}
}
@@ -209,7 +215,7 @@ class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
_worker_id(worker_id),
_rdc_local_qset(rdcqs),
_rdcq(&_rdc_local_qset),
_log_buffer_cl(&_rdcq) {
_log_buffer_cl(&_rdc_local_qset, &_rdcq) {
}

~RemoveSelfForwardPtrHRClosure() {
@@ -88,8 +88,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
// Used to check whether string dedup should be applied to an object.
Klass* _string_klass_or_null;

G1RedirtyCardsQueue& redirty_cards_queue() { return _rdcq; }
G1CardTable* ct() { return _ct; }
G1CardTable* ct() { return _ct; }

G1HeapRegionAttr dest(G1HeapRegionAttr original) const {
assert(original.is_valid(),
@@ -149,7 +148,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
size_t card_index = ct()->index_for(p);
// If the card hasn't been added to the buffer, do it.
if (_last_enqueued_card != card_index) {
redirty_cards_queue().enqueue(ct()->byte_for_index(card_index));
_rdc_local_qset.enqueue(_rdcq, ct()->byte_for_index(card_index));
_last_enqueued_card = card_index;
}
}
@@ -28,7 +28,7 @@
#include "utilities/debug.hpp"
#include "utilities/macros.hpp"

// G1RedirtyCardsQueueBase::LocalQSet
// G1RedirtyCardsLocalQueueSet

G1RedirtyCardsLocalQueueSet::G1RedirtyCardsLocalQueueSet(G1RedirtyCardsQueueSet* shared_qset) :
PtrQueueSet(shared_qset->allocator()),
@@ -53,6 +53,16 @@ void G1RedirtyCardsLocalQueueSet::enqueue_completed_buffer(BufferNode* node) {
}
}

void G1RedirtyCardsLocalQueueSet::enqueue(G1RedirtyCardsQueue& queue, void* value) {
if (!try_enqueue(queue, value)) {
BufferNode* old_node = exchange_buffer_with_new(queue);
if (old_node != nullptr) {
enqueue_completed_buffer(old_node);
}
retry_enqueue(queue, value);
}
}

void G1RedirtyCardsLocalQueueSet::flush() {
_shared_qset->add_bufferlist(_buffers);
_buffers = G1BufferNodeList();
@@ -70,10 +80,6 @@ G1RedirtyCardsQueue::~G1RedirtyCardsQueue() {
}
#endif // ASSERT

void G1RedirtyCardsQueue::handle_completed_buffer() {
enqueue_completed_buffer();
}

void G1RedirtyCardsQueue::flush() {
flush_impl();
}
@@ -29,6 +29,7 @@
#include "gc/shared/ptrQueue.hpp"
#include "memory/padded.hpp"

class G1RedirtyCardsQueue;
class G1RedirtyCardsQueueSet;

// Provide G1RedirtyCardsQueue with a thread-local qset. It provides an
@@ -42,6 +43,8 @@ class G1RedirtyCardsLocalQueueSet : public PtrQueueSet {
G1RedirtyCardsLocalQueueSet(G1RedirtyCardsQueueSet* shared_qset);
~G1RedirtyCardsLocalQueueSet() NOT_DEBUG(= default);

void enqueue(G1RedirtyCardsQueue& queue, void* value);

// Add the buffer to the local list.
virtual void enqueue_completed_buffer(BufferNode* node);

@@ -51,9 +54,6 @@ class G1RedirtyCardsLocalQueueSet : public PtrQueueSet {

// Worker-local queues of card table entries.
class G1RedirtyCardsQueue : public PtrQueue {
protected:
virtual void handle_completed_buffer();

public:
G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset);
~G1RedirtyCardsQueue() NOT_DEBUG(= default);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
*/

#include "precompiled.hpp"
#include "gc/g1/g1BarrierSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
@@ -37,10 +38,11 @@ G1SATBMarkQueueSet::G1SATBMarkQueueSet(BufferNode::Allocator* allocator) :
{}

void G1SATBMarkQueueSet::handle_zero_index_for_thread(Thread* t) {
G1ThreadLocalData::satb_mark_queue(t).handle_zero_index();
G1SATBMarkQueueSet& qset = G1BarrierSet::satb_mark_queue_set();
qset.handle_zero_index(qset.satb_queue_for_thread(t));
}

SATBMarkQueue& G1SATBMarkQueueSet::satb_queue_for_thread(Thread* const t) const{
SATBMarkQueue& G1SATBMarkQueueSet::satb_queue_for_thread(Thread* const t) const {
return G1ThreadLocalData::satb_mark_queue(t);
}

@@ -113,6 +115,6 @@ class G1SATBMarkQueueFilterFn {
}
};

void G1SATBMarkQueueSet::filter(SATBMarkQueue* queue) {
void G1SATBMarkQueueSet::filter(SATBMarkQueue& queue) {
apply_filter(G1SATBMarkQueueFilterFn(), queue);
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -39,7 +39,7 @@ class G1SATBMarkQueueSet : public SATBMarkQueueSet {

static void handle_zero_index_for_thread(Thread* t);
virtual SATBMarkQueue& satb_queue_for_thread(Thread* const t) const;
virtual void filter(SATBMarkQueue* queue);
virtual void filter(SATBMarkQueue& queue);
};

#endif // SHARE_GC_G1_G1SATBMARKQUEUESET_HPP
Loading

1 comment on commit cdb5342

@openjdk-notifier

This comment has been minimized.

Copy link

@openjdk-notifier openjdk-notifier bot commented on cdb5342 Dec 16, 2020

Please sign in to comment.