Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
8258254: Move PtrQueue flush to PtrQueueSet subclasses
Reviewed-by: tschatzl, shade
  • Loading branch information
Kim Barrett committed Jan 12, 2021
1 parent 2255ab7 commit 77f6290995dc681c55301fd032f642057523094a
Show file tree
Hide file tree
Showing 12 changed files with 78 additions and 92 deletions.
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -158,6 +158,14 @@ void G1BarrierSet::on_thread_attach(Thread* thread) {
void G1BarrierSet::on_thread_detach(Thread* thread) {
// Flush any deferred card marks.
CardTableBarrierSet::on_thread_detach(thread);
G1ThreadLocalData::satb_mark_queue(thread).flush();
G1ThreadLocalData::dirty_card_queue(thread).on_thread_detach();
{
SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(thread);
G1BarrierSet::satb_mark_queue_set().flush_queue(queue);
}
{
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thread);
G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
qset.flush_queue(queue);
qset.record_detached_refinement_stats(queue.refinement_stats());
}
}
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -55,21 +55,10 @@ G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) :
{ }

G1DirtyCardQueue::~G1DirtyCardQueue() {
flush();
G1BarrierSet::dirty_card_queue_set().flush_queue(*this);
delete _refinement_stats;
}

void G1DirtyCardQueue::flush() {
_refinement_stats->inc_dirtied_cards(size());
flush_impl();
}

void G1DirtyCardQueue::on_thread_detach() {
assert(this == &G1ThreadLocalData::dirty_card_queue(Thread::current()), "precondition");
flush();
dirty_card_qset()->record_detached_refinement_stats(_refinement_stats);
}

// Assumed to be zero by concurrent threads.
static uint par_ids_start() { return 0; }

@@ -95,6 +84,14 @@ uint G1DirtyCardQueueSet::num_par_ids() {
return (uint)os::initial_active_processor_count();
}

void G1DirtyCardQueueSet::flush_queue(G1DirtyCardQueue& queue) {
if (queue.buffer() != nullptr) {
G1ConcurrentRefineStats* stats = queue.refinement_stats();
stats->inc_dirtied_cards(buffer_size() - queue.index());
}
PtrQueueSet::flush_queue(queue);
}

void G1DirtyCardQueueSet::enqueue(G1DirtyCardQueue& queue,
volatile CardValue* card_ptr) {
CardValue* value = const_cast<CardValue*>(card_ptr);
@@ -645,13 +642,16 @@ void G1DirtyCardQueueSet::concatenate_logs() {
set_max_cards(MaxCardsUnlimited);

struct ConcatenateThreadLogClosure : public ThreadClosure {
G1DirtyCardQueueSet& _qset;
ConcatenateThreadLogClosure(G1DirtyCardQueueSet& qset) : _qset(qset) {}
virtual void do_thread(Thread* t) {
G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t);
if (!dcq.is_empty()) {
dcq.flush();
G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(t);
if ((queue.buffer() != nullptr) &&
(queue.index() != _qset.buffer_size())) {
_qset.flush_queue(queue);
}
}
} closure;
} closure(*this);
Threads::threads_do(&closure);

G1BarrierSet::shared_dirty_card_queue().flush();
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -49,19 +49,12 @@ class G1DirtyCardQueue: public PtrQueue {
// doing something else, with auto-flush on completion.
~G1DirtyCardQueue();

// Process queue entries and release resources.
void flush();

inline G1DirtyCardQueueSet* dirty_card_qset() const;

G1ConcurrentRefineStats* refinement_stats() const {
return _refinement_stats;
}

// To be called by the barrier set's on_thread_detach, to notify this
// object of the corresponding state change of its owning thread.
void on_thread_detach();

// Compiler support.
static ByteSize byte_offset_of_index() {
return PtrQueue::byte_offset_of_index<G1DirtyCardQueue>();
@@ -313,6 +306,8 @@ class G1DirtyCardQueueSet: public PtrQueueSet {

G1BufferNodeList take_all_completed_buffers();

void flush_queue(G1DirtyCardQueue& queue);

using CardValue = G1CardTable::CardValue;
void enqueue(G1DirtyCardQueue& queue, volatile CardValue* card_ptr);

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -219,7 +219,7 @@ class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
}

~RemoveSelfForwardPtrHRClosure() {
_rdcq.flush();
_rdc_local_qset.flush_queue(_rdcq);
_rdc_local_qset.flush();
}

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -114,7 +114,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
}

size_t G1ParScanThreadState::flush(size_t* surviving_young_words) {
_rdcq.flush();
_rdc_local_qset.flush_queue(_rdcq);
_rdc_local_qset.flush();
flush_numa_stats();
// Update allocation statistics.
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -68,6 +68,10 @@ void G1RedirtyCardsLocalQueueSet::flush() {
_buffers = G1BufferNodeList();
}

void G1RedirtyCardsLocalQueueSet::flush_queue(G1RedirtyCardsQueue& queue) {
PtrQueueSet::flush_queue(queue);
}

// G1RedirtyCardsQueue

G1RedirtyCardsQueue::G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset) :
@@ -76,14 +80,10 @@ G1RedirtyCardsQueue::G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset) :

#ifdef ASSERT
G1RedirtyCardsQueue::~G1RedirtyCardsQueue() {
assert(is_empty(), "unflushed queue");
assert(buffer() == nullptr, "unflushed queue");
}
#endif // ASSERT

void G1RedirtyCardsQueue::flush() {
flush_impl();
}

// G1RedirtyCardsQueueSet

G1RedirtyCardsQueueSet::G1RedirtyCardsQueueSet(BufferNode::Allocator* allocator) :
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -50,16 +50,15 @@ class G1RedirtyCardsLocalQueueSet : public PtrQueueSet {

// Transfer all completed buffers to the shared qset.
void flush();

void flush_queue(G1RedirtyCardsQueue& queue);
};

// Worker-local queues of card table entries.
class G1RedirtyCardsQueue : public PtrQueue {
public:
G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset);
~G1RedirtyCardsQueue() NOT_DEBUG(= default);

// Flushes all enqueued cards to qset.
void flush();
};

// Card table entries to be redirtied and the cards reprocessed later.
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,32 +46,6 @@ PtrQueue::~PtrQueue() {
assert(_buf == NULL, "queue must be flushed before delete");
}

void PtrQueue::flush_impl() {
if (_buf != NULL) {
BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
if (is_empty()) {
// No work to do.
qset()->deallocate_buffer(node);
} else {
qset()->enqueue_completed_buffer(node);
}
_buf = NULL;
set_index(0);
}
}

void PtrQueue::allocate_buffer() {
_buf = qset()->allocate_buffer();
reset();
}

void PtrQueue::enqueue_completed_buffer() {
assert(_buf != NULL, "precondition");
BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
qset()->enqueue_completed_buffer(node);
allocate_buffer();
}

BufferNode* BufferNode::allocate(size_t size) {
size_t byte_size = size * sizeof(void*);
void* data = NEW_C_HEAP_ARRAY(char, buffer_offset() + byte_size, mtGC);
@@ -225,6 +199,21 @@ PtrQueueSet::PtrQueueSet(BufferNode::Allocator* allocator) :

PtrQueueSet::~PtrQueueSet() {}

void PtrQueueSet::flush_queue(PtrQueue& queue) {
void** buffer = queue.buffer();
if (buffer != nullptr) {
size_t index = queue.index();
queue.set_buffer(nullptr);
queue.set_index(0);
BufferNode* node = BufferNode::make_node_from_buffer(buffer, index);
if (index == buffer_size()) {
deallocate_buffer(node);
} else {
enqueue_completed_buffer(node);
}
}
}

bool PtrQueueSet::try_enqueue(PtrQueue& queue, void* value) {
size_t index = queue.index();
if (index == 0) return false;
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -79,14 +79,6 @@ class PtrQueue {

PtrQueueSet* qset() const { return _qset; }

// Process queue entries and release resources.
void flush_impl();

void allocate_buffer();

// Enqueue the current buffer in the qset and allocate a new buffer.
void enqueue_completed_buffer();

// Initialize this queue to contain a null buffer, and be part of the
// given PtrQueueSet.
PtrQueue(PtrQueueSet* qset);
@@ -268,6 +260,10 @@ class PtrQueueSet {
PtrQueueSet(BufferNode::Allocator* allocator);
~PtrQueueSet();

// If queue has any buffered enqueued data, transfer it to this qset.
// Otherwise, deallocate queue's buffer.
void flush_queue(PtrQueue& queue);

// Add value to queue's buffer, returning true. If buffer is full
// or if queue doesn't have a buffer, does nothing and returns false.
bool try_enqueue(PtrQueue& queue, void* value);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -47,13 +47,6 @@ SATBMarkQueue::SATBMarkQueue(SATBMarkQueueSet* qset) :
_active(false)
{ }

void SATBMarkQueue::flush() {
// Filter now to possibly save work later. If filtering empties the
// buffer then flush_impl can deallocate the buffer.
filter();
flush_impl();
}

void SATBMarkQueue::apply_closure_and_empty(SATBBufferClosure* cl) {
assert(SafepointSynchronize::is_at_safepoint(),
"SATB queues must only be processed at safepoints");
@@ -235,6 +228,13 @@ bool SATBMarkQueueSet::apply_closure_to_completed_buffer(SATBBufferClosure* cl)
}
}

void SATBMarkQueueSet::flush_queue(SATBMarkQueue& queue) {
// Filter now to possibly save work later. If filtering empties the
// buffer then flush_queue can deallocate the buffer.
filter(queue);
PtrQueueSet::flush_queue(queue);
}

void SATBMarkQueueSet::enqueue_known_active(SATBMarkQueue& queue, oop obj) {
assert(queue.is_active(), "precondition");
void* value = cast_from_oop<void*>(obj);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -63,9 +63,6 @@ class SATBMarkQueue: public PtrQueue {
bool is_active() const { return _active; }
void set_active(bool value) { _active = value; }

// Process queue entries and free resources.
void flush();

inline SATBMarkQueueSet* satb_qset() const;

// Apply cl to the active part of the buffer.
@@ -150,6 +147,8 @@ class SATBMarkQueueSet: public PtrQueueSet {
// buffer; the leading entries may be excluded due to filtering.
bool apply_closure_to_completed_buffer(SATBBufferClosure* cl);

void flush_queue(SATBMarkQueue& queue);

// When active, add obj to queue by calling enqueue_known_active.
void enqueue(SATBMarkQueue& queue, oop obj) {
if (queue.is_active()) enqueue_known_active(queue, obj);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
* Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -116,7 +116,7 @@ void ShenandoahBarrierSet::on_thread_attach(Thread *thread) {

void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
queue.flush();
_satb_mark_queue_set.flush_queue(queue);
if (thread->is_Java_thread()) {
PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
if (gclab != NULL) {

0 comments on commit 77f6290

Please sign in to comment.