8258252: Move PtrQueue enqueue to PtrQueueSet subclasses

Reviewed-by: tschatzl, rkennke

Author: Kim Barrett

src/hotspot/share/gc/g1/g1BarrierSet.cpp

@@ -65,7 +65,8 @@ G1BarrierSet::G1BarrierSet(G1CardTable* card_table) :
 void G1BarrierSet::enqueue(oop pre_val) {
   // Nulls should have been already filtered.
   assert(oopDesc::is_oop(pre_val, true), "Error");
-  G1ThreadLocalData::satb_mark_queue(Thread::current()).enqueue(pre_val);
+  SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(Thread::current());
+  G1BarrierSet::satb_mark_queue_set().enqueue(queue, pre_val);
 }
 
 template <class T> void
@@ -99,7 +100,8 @@ void G1BarrierSet::write_ref_field_post_slow(volatile CardValue* byte) {
   if (*byte != G1CardTable::dirty_card_val()) {
     *byte = G1CardTable::dirty_card_val();
     Thread* thr = Thread::current();
-    G1ThreadLocalData::dirty_card_queue(thr).enqueue(byte);
+    G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thr);
+    G1BarrierSet::dirty_card_queue_set().enqueue(queue, byte);
   }
 }
 
@@ -116,13 +118,14 @@ void G1BarrierSet::invalidate(MemRegion mr) {
     OrderAccess::storeload();
     // Enqueue if necessary.
     Thread* thr = Thread::current();
+    G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
     G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thr);
     for (; byte <= last_byte; byte++) {
       CardValue bv = *byte;
       if ((bv != G1CardTable::g1_young_card_val()) &&
           (bv != G1CardTable::dirty_card_val())) {
         *byte = G1CardTable::dirty_card_val();
-        queue.enqueue(byte);
+        qset.enqueue(queue, byte);
       }
     }
   }

src/hotspot/share/gc/g1/g1BarrierSetRuntime.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,10 +52,13 @@ JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_pre_entry(oopDesc* orig, Jav
   }
   assert(oopDesc::is_oop(orig, true /* ignore mark word */), "Error");
   // store the original value that was in the field reference
-  G1ThreadLocalData::satb_mark_queue(thread).enqueue(orig);
+  SATBMarkQueue& queue = G1ThreadLocalData::satb_mark_queue(thread);
+  G1BarrierSet::satb_mark_queue_set().enqueue(queue, orig);
 JRT_END
 
 // G1 post write barrier slowpath
-JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_post_entry(void* card_addr, JavaThread* thread))
-  G1ThreadLocalData::dirty_card_queue(thread).enqueue(card_addr);
+JRT_LEAF(void, G1BarrierSetRuntime::write_ref_field_post_entry(volatile G1CardTable::CardValue* card_addr,
+                                                               JavaThread* thread))
+  G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(thread);
+  G1BarrierSet::dirty_card_queue_set().enqueue(queue, card_addr);
 JRT_END

src/hotspot/share/gc/g1/g1BarrierSetRuntime.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,8 @@
 #ifndef SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP
 #define SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP
 
-#include "memory/allocation.hpp"
+#include "gc/g1/g1CardTable.hpp"
+#include "memory/allStatic.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
@@ -35,14 +36,16 @@ class JavaThread;
 
 class G1BarrierSetRuntime: public AllStatic {
 public:
+  using CardValue = G1CardTable::CardValue;
+
   // Arraycopy stub generator
   static void write_ref_array_pre_oop_entry(oop* dst, size_t length);
   static void write_ref_array_pre_narrow_oop_entry(narrowOop* dst, size_t length);
   static void write_ref_array_post_entry(HeapWord* dst, size_t length);
 
   // C2 slow-path runtime calls.
   static void write_ref_field_pre_entry(oopDesc* orig, JavaThread *thread);
-  static void write_ref_field_post_entry(void* card_addr, JavaThread* thread);
+  static void write_ref_field_post_entry(volatile CardValue* card_addr, JavaThread* thread);
 };
 
 #endif // SHARE_GC_G1_G1BARRIERSETRUNTIME_HPP

src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1BarrierSet.inline.hpp"
 #include "gc/g1/g1BufferNodeList.hpp"
 #include "gc/g1/g1CardTableEntryClosure.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
@@ -71,14 +72,6 @@ void G1DirtyCardQueue::on_thread_detach() {
   dirty_card_qset()->record_detached_refinement_stats(_refinement_stats);
 }
 
-void G1DirtyCardQueue::handle_completed_buffer() {
-  assert(!is_empty(), "precondition");
-  _refinement_stats->inc_dirtied_cards(size());
-  BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
-  allocate_buffer();
-  dirty_card_qset()->handle_completed_buffer(node, _refinement_stats);
-}
-
 // Assumed to be zero by concurrent threads.
 static uint par_ids_start() { return 0; }
 
@@ -106,8 +99,28 @@ uint G1DirtyCardQueueSet::num_par_ids() {
   return (uint)os::initial_active_processor_count();
 }
 
+void G1DirtyCardQueueSet::enqueue(G1DirtyCardQueue& queue,
+                                  volatile CardValue* card_ptr) {
+  CardValue* value = const_cast<CardValue*>(card_ptr);
+  if (!try_enqueue(queue, value)) {
+    handle_zero_index(queue);
+    retry_enqueue(queue, value);
+  }
+}
+
+void G1DirtyCardQueueSet::handle_zero_index(G1DirtyCardQueue& queue) {
+  assert(queue.index() == 0, "precondition");
+  BufferNode* old_node = exchange_buffer_with_new(queue);
+  if (old_node != nullptr) {
+    G1ConcurrentRefineStats* stats = queue.refinement_stats();
+    stats->inc_dirtied_cards(buffer_size());
+    handle_completed_buffer(old_node, stats);
+  }
+}
+
 void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) {
-  G1ThreadLocalData::dirty_card_queue(t).handle_zero_index();
+  G1DirtyCardQueue& queue = G1ThreadLocalData::dirty_card_queue(t);
+  G1BarrierSet::dirty_card_queue_set().handle_zero_index(queue);
 }
 
 #ifdef ASSERT

src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp

@@ -27,6 +27,7 @@
 
 #include "gc/g1/g1BufferNodeList.hpp"
 #include "gc/g1/g1FreeIdSet.hpp"
+#include "gc/g1/g1CardTable.hpp"
 #include "gc/g1/g1ConcurrentRefineStats.hpp"
 #include "gc/shared/ptrQueue.hpp"
 #include "memory/allocation.hpp"
@@ -41,9 +42,6 @@ class Thread;
 class G1DirtyCardQueue: public PtrQueue {
   G1ConcurrentRefineStats* _refinement_stats;
 
-protected:
-  virtual void handle_completed_buffer();
-
 public:
   G1DirtyCardQueue(G1DirtyCardQueueSet* qset);
 
@@ -263,6 +261,19 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
   // if none available.
   BufferNode* get_completed_buffer();
 
+  // Called when queue is full or has no buffer.
+  void handle_zero_index(G1DirtyCardQueue& queue);
+
+  // Enqueue the buffer, and optionally perform refinement by the mutator.
+  // Mutator refinement is only done by Java threads, and only if there
+  // are more than max_cards (possibly padded) cards in the completed
+  // buffers.  Updates stats.
+  //
+  // Mutator refinement, if performed, stops processing a buffer if
+  // SuspendibleThreadSet::should_yield(), recording the incompletely
+  // processed buffer for later processing of the remainder.
+  void handle_completed_buffer(BufferNode* node, G1ConcurrentRefineStats* stats);
+
 public:
   G1DirtyCardQueueSet(BufferNode::Allocator* allocator);
   ~G1DirtyCardQueueSet();
@@ -302,16 +313,8 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
 
   G1BufferNodeList take_all_completed_buffers();
 
-  // Helper for G1DirtyCardQueue::handle_completed_buffer().
-  // Enqueue the buffer, and optionally perform refinement by the mutator.
-  // Mutator refinement is only done by Java threads, and only if there
-  // are more than max_cards (possibly padded) cards in the completed
-  // buffers.  Updates stats.
-  //
-  // Mutator refinement, if performed, stops processing a buffer if
-  // SuspendibleThreadSet::should_yield(), recording the incompletely
-  // processed buffer for later processing of the remainder.
-  void handle_completed_buffer(BufferNode* node, G1ConcurrentRefineStats* stats);
+  using CardValue = G1CardTable::CardValue;
+  void enqueue(G1DirtyCardQueue& queue, volatile CardValue* card_ptr);
 
   // If there are more than stop_at cards in the completed buffers, pop
   // a buffer, refine its contents, and return true.  Otherwise return

src/hotspot/share/gc/g1/g1EvacFailure.cpp

@@ -40,6 +40,7 @@
 class UpdateLogBuffersDeferred : public BasicOopIterateClosure {
 private:
   G1CollectedHeap* _g1h;
+  G1RedirtyCardsLocalQueueSet* _rdc_local_qset;
   G1RedirtyCardsQueue* _rdcq;
   G1CardTable*    _ct;
 
@@ -48,8 +49,13 @@ class UpdateLogBuffersDeferred : public BasicOopIterateClosure {
   size_t _last_enqueued_card;
 
 public:
-  UpdateLogBuffersDeferred(G1RedirtyCardsQueue* rdcq) :
-    _g1h(G1CollectedHeap::heap()), _rdcq(rdcq), _ct(_g1h->card_table()), _last_enqueued_card(SIZE_MAX) {}
+  UpdateLogBuffersDeferred(G1RedirtyCardsLocalQueueSet* rdc_local_qset,
+                           G1RedirtyCardsQueue* rdcq) :
+    _g1h(G1CollectedHeap::heap()),
+    _rdc_local_qset(rdc_local_qset),
+    _rdcq(rdcq),
+    _ct(_g1h->card_table()),
+    _last_enqueued_card(SIZE_MAX) {}
 
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   virtual void do_oop(      oop* p) { do_oop_work(p); }
@@ -67,7 +73,7 @@ class UpdateLogBuffersDeferred : public BasicOopIterateClosure {
     }
     size_t card_index = _ct->index_for(p);
     if (card_index != _last_enqueued_card) {
-      _rdcq->enqueue(_ct->byte_for_index(card_index));
+      _rdc_local_qset->enqueue(*_rdcq, _ct->byte_for_index(card_index));
       _last_enqueued_card = card_index;
     }
   }
@@ -209,7 +215,7 @@ class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
     _worker_id(worker_id),
     _rdc_local_qset(rdcqs),
     _rdcq(&_rdc_local_qset),
-    _log_buffer_cl(&_rdcq) {
+    _log_buffer_cl(&_rdc_local_qset, &_rdcq) {
   }
 
   ~RemoveSelfForwardPtrHRClosure() {

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -88,8 +88,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // Used to check whether string dedup should be applied to an object.
   Klass* _string_klass_or_null;
 
-  G1RedirtyCardsQueue& redirty_cards_queue()     { return _rdcq; }
-  G1CardTable* ct()                              { return _ct; }
+  G1CardTable* ct() { return _ct; }
 
   G1HeapRegionAttr dest(G1HeapRegionAttr original) const {
     assert(original.is_valid(),
@@ -149,7 +148,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
     size_t card_index = ct()->index_for(p);
     // If the card hasn't been added to the buffer, do it.
     if (_last_enqueued_card != card_index) {
-      redirty_cards_queue().enqueue(ct()->byte_for_index(card_index));
+      _rdc_local_qset.enqueue(_rdcq, ct()->byte_for_index(card_index));
       _last_enqueued_card = card_index;
     }
   }

src/hotspot/share/gc/g1/g1RedirtyCardsQueue.cpp

@@ -28,7 +28,7 @@
 #include "utilities/debug.hpp"
 #include "utilities/macros.hpp"
 
-// G1RedirtyCardsQueueBase::LocalQSet
+// G1RedirtyCardsLocalQueueSet
 
 G1RedirtyCardsLocalQueueSet::G1RedirtyCardsLocalQueueSet(G1RedirtyCardsQueueSet* shared_qset) :
   PtrQueueSet(shared_qset->allocator()),
@@ -53,6 +53,16 @@ void G1RedirtyCardsLocalQueueSet::enqueue_completed_buffer(BufferNode* node) {
   }
 }
 
+void G1RedirtyCardsLocalQueueSet::enqueue(G1RedirtyCardsQueue& queue, void* value) {
+  if (!try_enqueue(queue, value)) {
+    BufferNode* old_node = exchange_buffer_with_new(queue);
+    if (old_node != nullptr) {
+      enqueue_completed_buffer(old_node);
+    }
+    retry_enqueue(queue, value);
+  }
+}
+
 void G1RedirtyCardsLocalQueueSet::flush() {
   _shared_qset->add_bufferlist(_buffers);
   _buffers = G1BufferNodeList();
@@ -70,10 +80,6 @@ G1RedirtyCardsQueue::~G1RedirtyCardsQueue() {
 }
 #endif // ASSERT
 
-void G1RedirtyCardsQueue::handle_completed_buffer() {
-  enqueue_completed_buffer();
-}
-
 void G1RedirtyCardsQueue::flush() {
   flush_impl();
 }

src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp

@@ -29,6 +29,7 @@
 #include "gc/shared/ptrQueue.hpp"
 #include "memory/padded.hpp"
 
+class G1RedirtyCardsQueue;
 class G1RedirtyCardsQueueSet;
 
 // Provide G1RedirtyCardsQueue with a thread-local qset.  It provides an
@@ -42,6 +43,8 @@ class G1RedirtyCardsLocalQueueSet : public PtrQueueSet {
   G1RedirtyCardsLocalQueueSet(G1RedirtyCardsQueueSet* shared_qset);
   ~G1RedirtyCardsLocalQueueSet() NOT_DEBUG(= default);
 
+  void enqueue(G1RedirtyCardsQueue& queue, void* value);
+
   // Add the buffer to the local list.
   virtual void enqueue_completed_buffer(BufferNode* node);
 
@@ -51,9 +54,6 @@ class G1RedirtyCardsLocalQueueSet : public PtrQueueSet {
 
 // Worker-local queues of card table entries.
 class G1RedirtyCardsQueue : public PtrQueue {
-protected:
-  virtual void handle_completed_buffer();
-
 public:
   G1RedirtyCardsQueue(G1RedirtyCardsLocalQueueSet* qset);
   ~G1RedirtyCardsQueue() NOT_DEBUG(= default);

src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1BarrierSet.inline.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1SATBMarkQueueSet.hpp"
 #include "gc/g1/g1ThreadLocalData.hpp"
@@ -37,10 +38,11 @@ G1SATBMarkQueueSet::G1SATBMarkQueueSet(BufferNode::Allocator* allocator) :
 {}
 
 void G1SATBMarkQueueSet::handle_zero_index_for_thread(Thread* t) {
-  G1ThreadLocalData::satb_mark_queue(t).handle_zero_index();
+  G1SATBMarkQueueSet& qset = G1BarrierSet::satb_mark_queue_set();
+  qset.handle_zero_index(qset.satb_queue_for_thread(t));
 }
 
-SATBMarkQueue& G1SATBMarkQueueSet::satb_queue_for_thread(Thread* const t) const{
+SATBMarkQueue& G1SATBMarkQueueSet::satb_queue_for_thread(Thread* const t) const {
   return G1ThreadLocalData::satb_mark_queue(t);
 }
 
@@ -113,6 +115,6 @@ class G1SATBMarkQueueFilterFn {
   }
 };
 
-void G1SATBMarkQueueSet::filter(SATBMarkQueue* queue) {
+void G1SATBMarkQueueSet::filter(SATBMarkQueue& queue) {
   apply_filter(G1SATBMarkQueueFilterFn(), queue);
 }