8311163: Parallel: Improve large object handling during evacuation

Reviewed-by: tschatzl, ayang

Author: Kim Barrett

src/hotspot/share/gc/parallel/psPromotionManager.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,8 @@
 #include "gc/parallel/psScavenge.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/partialArrayState.hpp"
+#include "gc/shared/partialArrayTaskStepper.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "logging/log.hpp"
@@ -42,12 +44,14 @@
 #include "memory/resourceArea.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
+#include "utilities/checkedCast.hpp"
 
 PaddedEnd<PSPromotionManager>* PSPromotionManager::_manager_array = nullptr;
 PSPromotionManager::PSScannerTasksQueueSet* PSPromotionManager::_stack_array_depth = nullptr;
 PreservedMarksSet*             PSPromotionManager::_preserved_marks_set = nullptr;
 PSOldGen*                      PSPromotionManager::_old_gen = nullptr;
 MutableSpace*                  PSPromotionManager::_young_space = nullptr;
+PartialArrayStateAllocator*    PSPromotionManager::_partial_array_state_allocator = nullptr;
 
 void PSPromotionManager::initialize() {
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
@@ -62,11 +66,16 @@ void PSPromotionManager::initialize() {
   assert(_manager_array == nullptr, "Attempt to initialize twice");
   _manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(promotion_manager_num);
 
+  assert(_partial_array_state_allocator == nullptr, "Attempt to initialize twice");
+  _partial_array_state_allocator
+    = new PartialArrayStateAllocator(ParallelGCThreads);
+
   _stack_array_depth = new PSScannerTasksQueueSet(ParallelGCThreads);
 
   // Create and register the PSPromotionManager(s) for the worker threads.
   for(uint i=0; i<ParallelGCThreads; i++) {
     stack_array_depth()->register_queue(i, _manager_array[i].claimed_stack_depth());
+    _manager_array[i]._partial_array_state_allocator_index = i;
   }
   // The VMThread gets its own PSPromotionManager, which is not available
   // for work stealing.
@@ -124,6 +133,10 @@ bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) {
     manager->flush_labs();
     manager->flush_string_dedup_requests();
   }
+  // All PartialArrayStates have been returned to the allocator, since the
+  // claimed_stack_depths are all empty.  Leave them there for use by future
+  // collections.
+
   if (!promotion_failure_occurred) {
     // If there was no promotion failure, the preserved mark stacks
     // should be empty.
@@ -172,7 +185,10 @@ void PSPromotionManager::reset_stats() {
 }
 #endif // TASKQUEUE_STATS
 
-PSPromotionManager::PSPromotionManager() {
+// Most members are initialized either by initialize() or reset().
+PSPromotionManager::PSPromotionManager()
+  : _partial_array_stepper(ParallelGCThreads, ParGCArrayScanChunk)
+{
   // We set the old lab's start array.
   _old_lab.set_start_array(old_gen()->start_array());
 
@@ -182,9 +198,11 @@ PSPromotionManager::PSPromotionManager() {
     _target_stack_size = GCDrainStackTargetSize;
   }
 
-  _array_chunk_size = ParGCArrayScanChunk;
+  // Initialize to a bad value; fixed by initialize().
+  _partial_array_state_allocator_index = UINT_MAX;
+
   // let's choose 1.5x the chunk size
-  _min_array_size_for_chunking = 3 * _array_chunk_size / 2;
+  _min_array_size_for_chunking = (3 * ParGCArrayScanChunk / 2);
 
   _preserved_marks = nullptr;
 
@@ -277,37 +295,57 @@ template <class T> void PSPromotionManager::process_array_chunk_work(
   }
 }
 
-void PSPromotionManager::process_array_chunk(PartialArrayScanTask task) {
-  assert(PSChunkLargeArrays, "invariant");
-
-  oop old = task.to_source_array();
-  assert(old->is_objArray(), "invariant");
-  assert(old->is_forwarded(), "invariant");
-
+void PSPromotionManager::process_array_chunk(PartialArrayState* state) {
   TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
 
-  oop const obj = old->forwardee();
-
-  int start;
-  int const end = arrayOop(old)->length();
-  if (end > (int) _min_array_size_for_chunking) {
-    // we'll chunk more
-    start = end - _array_chunk_size;
-    assert(start > 0, "invariant");
-    arrayOop(old)->set_length(start);
-    push_depth(ScannerTask(PartialArrayScanTask(old)));
-    TASKQUEUE_STATS_ONLY(++_array_chunk_pushes);
+  // Claim a chunk.  Push additional tasks before processing the claimed
+  // chunk to allow other workers to steal while we're processing.
+  PartialArrayTaskStepper::Step step = _partial_array_stepper.next(state);
+  if (step._ncreate > 0) {
+    state->add_references(step._ncreate);
+    for (uint i = 0; i < step._ncreate; ++i) {
+      push_depth(ScannerTask(state));
+    }
+    TASKQUEUE_STATS_ONLY(_array_chunk_pushes += step._ncreate);
+  }
+  int start = checked_cast<int>(step._index);
+  int end = checked_cast<int>(step._index + _partial_array_stepper.chunk_size());
+  assert(start < end, "invariant");
+  if (UseCompressedOops) {
+    process_array_chunk_work<narrowOop>(state->destination(), start, end);
   } else {
-    // this is the final chunk for this array
-    start = 0;
-    int const actual_length = arrayOop(obj)->length();
-    arrayOop(old)->set_length(actual_length);
+    process_array_chunk_work<oop>(state->destination(), start, end);
   }
+  // Release reference to state, now that we're done with it.
+  _partial_array_state_allocator->release(_partial_array_state_allocator_index, state);
+}
 
+void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
+  assert(old_obj->is_objArray(), "precondition");
+  assert(old_obj->is_forwarded(), "precondition");
+  assert(old_obj->forwardee() == new_obj, "precondition");
+  assert(new_obj->is_objArray(), "precondition");
+
+  size_t array_length = objArrayOop(new_obj)->length();
+  PartialArrayTaskStepper::Step step = _partial_array_stepper.start(array_length);
+
+  if (step._ncreate > 0) {
+    TASKQUEUE_STATS_ONLY(++_arrays_chunked);
+    PartialArrayState* state =
+      _partial_array_state_allocator->allocate(_partial_array_state_allocator_index,
+                                               old_obj, new_obj,
+                                               step._index,
+                                               array_length,
+                                               step._ncreate);
+    for (uint i = 0; i < step._ncreate; ++i) {
+      push_depth(ScannerTask(state));
+    }
+    TASKQUEUE_STATS_ONLY(_array_chunk_pushes += step._ncreate);
+  }
   if (UseCompressedOops) {
-    process_array_chunk_work<narrowOop>(obj, start, end);
+    process_array_chunk_work<narrowOop>(new_obj, 0, checked_cast<int>(step._index));
   } else {
-    process_array_chunk_work<oop>(obj, start, end);
+    process_array_chunk_work<oop>(new_obj, 0, checked_cast<int>(step._index));
   }
 }
 

src/hotspot/share/gc/parallel/psPromotionManager.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "gc/parallel/psPromotionLAB.hpp"
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/taskqueue.hpp"
@@ -49,6 +50,8 @@
 class MutableSpace;
 class PSOldGen;
 class ParCompactionManager;
+class PartialArrayState;
+class PartialArrayStateAllocator;
 
 class PSPromotionManager {
   friend class PSScavenge;
@@ -85,7 +88,9 @@ class PSPromotionManager {
 
   uint                                _target_stack_size;
 
-  uint                                _array_chunk_size;
+  static PartialArrayStateAllocator*  _partial_array_state_allocator;
+  PartialArrayTaskStepper             _partial_array_stepper;
+  uint                                _partial_array_state_allocator_index;
   uint                                _min_array_size_for_chunking;
 
   PreservedMarks*                     _preserved_marks;
@@ -101,7 +106,8 @@ class PSPromotionManager {
 
   template <class T> void  process_array_chunk_work(oop obj,
                                                     int start, int end);
-  void process_array_chunk(PartialArrayScanTask task);
+  void process_array_chunk(PartialArrayState* state);
+  void push_objArray(oop old_obj, oop new_obj);
 
   void push_depth(ScannerTask task);
 

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -276,9 +276,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
     if (new_obj_size > _min_array_size_for_chunking &&
         new_obj->is_objArray() &&
         PSChunkLargeArrays) {
-      // we'll chunk it
-      push_depth(ScannerTask(PartialArrayScanTask(o)));
-      TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_array_chunk_pushes);
+      push_objArray(o, new_obj);
     } else {
       // we'll just push its contents
       push_contents(new_obj);
@@ -322,9 +320,9 @@ inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) {
 }
 
 inline void PSPromotionManager::process_popped_location_depth(ScannerTask task) {
-  if (task.is_partial_array_task()) {
+  if (task.is_partial_array_state()) {
     assert(PSChunkLargeArrays, "invariant");
-    process_array_chunk(task.to_partial_array_task());
+    process_array_chunk(task.to_partial_array_state());
   } else {
     if (task.is_narrow_oop_ptr()) {
       assert(UseCompressedOops, "Error");
@@ -341,7 +339,7 @@ inline bool PSPromotionManager::steal_depth(int queue_num, ScannerTask& t) {
 
 #if TASKQUEUE_STATS
 void PSPromotionManager::record_steal(ScannerTask task) {
-  if (task.is_partial_array_task()) {
+  if (task.is_partial_array_state()) {
     ++_array_chunk_steals;
   }
 }

src/hotspot/share/gc/shared/partialArrayState.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_SHARED_PARTIALARRAYSTATE_HPP
 #define SHARE_GC_SHARED_PARTIALARRAYSTATE_HPP
 
+#include "memory/allocation.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
@@ -109,7 +110,7 @@ class PartialArrayState {
 // the worker_id used in the operation.  This avoids locking and such on those
 // data structures, at the cost of possibly doing more total arena allocation
 // that would be needed with a single shared arena and free-list.
-class PartialArrayStateAllocator {
+class PartialArrayStateAllocator : public CHeapObj<mtGC> {
   class Impl;
   Impl* _impl;
 

src/hotspot/share/gc/shared/taskqueue.hpp

@@ -573,20 +573,6 @@ class ObjArrayTask
   int _index;
 };
 
-// Wrapper over an oop that is a partially scanned array.
-// Can be converted to a ScannerTask for placement in associated task queues.
-// Refers to the partially copied source array oop.
-// Temporarily retained to support ParallelGC until it adopts PartialArrayState.
-class PartialArrayScanTask {
-  oop _src;
-
-public:
-  explicit PartialArrayScanTask(oop src_array) : _src(src_array) {}
-  // Trivially copyable.
-
-  oop to_source_array() const { return _src; }
-};
-
 class PartialArrayState;
 
 // Discriminated union over oop*, narrowOop*, and PartialArrayState.
@@ -627,10 +613,6 @@ class ScannerTask {
 
   explicit ScannerTask(narrowOop* p) : _p(encode(p, NarrowOopTag)) {}
 
-  // Temporarily retained to support ParallelGC until it adopts PartialArrayState.
-  explicit ScannerTask(PartialArrayScanTask t) :
-    _p(encode(t.to_source_array(), PartialArrayTag)) {}
-
   explicit ScannerTask(PartialArrayState* state) :
     _p(encode(state, PartialArrayTag)) {}
 
@@ -646,11 +628,6 @@ class ScannerTask {
     return (raw_value() & NarrowOopTag) != 0;
   }
 
-  // Temporarily retained to support ParallelGC until it adopts PartialArrayState.
-  bool is_partial_array_task() const {
-    return (raw_value() & PartialArrayTag) != 0;
-  }
-
   bool is_partial_array_state() const {
     return (raw_value() & PartialArrayTag) != 0;
   }
@@ -663,11 +640,6 @@ class ScannerTask {
     return static_cast<narrowOop*>(decode(NarrowOopTag));
   }
 
-  // Temporarily retained to support ParallelGC until it adopts PartialArrayState.
-  PartialArrayScanTask to_partial_array_task() const {
-    return PartialArrayScanTask(cast_to_oop(decode(PartialArrayTag)));
-  }
-
   PartialArrayState* to_partial_array_state() const {
     return static_cast<PartialArrayState*>(decode(PartialArrayTag));
   }