8323211: Implement new Serial Full GC

src/hotspot/share/gc/serial/serialCompressor.hpp

@@ -0,0 +1,116 @@
+
+#ifndef SHARE_GC_SERIAL_SERIALCOMPRESSOR_HPP
+#define SHARE_GC_SERIAL_SERIALCOMPRESSOR_HPP
+
+#include "gc/shared/gcTrace.hpp"
+#include "gc/shared/markBitMap.hpp"
+#include "gc/shared/space.hpp"
+#include "gc/shared/stringdedup/stringDedup.hpp"
+#include "gc/shared/taskqueue.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+#include "utilities/stack.hpp"
+
+class ReferenceProcessor;
+class STWGCTimer;
+
+/**
+ * Implements compacting full-GC for the Serial GC. This is based on
+ * Abuaiadh et al. [2004] and Kermany and Petrank [2006], as described in
+ * The Garbage Collection Handbook, Second Edition by Jones, Hosking and Moss [2023].
+ *
+ * The Full GC is carried out in 3 phases:
+ * 1. Marking
+ * 2. Preparation
+ * 3. Compaction
+ *
+ * The algorithm uses 2 major data-structures:
+ * - A marking bitmap. Each bit represents one word of the heap (or larger blocks
+ *   according to MinObjAlignment).
+ * - A block-offset-table. Each word of the table stores the destination address
+ *   of each block of the heap. A block spans 64 words of the heap. Note that the
+ *   sizes have been chosen such that we achieve a reasonable compromise between
+ *   the size of the table (1/64th of the heap size) and performance (for each
+ *   forwarding, we only need to scan at most 64 bits - which can be done very
+ *   efficiently, see population_count.hpp.
+ *
+ * The algorithm then works as follows:
+ *
+ * 1. Marking: This is pretty much a textbook marking algorithm, with the difference
+ *    that we are setting one bit for each live object in the heap, not only one
+ *    bit per object. We are going to use this information to calculate the
+ *    forwarding pointers of each object.
+ * 2. Preparation: Here we are building the block-offset-table. The basic idea
+ *    is to scan the heap bottom to top, keep track of compaction-top for each
+ *    block and record the compaction target for the first live word of each
+ *    block in the block-offset-table. (Notice that the first live word of a block
+ *    will often be from an object that is overlapping from a previous block.)
+ *    Later (during compaction) we can easily calculate the forwarding address
+ *    of each object by finding its block, loading the corresponding
+ *    block-destination, and adding the number of live words preceding the object
+ *    in its block:
+ *    forwarding(obj) = bot[block(obj)] + count_live_words(block_base(obj), obj)
+ * 3. Compaction: This compacts the heap and updates all references in a single
+ *    sequential pass over the heap.
+ *    Scan heap bottom to top, for each live object:
+ *    - Update all its references to point to their forwarded locations
+ *    - Copy the object itself to its forwarded location
+ *
+ * Notice that the actual implementation is more complex than this description.
+ * In particular, during marking, we also need to take care of reference-processing,
+ * class-unloading, string-deduplication. The preparation phase is complicated by
+ * the heap being divided into generations and spaces - we need to ensure that whole
+ * blocks are compacted into the same space, and that all its objects, including
+ * the tails that overlap into an adjacent block, fit into the destination space.
+ */
+
+class SerialCompressor : public StackObj {
+  friend class SCDeadSpacer;
+  friend class SCFollowRootClosure;
+  friend class SCFollowStackClosure;
+  friend class SCKeepAliveClosure;
+  friend class SCMarkAndPushClosure;
+private:
+
+  static uint _total_invocations;
+
+  // Memory area of the underlying marking bitmap.
+  MemRegion  _mark_bitmap_region;
+  // The marking bitmap.
+  MarkBitMap _mark_bitmap;
+  // The marking stack.
+  Stack<oop,mtGC> _marking_stack;
+  // Separate marking stack for object-array-chunks.
+  Stack<ObjArrayTask, mtGC> _objarray_stack;
+
+  // String-dedup support.
+  StringDedup::Requests _string_dedup_requests;
+
+  STWGCTimer* _gc_timer;
+  SerialOldTracer _gc_tracer;
+  ReferenceProcessor* _ref_processor;
+
+  // Phase 1: Marking. (Phases 2 and 3 are implemented in the SCCompacter class,
+  // see serialCompressor.cpp)
+  void phase1_mark(bool clear_all_softrefs);
+
+  // Various marking support methods.
+  bool mark_object(oop obj);
+  void follow_array(objArrayOop array);
+  void follow_array_chunk(objArrayOop array, int index);
+  void follow_object(oop obj);
+  void push_objarray(objArrayOop array, size_t index);
+  ReferenceProcessor* ref_processor() const { return _ref_processor; }
+  void follow_stack();
+  template<class T>
+  void mark_and_push(T* ptr);
+
+public:
+  explicit SerialCompressor(STWGCTimer* gc_timer);
+  ~SerialCompressor();
+
+  // Entry point.
+  void invoke_at_safepoint(bool clear_all_softrefs);
+};
+
+#endif // SHARE_GC_SERIAL_SERIALCOMPRESSOR_HPP

src/hotspot/share/gc/serial/tenuredGeneration.cpp

@@ -26,6 +26,7 @@
 #include "gc/serial/cardTableRS.hpp"
 #include "gc/serial/genMarkSweep.hpp"
 #include "gc/serial/serialBlockOffsetTable.inline.hpp"
+#include "gc/serial/serialCompressor.hpp"
 #include "gc/serial/serialHeap.hpp"
 #include "gc/serial/tenuredGeneration.inline.hpp"
 #include "gc/shared/collectorCounters.hpp"
@@ -452,7 +453,12 @@ void TenuredGeneration::collect(bool   full,
 
   gch->pre_full_gc_dump(gc_timer);
 
-  GenMarkSweep::invoke_at_safepoint(clear_all_soft_refs);
+  if (UseCompressorFullGC) {
+    SerialCompressor full_gc = SerialCompressor(gc_timer);
+    full_gc.invoke_at_safepoint(clear_all_soft_refs);
+  } else {
+    GenMarkSweep::invoke_at_safepoint(clear_all_soft_refs);
+  }
 
   gch->post_full_gc_dump(gc_timer);
 

src/hotspot/share/gc/shared/gc_globals.hpp

@@ -665,7 +665,11 @@
   product(uint, GCCardSizeInBytes, 512,                                     \
           "Card table entry size (in bytes) for card based collectors")     \
           range(128, NOT_LP64(512) LP64_ONLY(1024))                         \
-          constraint(GCCardSizeInBytesConstraintFunc,AtParse)
+          constraint(GCCardSizeInBytesConstraintFunc,AtParse)               \
+                                                                            \
+  product(bool, UseCompressorFullGC, false, EXPERIMENTAL,                   \
+          "Use compressor-style full GC")                                   \
+
   // end of GC_FLAGS
 
 DECLARE_FLAGS(GC_FLAGS)

src/hotspot/share/gc/shared/markBitMap.hpp

@@ -83,6 +83,10 @@ class MarkBitMap {
   // such bit, returns "limit" if that is non-null, or else "endWord()".
   inline HeapWord* get_next_marked_addr(const HeapWord* addr,
                                         HeapWord* limit) const;
+  inline HeapWord* get_next_unmarked_addr(const HeapWord* addr,
+                                          HeapWord* limit) const;
+  inline HeapWord* get_last_unmarked_addr(const HeapWord* start,
+                                          HeapWord* limit) const;
 
   void print_on_error(outputStream* st, const char* prefix) const;
 
@@ -94,10 +98,16 @@ class MarkBitMap {
   inline bool par_mark(HeapWord* addr);
   inline bool par_mark(oop obj);
 
+  inline void mark_range(HeapWord* addr, size_t num_words);
+
   // Clear bitmap.
   void clear()                         { do_clear(_covered, true); }
   void clear_range(MemRegion mr)       { do_clear(mr, false);      }
   void clear_range_large(MemRegion mr) { do_clear(mr, true);       }
+
+  // Count the number of marked words in the range [start, end).
+  inline size_t count_marked_words(HeapWord* start, HeapWord* end) const;
+  inline size_t count_marked_words_in_block(HeapWord* start, HeapWord* end) const;
 };
 
 #endif // SHARE_GC_SHARED_MARKBITMAP_HPP

src/hotspot/share/gc/shared/markBitMap.inline.hpp

@@ -43,6 +43,26 @@ inline HeapWord* MarkBitMap::get_next_marked_addr(const HeapWord* const addr,
   return offset_to_addr(nextOffset);
 }
 
+inline HeapWord* MarkBitMap::get_next_unmarked_addr(const HeapWord* const addr,
+                                                    HeapWord* const limit) const {
+  assert(limit != nullptr, "limit must not be null");
+  // Round addr up to a possible object boundary to be safe.
+  size_t const addr_offset = addr_to_offset(align_up(addr, HeapWordSize << _shifter));
+  size_t const limit_offset = addr_to_offset(limit);
+  size_t const nextOffset = _bm.find_first_clear_bit(addr_offset, limit_offset);
+  return offset_to_addr(nextOffset);
+}
+
+inline HeapWord* MarkBitMap::get_last_unmarked_addr(const HeapWord* const start,
+                                                    HeapWord* const limit) const {
+  assert(limit != nullptr, "limit must not be null");
+  // Round addr up to a possible object boundary to be safe.
+  size_t const start_offset = addr_to_offset(align_up(start, HeapWordSize << _shifter));
+  size_t const limit_offset = addr_to_offset(limit);
+  size_t const nextOffset = _bm.find_last_clear_bit(start_offset, limit_offset);
+  return offset_to_addr(nextOffset);
+}
+
 inline void MarkBitMap::mark(HeapWord* addr) {
   check_mark(addr);
   _bm.set_bit(addr_to_offset(addr));
@@ -52,6 +72,12 @@ inline void MarkBitMap::mark(oop obj) {
   return mark(cast_from_oop<HeapWord*>(obj));
 }
 
+inline void MarkBitMap::mark_range(HeapWord* addr, size_t num_words) {
+  size_t beg = addr_to_offset(addr);
+  size_t end = addr_to_offset(addr + num_words);
+  _bm.set_range(beg, end);
+}
+
 inline void MarkBitMap::clear(HeapWord* addr) {
   check_mark(addr);
   _bm.clear_bit(addr_to_offset(addr));
@@ -74,4 +100,12 @@ inline void MarkBitMap::clear(oop obj) {
   clear(cast_from_oop<HeapWord*>(obj));
 }
 
+inline size_t MarkBitMap::count_marked_words(HeapWord* start, HeapWord* end) const {
+  return _bm.count_one_bits(addr_to_offset(start), addr_to_offset(end)) /*<< _shifter */;
+}
+
+inline size_t MarkBitMap::count_marked_words_in_block(HeapWord* start, HeapWord* end) const {
+  return _bm.count_one_bits_within_word(addr_to_offset(start), addr_to_offset(end)) /*<< _shifter */;
+}
+
 #endif // SHARE_GC_SHARED_MARKBITMAP_INLINE_HPP

src/hotspot/share/gc/shared/space.hpp

@@ -174,7 +174,7 @@ class ContiguousSpace: public Space {
 
  public:
   ContiguousSpace();
-  ~ContiguousSpace();
+  virtual ~ContiguousSpace();
 
   // Initialization.
   // "initialize" should be called once on a space, before it is used for

src/hotspot/share/memory/metaspace/commitMask.hpp

@@ -26,7 +26,7 @@
 #ifndef SHARE_MEMORY_METASPACE_COMMITMASK_HPP
 #define SHARE_MEMORY_METASPACE_COMMITMASK_HPP
 
-#include "utilities/bitMap.hpp"
+#include "utilities/bitMap.inline.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
 

src/hotspot/share/utilities/bitMap.cpp

@@ -177,15 +177,6 @@ void BitMap::pretouch() {
   os::pretouch_memory(word_addr(0), word_addr(size()));
 }
 
-void BitMap::set_range_within_word(idx_t beg, idx_t end) {
-  // With a valid range (beg <= end), this test ensures that end != 0, as
-  // required by inverted_bit_mask_for_range.  Also avoids an unnecessary write.
-  if (beg != end) {
-    bm_word_t mask = inverted_bit_mask_for_range(beg, end);
-    *word_addr(beg) |= ~mask;
-  }
-}
-
 void BitMap::clear_range_within_word(idx_t beg, idx_t end) {
   // With a valid range (beg <= end), this test ensures that end != 0, as
   // required by inverted_bit_mask_for_range.  Also avoids an unnecessary write.
@@ -213,25 +204,6 @@ void BitMap::par_put_range_within_word(idx_t beg, idx_t end, bool value) {
   }
 }
 
-void BitMap::set_range(idx_t beg, idx_t end) {
-  verify_range(beg, end);
-
-  idx_t beg_full_word = to_words_align_up(beg);
-  idx_t end_full_word = to_words_align_down(end);
-
-  if (beg_full_word < end_full_word) {
-    // The range includes at least one full word.
-    set_range_within_word(beg, bit_index(beg_full_word));
-    set_range_of_words(beg_full_word, end_full_word);
-    set_range_within_word(bit_index(end_full_word), end);
-  } else {
-    // The range spans at most 2 partial words.
-    idx_t boundary = MIN2(bit_index(beg_full_word), end);
-    set_range_within_word(beg, boundary);
-    set_range_within_word(boundary, end);
-  }
-}
-
 void BitMap::clear_range(idx_t beg, idx_t end) {
   verify_range(beg, end);
 
@@ -595,57 +567,6 @@ void BitMap::clear_large() {
   clear_large_range_of_words(0, size_in_words());
 }
 
-BitMap::idx_t BitMap::count_one_bits_in_range_of_words(idx_t beg_full_word, idx_t end_full_word) const {
-  idx_t sum = 0;
-  for (idx_t i = beg_full_word; i < end_full_word; i++) {
-    bm_word_t w = map()[i];
-    sum += population_count(w);
-  }
-  return sum;
-}
-
-BitMap::idx_t BitMap::count_one_bits_within_word(idx_t beg, idx_t end) const {
-  if (beg != end) {
-    assert(end > beg, "must be");
-    bm_word_t mask = ~inverted_bit_mask_for_range(beg, end);
-    bm_word_t w = *word_addr(beg);
-    w &= mask;
-    return population_count(w);
-  }
-  return 0;
-}
-
-BitMap::idx_t BitMap::count_one_bits() const {
-  return count_one_bits(0, size());
-}
-
-// Returns the number of bits set within  [beg, end).
-BitMap::idx_t BitMap::count_one_bits(idx_t beg, idx_t end) const {
-  verify_range(beg, end);
-
-  idx_t beg_full_word = to_words_align_up(beg);
-  idx_t end_full_word = to_words_align_down(end);
-
-  idx_t sum = 0;
-
-  if (beg_full_word < end_full_word) {
-    // The range includes at least one full word.
-    sum += count_one_bits_within_word(beg, bit_index(beg_full_word));
-    sum += count_one_bits_in_range_of_words(beg_full_word, end_full_word);
-    sum += count_one_bits_within_word(bit_index(end_full_word), end);
-  } else {
-    // The range spans at most 2 partial words.
-    idx_t boundary = MIN2(bit_index(beg_full_word), end);
-    sum += count_one_bits_within_word(beg, boundary);
-    sum += count_one_bits_within_word(boundary, end);
-  }
-
-  assert(sum <= (end - beg), "must be");
-
-  return sum;
-
-}
-
 void BitMap::print_on_error(outputStream* st, const char* prefix) const {
   st->print_cr("%s[" PTR_FORMAT ", " PTR_FORMAT ")",
       prefix, p2i(map()), p2i((char*)map() + (size() >> LogBitsPerByte)));

src/hotspot/share/utilities/bitMap.hpp

@@ -163,7 +163,7 @@ class BitMap {
 
   // Ranges within a single word.
   bm_word_t inverted_bit_mask_for_range(idx_t beg, idx_t end) const;
-  void  set_range_within_word      (idx_t beg, idx_t end);
+  inline void  set_range_within_word      (idx_t beg, idx_t end);
   void  clear_range_within_word    (idx_t beg, idx_t end);
   void  par_put_range_within_word  (idx_t beg, idx_t end, bool value);
 
@@ -175,8 +175,10 @@ class BitMap {
 
   static void clear_range_of_words(bm_word_t* map, idx_t beg, idx_t end);
 
-  idx_t count_one_bits_within_word(idx_t beg, idx_t end) const;
-  idx_t count_one_bits_in_range_of_words(idx_t beg_full_word, idx_t end_full_word) const;
+public:
+  inline idx_t count_one_bits_within_word(idx_t beg, idx_t end) const;
+protected:
+  inline idx_t count_one_bits_in_range_of_words(idx_t beg_full_word, idx_t end_full_word) const;
 
   // Set the map and size.
   void update(bm_word_t* map, idx_t size) {
@@ -233,7 +235,7 @@ class BitMap {
   bool par_at_put(idx_t bit, bool value);
 
   // Update a range of bits.  Ranges are half-open [beg, end).
-  void set_range   (idx_t beg, idx_t end);
+  inline void set_range   (idx_t beg, idx_t end);
   void clear_range (idx_t beg, idx_t end);
   void set_large_range   (idx_t beg, idx_t end);
   void clear_large_range (idx_t beg, idx_t end);
@@ -356,10 +358,10 @@ class BitMap {
   idx_t find_last_set_bit_aligned_left(idx_t beg, idx_t end) const;
 
   // Returns the number of bits set in the bitmap.
-  idx_t count_one_bits() const;
+  inline idx_t count_one_bits() const;
 
   // Returns the number of bits set within  [beg, end).
-  idx_t count_one_bits(idx_t beg, idx_t end) const;
+  inline idx_t count_one_bits(idx_t beg, idx_t end) const;
 
   // Set operations.
   void set_union(const BitMap& bits);