8338912: CDS: Segmented roots array

Reviewed-by: ccheung, iklam

Author: shipilev

src/hotspot/share/cds/archiveBuilder.cpp

@@ -1104,6 +1104,17 @@ class ArchiveBuilder::CDSMapLogger : AllStatic {
 
     LogStreamHandle(Info, cds, map) st;
 
+    HeapRootSegments segments = heap_info->heap_root_segments();
+    assert(segments.base_offset() == 0, "Sanity");
+
+    for (size_t seg_idx = 0; seg_idx < segments.count(); seg_idx++) {
+      address requested_start = ArchiveHeapWriter::buffered_addr_to_requested_addr(start);
+      st.print_cr(PTR_FORMAT ": Heap roots segment [%d]",
+                  p2i(requested_start), segments.size_in_elems(seg_idx));
+      start += segments.size_in_bytes(seg_idx);
+    }
+    log_heap_roots();
+
     while (start < end) {
       size_t byte_size;
       oop source_oop = ArchiveHeapWriter::buffered_addr_to_source_obj(start);
@@ -1114,12 +1125,6 @@ class ArchiveBuilder::CDSMapLogger : AllStatic {
         // This is a regular oop that got archived.
         print_oop_with_requested_addr_cr(&st, source_oop, false);
         byte_size = source_oop->size() * BytesPerWord;
-      } else if (start == ArchiveHeapWriter::buffered_heap_roots_addr()) {
-        // HeapShared::roots() is copied specially, so it doesn't exist in
-        // ArchiveHeapWriter::BufferOffsetToSourceObjectTable.
-        // See ArchiveHeapWriter::copy_roots_to_buffer().
-        st.print_cr("HeapShared::roots[%d]", HeapShared::pending_roots()->length());
-        byte_size = ArchiveHeapWriter::heap_roots_word_size() * BytesPerWord;
       } else if ((byte_size = ArchiveHeapWriter::get_filler_size_at(start)) > 0) {
         // We have a filler oop, which also does not exist in BufferOffsetToSourceObjectTable.
         st.print_cr("filler " SIZE_FORMAT " bytes", byte_size);
@@ -1132,8 +1137,6 @@ class ArchiveBuilder::CDSMapLogger : AllStatic {
 
       if (source_oop != nullptr) {
         log_oop_details(heap_info, source_oop, /*buffered_addr=*/start);
-      } else if (start == ArchiveHeapWriter::buffered_heap_roots_addr()) {
-        log_heap_roots();
       }
       start = oop_end;
     }

src/hotspot/share/cds/archiveHeapLoader.cpp

@@ -374,8 +374,18 @@ void ArchiveHeapLoader::finish_initialization() {
   if (is_in_use()) {
     patch_native_pointers();
     intptr_t bottom = is_loaded() ? _loaded_heap_bottom : _mapped_heap_bottom;
-    intptr_t roots_oop = bottom + FileMapInfo::current_info()->heap_roots_offset();
-    HeapShared::init_roots(cast_to_oop(roots_oop));
+
+    // The heap roots are stored in one or more segments that are laid out consecutively.
+    // The byte size of each segment (except for the last one) is max_size.
+    HeapRootSegments segments = FileMapInfo::current_info()->heap_root_segments();
+    int max_size = segments.max_size_in_bytes();
+    HeapShared::init_root_segment_sizes(max_size);
+    intptr_t first_segment_addr = bottom + segments.base_offset();
+    for (size_t c = 0; c < segments.count(); c++) {
+      oop segment_oop = cast_to_oop(first_segment_addr + (c * max_size));
+      assert(segment_oop->is_objArray(), "Must be");
+      HeapShared::add_root_segment((objArrayOop)segment_oop);
+    }
   }
 }
 

src/hotspot/share/cds/archiveHeapWriter.cpp

@@ -52,9 +52,9 @@ GrowableArrayCHeap<u1, mtClassShared>* ArchiveHeapWriter::_buffer = nullptr;
 
 // The following are offsets from buffer_bottom()
 size_t ArchiveHeapWriter::_buffer_used;
-size_t ArchiveHeapWriter::_heap_roots_offset;
 
-size_t ArchiveHeapWriter::_heap_roots_word_size;
+// Heap root segments
+HeapRootSegments ArchiveHeapWriter::_heap_root_segments;
 
 address ArchiveHeapWriter::_requested_bottom;
 address ArchiveHeapWriter::_requested_top;
@@ -164,10 +164,6 @@ address ArchiveHeapWriter::buffered_addr_to_requested_addr(address buffered_addr
   return _requested_bottom + buffered_address_to_offset(buffered_addr);
 }
 
-oop ArchiveHeapWriter::heap_roots_requested_address() {
-  return cast_to_oop(_requested_bottom + _heap_roots_offset);
-}
-
 address ArchiveHeapWriter::requested_address() {
   assert(_buffer != nullptr, "must be initialized");
   return _requested_bottom;
@@ -186,47 +182,71 @@ void ArchiveHeapWriter::ensure_buffer_space(size_t min_bytes) {
   _buffer->at_grow(to_array_index(min_bytes));
 }
 
-void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots) {
-  Klass* k = Universe::objectArrayKlass(); // already relocated to point to archived klass
-  int length = roots->length();
-  _heap_roots_word_size = objArrayOopDesc::object_size(length);
-  size_t byte_size = _heap_roots_word_size * HeapWordSize;
-  if (byte_size >= MIN_GC_REGION_ALIGNMENT) {
-    log_error(cds, heap)("roots array is too large. Please reduce the number of classes");
-    vm_exit(1);
-  }
+objArrayOop ArchiveHeapWriter::allocate_root_segment(size_t offset, int element_count) {
+  HeapWord* mem = offset_to_buffered_address<HeapWord *>(offset);
+  memset(mem, 0, objArrayOopDesc::object_size(element_count));
 
-  maybe_fill_gc_region_gap(byte_size);
-
-  size_t new_used = _buffer_used + byte_size;
-  ensure_buffer_space(new_used);
+  // The initialization code is copied from MemAllocator::finish and ObjArrayAllocator::initialize.
+  oopDesc::set_mark(mem, markWord::prototype());
+  oopDesc::release_set_klass(mem, Universe::objectArrayKlass());
+  arrayOopDesc::set_length(mem, element_count);
+  return objArrayOop(cast_to_oop(mem));
+}
 
-  HeapWord* mem = offset_to_buffered_address<HeapWord*>(_buffer_used);
-  memset(mem, 0, byte_size);
-  {
-    // This is copied from MemAllocator::finish
-    oopDesc::set_mark(mem, markWord::prototype());
-    oopDesc::release_set_klass(mem, k);
-  }
-  {
-    // This is copied from ObjArrayAllocator::initialize
-    arrayOopDesc::set_length(mem, length);
+void ArchiveHeapWriter::root_segment_at_put(objArrayOop segment, int index, oop root) {
+  // Do not use arrayOop->obj_at_put(i, o) as arrayOop is outside the real heap!
+  if (UseCompressedOops) {
+    *segment->obj_at_addr<narrowOop>(index) = CompressedOops::encode(root);
+  } else {
+    *segment->obj_at_addr<oop>(index) = root;
   }
+}
 
-  objArrayOop arrayOop = objArrayOop(cast_to_oop(mem));
-  for (int i = 0; i < length; i++) {
-    // Do not use arrayOop->obj_at_put(i, o) as arrayOop is outside of the real heap!
-    oop o = roots->at(i);
-    if (UseCompressedOops) {
-      * arrayOop->obj_at_addr<narrowOop>(i) = CompressedOops::encode(o);
-    } else {
-      * arrayOop->obj_at_addr<oop>(i) = o;
+void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots) {
+  // Depending on the number of classes we are archiving, a single roots array may be
+  // larger than MIN_GC_REGION_ALIGNMENT. Roots are allocated first in the buffer, which
+  // allows us to chop the large array into a series of "segments". Current layout
+  // starts with zero or more segments exactly fitting MIN_GC_REGION_ALIGNMENT, and end
+  // with a single segment that may be smaller than MIN_GC_REGION_ALIGNMENT.
+  // This is simple and efficient. We do not need filler objects anywhere between the segments,
+  // or immediately after the last segment. This allows starting the object dump immediately
+  // after the roots.
+
+  assert((_buffer_used % MIN_GC_REGION_ALIGNMENT) == 0,
+         "Pre-condition: Roots start at aligned boundary: " SIZE_FORMAT, _buffer_used);
+
+  int max_elem_count = ((MIN_GC_REGION_ALIGNMENT - arrayOopDesc::header_size_in_bytes()) / heapOopSize);
+  assert(objArrayOopDesc::object_size(max_elem_count)*HeapWordSize == MIN_GC_REGION_ALIGNMENT,
+         "Should match exactly");
+
+  HeapRootSegments segments(_buffer_used,
+                            roots->length(),
+                            MIN_GC_REGION_ALIGNMENT,
+                            max_elem_count);
+
+  for (size_t seg_idx = 0; seg_idx < segments.count(); seg_idx++) {
+    int size_elems = segments.size_in_elems(seg_idx);
+    size_t size_bytes = segments.size_in_bytes(seg_idx);
+
+    size_t oop_offset = _buffer_used;
+    _buffer_used = oop_offset + size_bytes;
+    ensure_buffer_space(_buffer_used);
+
+    assert((oop_offset % MIN_GC_REGION_ALIGNMENT) == 0,
+           "Roots segment " SIZE_FORMAT " start is not aligned: " SIZE_FORMAT,
+           segments.count(), oop_offset);
+
+    int root_index = 0;
+    objArrayOop seg_oop = allocate_root_segment(oop_offset, size_elems);
+    for (int i = 0; i < size_elems; i++) {
+      root_segment_at_put(seg_oop, i, roots->at(root_index++));
     }
+
+    log_info(cds, heap)("archived obj root segment [%d] = " SIZE_FORMAT " bytes, obj = " PTR_FORMAT,
+                        size_elems, size_bytes, p2i(seg_oop));
   }
-  log_info(cds, heap)("archived obj roots[%d] = " SIZE_FORMAT " bytes, klass = %p, obj = %p", length, byte_size, k, mem);
 
-  _heap_roots_offset = _buffer_used;
-  _buffer_used = new_used;
+  _heap_root_segments = segments;
 }
 
 static int oop_sorting_rank(oop o) {
@@ -282,6 +302,10 @@ void ArchiveHeapWriter::sort_source_objs() {
 }
 
 void ArchiveHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots) {
+  // There could be multiple root segments, which we want to be aligned by region.
+  // Putting them ahead of objects makes sure we waste no space.
+  copy_roots_to_buffer(roots);
+
   sort_source_objs();
   for (int i = 0; i < _source_objs_order->length(); i++) {
     int src_obj_index = _source_objs_order->at(i)._index;
@@ -295,8 +319,6 @@ void ArchiveHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeap<oop, mtCla
     _buffer_offset_to_source_obj_table->maybe_grow();
   }
 
-  copy_roots_to_buffer(roots);
-
   log_info(cds)("Size of heap region = " SIZE_FORMAT " bytes, %d objects, %d roots, %d native ptrs",
                 _buffer_used, _source_objs->length() + 1, roots->length(), _num_native_ptrs);
 }
@@ -455,7 +477,7 @@ void ArchiveHeapWriter::set_requested_address(ArchiveHeapInfo* info) {
 
   info->set_buffer_region(MemRegion(offset_to_buffered_address<HeapWord*>(0),
                                     offset_to_buffered_address<HeapWord*>(_buffer_used)));
-  info->set_heap_roots_offset(_heap_roots_offset);
+  info->set_heap_root_segments(_heap_root_segments);
 }
 
 // Oop relocation
@@ -543,12 +565,6 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s
   }
 }
 
-// Relocate an element in the buffered copy of HeapShared::roots()
-template <typename T> void ArchiveHeapWriter::relocate_root_at(oop requested_roots, int index, CHeapBitMap* oopmap) {
-  size_t offset = (size_t)((objArrayOop)requested_roots)->obj_at_offset<T>(index);
-  relocate_field_in_buffer<T>((T*)(buffered_heap_roots_addr() + offset), oopmap);
-}
-
 class ArchiveHeapWriter::EmbeddedOopRelocator: public BasicOopIterateClosure {
   oop _src_obj;
   address _buffered_obj;
@@ -600,14 +616,24 @@ void ArchiveHeapWriter::relocate_embedded_oops(GrowableArrayCHeap<oop, mtClassSh
 
   // Relocate HeapShared::roots(), which is created in copy_roots_to_buffer() and
   // doesn't have a corresponding src_obj, so we can't use EmbeddedOopRelocator on it.
-  oop requested_roots = requested_obj_from_buffer_offset(_heap_roots_offset);
-  update_header_for_requested_obj(requested_roots, nullptr, Universe::objectArrayKlass());
-  int length = roots != nullptr ? roots->length() : 0;
-  for (int i = 0; i < length; i++) {
+  for (size_t seg_idx = 0; seg_idx < _heap_root_segments.count(); seg_idx++) {
+    size_t seg_offset = _heap_root_segments.segment_offset(seg_idx);
+
+    objArrayOop requested_obj = (objArrayOop)requested_obj_from_buffer_offset(seg_offset);
+    update_header_for_requested_obj(requested_obj, nullptr, Universe::objectArrayKlass());
+    address buffered_obj = offset_to_buffered_address<address>(seg_offset);
+    int length = _heap_root_segments.size_in_elems(seg_idx);
+
     if (UseCompressedOops) {
-      relocate_root_at<narrowOop>(requested_roots, i, heap_info->oopmap());
+      for (int i = 0; i < length; i++) {
+        narrowOop* addr = (narrowOop*)(buffered_obj + objArrayOopDesc::obj_at_offset<narrowOop>(i));
+        relocate_field_in_buffer<narrowOop>(addr, heap_info->oopmap());
+      }
     } else {
-      relocate_root_at<oop>(requested_roots, i, heap_info->oopmap());
+      for (int i = 0; i < length; i++) {
+        oop* addr = (oop*)(buffered_obj + objArrayOopDesc::obj_at_offset<oop>(i));
+        relocate_field_in_buffer<oop>(addr, heap_info->oopmap());
+      }
     }
   }
 

src/hotspot/share/cds/archiveHeapWriter.hpp

@@ -41,8 +41,7 @@ class ArchiveHeapInfo {
   MemRegion _buffer_region;             // Contains the archived objects to be written into the CDS archive.
   CHeapBitMap _oopmap;
   CHeapBitMap _ptrmap;
-  size_t _heap_roots_offset;            // Offset of the HeapShared::roots() object, from the bottom
-                                        // of the archived heap objects, in bytes.
+  HeapRootSegments _heap_root_segments;
 
 public:
   ArchiveHeapInfo() : _buffer_region(), _oopmap(128, mtClassShared), _ptrmap(128, mtClassShared) {}
@@ -57,8 +56,8 @@ class ArchiveHeapInfo {
   CHeapBitMap* oopmap() { return &_oopmap; }
   CHeapBitMap* ptrmap() { return &_ptrmap; }
 
-  void set_heap_roots_offset(size_t n) { _heap_roots_offset = n; }
-  size_t heap_roots_offset() const { return _heap_roots_offset; }
+  void set_heap_root_segments(HeapRootSegments segments) { _heap_root_segments = segments; };
+  HeapRootSegments heap_root_segments() { return _heap_root_segments; }
 };
 
 #if INCLUDE_CDS_JAVA_HEAP
@@ -130,9 +129,8 @@ class ArchiveHeapWriter : AllStatic {
   // The number of bytes that have written into _buffer (may be smaller than _buffer->length()).
   static size_t _buffer_used;
 
-  // The bottom of the copy of Heap::roots() inside this->_buffer.
-  static size_t _heap_roots_offset;
-  static size_t _heap_roots_word_size;
+  // The heap root segments information.
+  static HeapRootSegments _heap_root_segments;
 
   // The address range of the requested location of the archived heap objects.
   static address _requested_bottom;
@@ -193,6 +191,8 @@ class ArchiveHeapWriter : AllStatic {
     return buffered_addr - buffer_bottom();
   }
 
+  static void root_segment_at_put(objArrayOop segment, int index, oop root);
+  static objArrayOop allocate_root_segment(size_t offset, int element_count);
   static void copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots);
   static void copy_source_objs_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots);
   static size_t copy_one_source_obj_to_buffer(oop src_obj);
@@ -219,7 +219,6 @@ class ArchiveHeapWriter : AllStatic {
   template <typename T> static T* requested_addr_to_buffered_addr(T* p);
   template <typename T> static void relocate_field_in_buffer(T* field_addr_in_buffer, CHeapBitMap* oopmap);
   template <typename T> static void mark_oop_pointer(T* buffered_addr, CHeapBitMap* oopmap);
-  template <typename T> static void relocate_root_at(oop requested_roots, int index, CHeapBitMap* oopmap);
 
   static void update_header_for_requested_obj(oop requested_obj, oop src_obj, Klass* src_klass);
 
@@ -234,13 +233,6 @@ class ArchiveHeapWriter : AllStatic {
   static bool is_string_too_large_to_archive(oop string);
   static void write(GrowableArrayCHeap<oop, mtClassShared>*, ArchiveHeapInfo* heap_info);
   static address requested_address();  // requested address of the lowest achived heap object
-  static oop heap_roots_requested_address(); // requested address of HeapShared::roots()
-  static address buffered_heap_roots_addr() {
-    return offset_to_buffered_address<address>(_heap_roots_offset);
-  }
-  static size_t heap_roots_word_size() {
-    return _heap_roots_word_size;
-  }
   static size_t get_filler_size_at(address buffered_addr);
 
   static void mark_native_pointer(oop src_obj, int offset);

src/hotspot/share/cds/archiveUtils.cpp

@@ -369,3 +369,24 @@ void ArchiveUtils::log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) {
     }
   }
 }
+
+size_t HeapRootSegments::size_in_bytes(size_t seg_idx) {
+  assert(seg_idx < _count, "In range");
+  return objArrayOopDesc::object_size(size_in_elems(seg_idx)) * HeapWordSize;
+}
+
+int HeapRootSegments::size_in_elems(size_t seg_idx) {
+  assert(seg_idx < _count, "In range");
+  if (seg_idx != _count - 1) {
+    return _max_size_in_elems;
+  } else {
+    // Last slice, leftover
+    return _roots_count % _max_size_in_elems;
+  }
+}
+
+size_t HeapRootSegments::segment_offset(size_t seg_idx) {
+  assert(seg_idx < _count, "In range");
+  return _base_offset + seg_idx * _max_size_in_bytes;
+}
+

src/hotspot/share/cds/archiveUtils.hpp

@@ -250,4 +250,45 @@ class ArchiveUtils {
   static void log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) NOT_CDS_RETURN;
 };
 
+class HeapRootSegments {
+private:
+  size_t _base_offset;
+  size_t _count;
+  int _roots_count;
+  int _max_size_in_bytes;
+  int _max_size_in_elems;
+
+public:
+  size_t base_offset() { return _base_offset; }
+  size_t count() { return _count; }
+  int roots_count() { return _roots_count; }
+  int max_size_in_bytes() { return _max_size_in_bytes; }
+  int max_size_in_elems() { return _max_size_in_elems; }
+
+  size_t size_in_bytes(size_t seg_idx);
+  int size_in_elems(size_t seg_idx);
+  size_t segment_offset(size_t seg_idx);
+
+  // Trivial copy assignments are allowed to copy the entire object representation.
+  // We also inline this class into archive header. Therefore, it is important to make
+  // sure any gaps in object representation are initialized to zeroes. This is why
+  // constructors memset before doing field assignments.
+  HeapRootSegments() {
+    memset(this, 0, sizeof(*this));
+  }
+  HeapRootSegments(size_t base_offset, int roots_count, int max_size_in_bytes, int max_size_in_elems) {
+    assert(is_power_of_2(max_size_in_bytes), "must be");
+    memset(this, 0, sizeof(*this));
+    _base_offset = base_offset;
+    _count = (roots_count + max_size_in_elems - 1) / max_size_in_elems;
+    _roots_count = roots_count;
+    _max_size_in_bytes = max_size_in_bytes;
+    _max_size_in_elems = max_size_in_elems;
+  }
+
+  // This class is trivially copyable and assignable.
+  HeapRootSegments(const HeapRootSegments&) = default;
+  HeapRootSegments& operator=(const HeapRootSegments&) = default;
+};
+
 #endif // SHARE_CDS_ARCHIVEUTILS_HPP