8233093: Move CDS heap oopmaps into new MetaspaceShared::bm region

Moved the _closed_archive_heap_oopmaps and _open_archive_heap_oopmaps from the ro to the bm region.

Reviewed-by: iklam, minqi

src/hotspot/share/memory/dynamicArchive.cpp

@@ -1020,7 +1020,7 @@ void DynamicArchiveBuilder::write_archive(char* serialized_data) {
   // Now write the archived data including the file offsets.
   const char* archive_name = Arguments::GetSharedDynamicArchivePath();
   dynamic_info->open_for_write(archive_name);
-  MetaspaceShared::write_core_archive_regions(dynamic_info);
+  MetaspaceShared::write_core_archive_regions(dynamic_info, NULL, NULL);
   dynamic_info->set_final_requested_base((char*)Arguments::default_SharedBaseAddress());
   dynamic_info->set_header_crc(dynamic_info->compute_header_crc());
   dynamic_info->write_header();

src/hotspot/share/memory/filemap.cpp

@@ -1209,15 +1209,43 @@ void FileMapInfo::write_region(int region, char* base, size_t size,
   }
 }
 
+size_t FileMapInfo::set_oopmaps_offset(GrowableArray<ArchiveHeapOopmapInfo>* oopmaps, size_t curr_size) {
+  for (int i = 0; i < oopmaps->length(); i++) {
+    oopmaps->at(i)._offset = curr_size;
+    curr_size += oopmaps->at(i)._oopmap_size_in_bytes;
+  }
+  return curr_size;
+}
+
+size_t FileMapInfo::write_oopmaps(GrowableArray<ArchiveHeapOopmapInfo>* oopmaps, size_t curr_offset, uintptr_t* buffer) {
+  for (int i = 0; i < oopmaps->length(); i++) {
+    memcpy(((char*)buffer) + curr_offset, oopmaps->at(i)._oopmap, oopmaps->at(i)._oopmap_size_in_bytes);
+    curr_offset += oopmaps->at(i)._oopmap_size_in_bytes;
+  }
+  return curr_offset;
+}
 
-void FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap) {
+void FileMapInfo::write_bitmap_region(const CHeapBitMap* ptrmap,
+                                      GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
+                                      GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps) {
   ResourceMark rm;
   size_t size_in_bits = ptrmap->size();
   size_t size_in_bytes = ptrmap->size_in_bytes();
+
+  if (closed_oopmaps != NULL && open_oopmaps != NULL) {
+    size_in_bytes = set_oopmaps_offset(closed_oopmaps, size_in_bytes);
+    size_in_bytes = set_oopmaps_offset(open_oopmaps, size_in_bytes);
+  }
+
   uintptr_t* buffer = (uintptr_t*)NEW_RESOURCE_ARRAY(char, size_in_bytes);
-  ptrmap->write_to(buffer, size_in_bytes);
+  ptrmap->write_to(buffer, ptrmap->size_in_bytes());
   header()->set_ptrmap_size_in_bits(size_in_bits);
 
+  if (closed_oopmaps != NULL && open_oopmaps != NULL) {
+    size_t curr_offset = write_oopmaps(closed_oopmaps, ptrmap->size_in_bytes(), buffer);
+    write_oopmaps(open_oopmaps, curr_offset, buffer);
+  }
+
   log_debug(cds)("ptrmap = " INTPTR_FORMAT " (" SIZE_FORMAT " bytes)",
                  p2i(buffer), size_in_bytes);
   write_region(MetaspaceShared::bm, (char*)buffer, size_in_bytes, /*read_only=*/true, /*allow_exec=*/false);
@@ -1284,9 +1312,7 @@ size_t FileMapInfo::write_archive_heap_regions(GrowableArray<MemRegion> *heap_me
                    i, p2i(start), p2i(start + size), size);
     write_region(i, start, size, false, false);
     if (size > 0) {
-      address oopmap = oopmaps->at(arr_idx)._oopmap;
-      assert(oopmap >= (address)SharedBaseAddress, "must be");
-      space_at(i)->init_oopmap(oopmap - (address)SharedBaseAddress,
+      space_at(i)->init_oopmap(oopmaps->at(arr_idx)._offset,
                                oopmaps->at(arr_idx)._oopmap_size_in_bits);
     }
   }
@@ -1510,41 +1536,44 @@ MapArchiveResult FileMapInfo::map_region(int i, intx addr_delta, char* mapped_ba
   return MAP_ARCHIVE_SUCCESS;
 }
 
-char* FileMapInfo::map_relocation_bitmap(size_t& bitmap_size) {
+// The return value is the location of the archive relocation bitmap.
+char* FileMapInfo::map_bitmap_region() {
   FileMapRegion* si = space_at(MetaspaceShared::bm);
-  bitmap_size = si->used_aligned();
+  if (si->mapped_base() != NULL) {
+    return si->mapped_base();
+  }
   bool read_only = true, allow_exec = false;
   char* requested_addr = NULL; // allow OS to pick any location
   char* bitmap_base = os::map_memory(_fd, _full_path, si->file_offset(),
-                                     requested_addr, bitmap_size, read_only, allow_exec);
+                                     requested_addr, si->used_aligned(), read_only, allow_exec);
   if (bitmap_base == NULL) {
     log_error(cds)("failed to map relocation bitmap");
     return NULL;
   }
 
-  if (VerifySharedSpaces && !region_crc_check(bitmap_base, bitmap_size, si->crc())) {
+  if (VerifySharedSpaces && !region_crc_check(bitmap_base, si->used_aligned(), si->crc())) {
     log_error(cds)("relocation bitmap CRC error");
-    if (!os::unmap_memory(bitmap_base, bitmap_size)) {
+    if (!os::unmap_memory(bitmap_base, si->used_aligned())) {
       fatal("os::unmap_memory of relocation bitmap failed");
     }
     return NULL;
   }
 
+  si->set_mapped_base(bitmap_base);
+  si->set_mapped_from_file(true);
   return bitmap_base;
 }
 
 bool FileMapInfo::relocate_pointers(intx addr_delta) {
   log_debug(cds, reloc)("runtime archive relocation start");
-  size_t bitmap_size;
-  char* bitmap_base = map_relocation_bitmap(bitmap_size);
+  char* bitmap_base = map_bitmap_region();
 
   if (bitmap_base == NULL) {
     return false;
   } else {
     size_t ptrmap_size_in_bits = header()->ptrmap_size_in_bits();
-    log_debug(cds, reloc)("mapped relocation bitmap @ " INTPTR_FORMAT " (" SIZE_FORMAT
-                          " bytes = " SIZE_FORMAT " bits)",
-                          p2i(bitmap_base), bitmap_size, ptrmap_size_in_bits);
+    log_debug(cds, reloc)("mapped relocation bitmap @ " INTPTR_FORMAT " (" SIZE_FORMAT " bits)",
+                          p2i(bitmap_base), ptrmap_size_in_bits);
 
     BitMapView ptrmap((BitMap::bm_word_t*)bitmap_base, ptrmap_size_in_bits);
 
@@ -1567,9 +1596,8 @@ bool FileMapInfo::relocate_pointers(intx addr_delta) {
                                        valid_new_base, valid_new_end, addr_delta);
     ptrmap.iterate(&patcher);
 
-    if (!os::unmap_memory(bitmap_base, bitmap_size)) {
-      fatal("os::unmap_memory of relocation bitmap failed");
-    }
+    // The MetaspaceShared::bm region will be unmapped in MetaspaceShared::initialize_shared_spaces().
+
     log_debug(cds, reloc)("runtime archive relocation done");
     return true;
   }
@@ -1870,10 +1898,16 @@ void FileMapInfo::patch_archived_heap_embedded_pointers() {
 
 void FileMapInfo::patch_archived_heap_embedded_pointers(MemRegion* ranges, int num_ranges,
                                                         int first_region_idx) {
+  char* bitmap_base = map_bitmap_region();
+  if (bitmap_base == NULL) {
+    return;
+  }
   for (int i=0; i<num_ranges; i++) {
     FileMapRegion* si = space_at(i + first_region_idx);
-    HeapShared::patch_archived_heap_embedded_pointers(ranges[i], (address)(SharedBaseAddress + si->oopmap_offset()),
-                                                      si->oopmap_size_in_bits());
+    HeapShared::patch_archived_heap_embedded_pointers(
+      ranges[i],
+      (address)(space_at(MetaspaceShared::bm)->mapped_base()) + si->oopmap_offset(),
+      si->oopmap_size_in_bits());
   }
 }
 

src/hotspot/share/memory/filemap.hpp

@@ -100,7 +100,9 @@ class SharedClassPathEntry {
 
 struct ArchiveHeapOopmapInfo {
   address _oopmap;               // bitmap for relocating embedded oops
+  size_t  _offset;               // this oopmap is stored at this offset from the bottom of the BM region
   size_t  _oopmap_size_in_bits;
+  size_t  _oopmap_size_in_bytes;
 };
 
 class SharedPathTable {
@@ -448,7 +450,9 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   void  write_header();
   void  write_region(int region, char* base, size_t size,
                      bool read_only, bool allow_exec);
-  void  write_bitmap_region(const CHeapBitMap* ptrmap);
+  void  write_bitmap_region(const CHeapBitMap* ptrmap,
+                            GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
+                            GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps);
   size_t write_archive_heap_regions(GrowableArray<MemRegion> *heap_mem,
                                     GrowableArray<ArchiveHeapOopmapInfo> *oopmaps,
                                     int first_region_id, int max_num_regions);
@@ -534,6 +538,10 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   FileMapRegion* first_core_space() const;
   FileMapRegion* last_core_space() const;
 
+  FileMapRegion* space_at(int i) const {
+    return header()->space_at(i);
+  }
+
  private:
   void  seek_to_position(size_t pos);
   char* skip_first_path_entry(const char* path) NOT_CDS_RETURN_(NULL);
@@ -549,14 +557,12 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   bool  region_crc_check(char* buf, size_t size, int expected_crc) NOT_CDS_RETURN_(false);
   void  dealloc_archive_heap_regions(MemRegion* regions, int num) NOT_CDS_JAVA_HEAP_RETURN;
   void  map_heap_regions_impl() NOT_CDS_JAVA_HEAP_RETURN;
-  char* map_relocation_bitmap(size_t& bitmap_size);
+  char* map_bitmap_region();
   MapArchiveResult map_region(int i, intx addr_delta, char* mapped_base_address, ReservedSpace rs);
   bool  read_region(int i, char* base, size_t size);
   bool  relocate_pointers(intx addr_delta);
-
-  FileMapRegion* space_at(int i) const {
-    return header()->space_at(i);
-  }
+  static size_t set_oopmaps_offset(GrowableArray<ArchiveHeapOopmapInfo> *oopmaps, size_t curr_size);
+  static size_t write_oopmaps(GrowableArray<ArchiveHeapOopmapInfo> *oopmaps, size_t curr_offset, uintptr_t* buffer);
 
   // The starting address of spc, as calculated with CompressedOop::decode_non_null()
   address start_address_as_decoded_with_current_oop_encoding_mode(FileMapRegion* spc) {

src/hotspot/share/memory/metaspaceShared.cpp

@@ -1103,7 +1103,7 @@ class VM_PopulateDumpSharedSpace: public VM_Operation {
   void dump_symbols();
   char* dump_read_only_tables();
   void print_class_stats();
-  void print_region_stats();
+  void print_region_stats(FileMapInfo* map_info);
   void print_bitmap_region_stats(size_t size, size_t total_size);
   void print_heap_region_stats(GrowableArray<MemRegion> *heap_mem,
                                const char *name, size_t total_size);
@@ -1423,6 +1423,8 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() {
   MetaspaceShared::serialize(&wc);
 
   // Write the bitmaps for patching the archive heap regions
+  _closed_archive_heap_oopmaps = NULL;
+  _open_archive_heap_oopmaps = NULL;
   dump_archive_heap_oopmaps();
 
   return start;
@@ -1572,7 +1574,7 @@ void VM_PopulateDumpSharedSpace::doit() {
   mapinfo->set_i2i_entry_code_buffers(MetaspaceShared::i2i_entry_code_buffers(),
                                       MetaspaceShared::i2i_entry_code_buffers_size());
   mapinfo->open_for_write();
-  MetaspaceShared::write_core_archive_regions(mapinfo);
+  MetaspaceShared::write_core_archive_regions(mapinfo, _closed_archive_heap_oopmaps, _open_archive_heap_oopmaps);
   _total_closed_archive_region_size = mapinfo->write_archive_heap_regions(
                                         _closed_archive_heap_regions,
                                         _closed_archive_heap_oopmaps,
@@ -1587,10 +1589,9 @@ void VM_PopulateDumpSharedSpace::doit() {
   mapinfo->set_final_requested_base((char*)Arguments::default_SharedBaseAddress());
   mapinfo->set_header_crc(mapinfo->compute_header_crc());
   mapinfo->write_header();
+  print_region_stats(mapinfo);
   mapinfo->close();
 
-  print_region_stats();
-
   if (log_is_enabled(Info, cds)) {
     ArchiveCompactor::alloc_stats()->print_stats(int(_ro_region.used()), int(_rw_region.used()),
                                                  int(_mc_region.used()));
@@ -1611,10 +1612,10 @@ void VM_PopulateDumpSharedSpace::doit() {
   vm_direct_exit(0);
 }
 
-void VM_PopulateDumpSharedSpace::print_region_stats() {
+void VM_PopulateDumpSharedSpace::print_region_stats(FileMapInfo *map_info) {
   // Print statistics of all the regions
-  const size_t bitmap_used = ArchivePtrMarker::ptrmap()->size_in_bytes();
-  const size_t bitmap_reserved = align_up(bitmap_used, Metaspace::reserve_alignment());
+  const size_t bitmap_used = map_info->space_at(MetaspaceShared::bm)->used();
+  const size_t bitmap_reserved = map_info->space_at(MetaspaceShared::bm)->used_aligned();
   const size_t total_reserved = _ro_region.reserved()  + _rw_region.reserved() +
                                 _mc_region.reserved()  +
                                 bitmap_reserved +
@@ -1656,7 +1657,9 @@ void VM_PopulateDumpSharedSpace::print_heap_region_stats(GrowableArray<MemRegion
   }
 }
 
-void MetaspaceShared::write_core_archive_regions(FileMapInfo* mapinfo) {
+void MetaspaceShared::write_core_archive_regions(FileMapInfo* mapinfo,
+                                                 GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
+                                                 GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps) {
   // Make sure NUM_CDS_REGIONS (exported in cds.h) agrees with
   // MetaspaceShared::n_regions (internal to hotspot).
   assert(NUM_CDS_REGIONS == MetaspaceShared::n_regions, "sanity");
@@ -1666,7 +1669,7 @@ void MetaspaceShared::write_core_archive_regions(FileMapInfo* mapinfo) {
   write_region(mapinfo, mc, &_mc_region, /*read_only=*/false,/*allow_exec=*/true);
   write_region(mapinfo, rw, &_rw_region, /*read_only=*/false,/*allow_exec=*/false);
   write_region(mapinfo, ro, &_ro_region, /*read_only=*/true, /*allow_exec=*/false);
-  mapinfo->write_bitmap_region(ArchivePtrMarker::ptrmap());
+  mapinfo->write_bitmap_region(ArchivePtrMarker::ptrmap(), closed_oopmaps, open_oopmaps);
 }
 
 void MetaspaceShared::write_region(FileMapInfo* mapinfo, int region_idx, DumpRegion* dump_region, bool read_only,  bool allow_exec) {
@@ -1915,7 +1918,7 @@ void VM_PopulateDumpSharedSpace::dump_archive_heap_oopmaps(GrowableArray<MemRegi
     ResourceBitMap oopmap = HeapShared::calculate_oopmap(regions->at(i));
     size_t size_in_bits = oopmap.size();
     size_t size_in_bytes = oopmap.size_in_bytes();
-    uintptr_t* buffer = (uintptr_t*)_ro_region.allocate(size_in_bytes, sizeof(intptr_t));
+    uintptr_t* buffer = (uintptr_t*)NEW_C_HEAP_ARRAY(char, size_in_bytes, mtInternal);
     oopmap.write_to(buffer, size_in_bytes);
     log_info(cds, heap)("Oopmap = " INTPTR_FORMAT " (" SIZE_FORMAT_W(6) " bytes) for heap region "
                         INTPTR_FORMAT " (" SIZE_FORMAT_W(8) " bytes)",
@@ -1925,6 +1928,7 @@ void VM_PopulateDumpSharedSpace::dump_archive_heap_oopmaps(GrowableArray<MemRegi
     ArchiveHeapOopmapInfo info;
     info._oopmap = (address)buffer;
     info._oopmap_size_in_bits = size_in_bits;
+    info._oopmap_size_in_bytes = size_in_bytes;
     oopmaps->append(info);
   }
 }
@@ -2348,6 +2352,8 @@ void MetaspaceShared::initialize_shared_spaces() {
   // Close the mapinfo file
   static_mapinfo->close();
 
+  static_mapinfo->unmap_region(MetaspaceShared::bm);
+
   FileMapInfo *dynamic_mapinfo = FileMapInfo::dynamic_info();
   if (dynamic_mapinfo != NULL) {
     intptr_t* buffer = (intptr_t*)dynamic_mapinfo->serialized_data();

src/hotspot/share/memory/metaspaceShared.hpp

@@ -38,6 +38,7 @@
 
 class FileMapInfo;
 class CHeapBitMap;
+struct ArchiveHeapOopmapInfo;
 
 enum MapArchiveResult {
   MAP_ARCHIVE_SUCCESS,
@@ -358,7 +359,9 @@ class MetaspaceShared : AllStatic {
     return is_windows;
   }
 
-  static void write_core_archive_regions(FileMapInfo* mapinfo);
+  static void write_core_archive_regions(FileMapInfo* mapinfo,
+                                         GrowableArray<ArchiveHeapOopmapInfo>* closed_oopmaps,
+                                         GrowableArray<ArchiveHeapOopmapInfo>* open_oopmaps);
 private:
 #if INCLUDE_CDS
   static void write_region(FileMapInfo* mapinfo, int region_idx, DumpRegion* dump_region,