8305895: Implement JEP 450: Compact Object Headers (Experimental)

src/hotspot/share/cds/archiveHeapWriter.cpp

@@ -187,7 +187,7 @@ void ArchiveHeapWriter::ensure_buffer_space(size_t min_bytes) {
 }
 
 void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShared>* roots) {
-  Klass* k = Universe::objectArrayKlass(); // already relocated to point to archived klass
+  Klass* k = Universe::objectArrayKlass();
   int length = roots->length();
   _heap_roots_word_size = objArrayOopDesc::object_size(length);
   size_t byte_size = _heap_roots_word_size * HeapWordSize;
@@ -206,8 +206,7 @@ void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShar
   {
     // This is copied from MemAllocator::finish
     if (UseCompactObjectHeaders) {
-      narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(k);
-      oopDesc::release_set_mark(mem, markWord::prototype().set_narrow_klass(nk));
+      oopDesc::release_set_mark(mem, k->prototype_header());
     } else {
       oopDesc::set_mark(mem, markWord::prototype());
       oopDesc::release_set_klass(mem, k);

src/hotspot/share/cds/filemap.cpp

@@ -2493,7 +2493,7 @@ bool FileMapHeader::validate() {
   log_info(cds)("Archive was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d",
                           compressed_oops(), compressed_class_pointers(), compact_headers());
   if (compressed_oops() != UseCompressedOops || compressed_class_pointers() != UseCompressedClassPointers) {
-    log_info(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
+    log_warning(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
                                "different from runtime, CDS will be disabled.");
     return false;
   }
@@ -2504,10 +2504,10 @@ bool FileMapHeader::validate() {
   }
 
   if (compact_headers() != UseCompactObjectHeaders) {
-    log_info(cds)("The shared archive file's UseCompactObjectHeaders setting (%s)"
-                  " does not equal the current UseCompactObjectHeaders setting (%s).",
-                  _compact_headers          ? "enabled" : "disabled",
-                  UseCompactObjectHeaders   ? "enabled" : "disabled");
+    log_warning(cds)("Unable to use shared archive.\nThe shared archive file's UseCompactObjectHeaders setting (%s)"
+                     " does not equal the current UseCompactObjectHeaders setting (%s).",
+                     _compact_headers          ? "enabled" : "disabled",
+                     UseCompactObjectHeaders   ? "enabled" : "disabled");
     return false;
   }
 

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1435,7 +1435,7 @@ jint G1CollectedHeap::initialize() {
 
   G1InitLogger::print();
 
-  GCForwarding::initialize(heap_rs.region());
+  GCForwarding::initialize();
 
   return JNI_OK;
 }

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -130,7 +130,7 @@ jint ParallelScavengeHeap::initialize() {
 
   ParallelInitLogger::print();
 
-  GCForwarding::initialize(heap_rs.region());
+  GCForwarding::initialize();
 
   return JNI_OK;
 }

src/hotspot/share/gc/serial/serialHeap.cpp

@@ -201,7 +201,7 @@ jint SerialHeap::initialize() {
 
   GCInitLogger::print();
 
-  GCForwarding::initialize(_reserved);
+  GCForwarding::initialize();
 
   return JNI_OK;
 }

src/hotspot/share/gc/shared/gcForwarding.cpp

@@ -34,20 +34,22 @@ void GCForwarding::initialize_flags(size_t max_heap_size) {
 #ifdef _LP64
   size_t max_narrow_heap_size = right_n_bits(NumLowBitsNarrow - Shift);
   if (UseCompactObjectHeaders && max_heap_size > max_narrow_heap_size * HeapWordSize) {
-    warning("Compact object headers require a java heap size smaller than %zu (given: %zu). "
-            "Disabling compact object headers.", max_narrow_heap_size * HeapWordSize, max_heap_size);
+    warning("Compact object headers require a java heap size smaller than " SIZE_FORMAT
+            "%s (given: " SIZE_FORMAT "%s). Disabling compact object headers.",
+            byte_size_in_proper_unit(max_narrow_heap_size * HeapWordSize),
+            proper_unit_for_byte_size(max_narrow_heap_size * HeapWordSize),
+            byte_size_in_proper_unit(max_heap_size),
+            proper_unit_for_byte_size(max_heap_size));
     FLAG_SET_ERGO(UseCompactObjectHeaders, false);
   }
 #endif
 }
 
-void GCForwarding::initialize(MemRegion heap) {
+void GCForwarding::initialize() {
 #ifdef _LP64
-  _heap_base = heap.start();
-  if (heap.word_size() <= right_n_bits(NumLowBitsNarrow - Shift)) {
+  if (UseCompactObjectHeaders) {
     _num_low_bits = NumLowBitsNarrow;
   } else {
-    assert(!UseCompactObjectHeaders, "Compact object headers should be turned off for large heaps");
     _num_low_bits = NumLowBitsWide;
   }
 #endif

src/hotspot/share/gc/shared/gcForwarding.hpp

@@ -37,6 +37,9 @@
  * compressed-oops encoding: it basically subtracts the forwardee address from the
  * heap-base, shifts that difference into the right place, and sets the lowest two
  * bits (to indicate 'forwarded' state as usual).
+ * With compact-headers, we have 40 bits to encode forwarding pointers. This is
+ * enough to address 8TB of heap. If the heap size exceeds that limit, we turn off
+ * compact headers.
  */
 class GCForwarding : public AllStatic {
   static const int NumLowBitsNarrow = LP64_ONLY(markWord::klass_shift) NOT_LP64(0 /*unused*/);
@@ -47,7 +50,7 @@ class GCForwarding : public AllStatic {
   static int _num_low_bits;
 public:
   static void initialize_flags(size_t max_heap_size);
-  static void initialize(MemRegion heap);
+  static void initialize();
   static inline void forward_to(oop from, oop to);
   static inline oop forwardee(oop from);
   static inline bool is_forwarded(oop obj);

src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp

@@ -197,7 +197,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
                   file, line);
   }
 
-  Klass* obj_klass = obj->forward_safe_klass();
+  Klass* obj_klass = ShenandoahForwarding::klass(obj);
   if (obj_klass == nullptr) {
     print_failure(_safe_unknown, obj, interior_loc, nullptr, "Shenandoah assert_correct failed",
                   "Object klass pointer should not be null",
@@ -235,7 +235,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
                     file, line);
     }
 
-    if (obj_klass != fwd->forward_safe_klass()) {
+    if (obj_klass != ShenandoahForwarding::klass(fwd)) {
       print_failure(_safe_oop, obj, interior_loc, nullptr, "Shenandoah assert_correct failed",
                     "Forwardee klass disagrees with object class",
                     file, line);

src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp

@@ -62,6 +62,8 @@ class ShenandoahForwarding {
    */
   static inline oop try_update_forwardee(oop obj, oop update);
 
+  static inline size_t size(oop obj);
+  static inline Klass* klass(oop obj);
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP

src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp

@@ -90,4 +90,21 @@ inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
   }
 }
 
+inline Klass* ShenandoahForwarding::klass(oop obj) {
+  if (UseCompactObjectHeaders) {
+    markWord mark = obj->mark();
+    if (mark.is_marked()) {
+      oop fwd = cast_to_oop(mark.clear_lock_bits().to_pointer());
+      mark = fwd->mark();
+    }
+    return mark.klass();
+  } else {
+    return obj->klass();
+  }
+}
+
+inline size_t ShenandoahForwarding::size(oop obj) {
+  return obj->size_given_klass(klass(obj));
+}
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -423,7 +423,7 @@ jint ShenandoahHeap::initialize() {
 
   ShenandoahInitLogger::print();
 
-  GCForwarding::initialize(_heap_region);
+  GCForwarding::initialize();
 
   return JNI_OK;
 }
@@ -1133,7 +1133,7 @@ oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
 
   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
 
-  size_t size = p->forward_safe_size();
+  size_t size = ShenandoahForwarding::size(p);
 
   assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -434,7 +434,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
     oop obj = cast_to_oop(cs);
     assert(oopDesc::is_oop(obj), "sanity");
     assert(ctx->is_marked(obj), "object expected to be marked");
-    size_t size = obj->forward_safe_size();
+    size_t size = ShenandoahForwarding::size(obj);
     cl->do_object(obj);
     cs += size;
   }

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

@@ -102,7 +102,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
     T o = RawAccess<>::oop_load(p);
     if (!CompressedOops::is_null(o)) {
       oop obj = CompressedOops::decode_not_null(o);
-      if (is_instance_ref_klass(obj->forward_safe_klass())) {
+      if (is_instance_ref_klass(ShenandoahForwarding::klass(obj))) {
         obj = ShenandoahForwarding::get_forwardee(obj);
       }
       // Single threaded verification can use faster non-atomic stack and bitmap
@@ -129,7 +129,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
               "oop must be aligned");
 
     ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
-    Klass* obj_klass = obj->forward_safe_klass();
+    Klass* obj_klass = ShenandoahForwarding::klass(obj);
 
     // Verify that obj is not in dead space:
     {
@@ -144,11 +144,11 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
              "Object start should be within the region");
 
       if (!obj_reg->is_humongous()) {
-        check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->forward_safe_size()) <= obj_reg->top(),
+        check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + ShenandoahForwarding::size(obj)) <= obj_reg->top(),
                "Object end should be within the region");
       } else {
         size_t humongous_start = obj_reg->index();
-        size_t humongous_end = humongous_start + (obj->forward_safe_size() >> ShenandoahHeapRegion::region_size_words_shift());
+        size_t humongous_end = humongous_start + (ShenandoahForwarding::size(obj) >> ShenandoahHeapRegion::region_size_words_shift());
         for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
           check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
                  "Humongous object is in continuation that fits it");
@@ -165,7 +165,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
           // skip
           break;
         case ShenandoahVerifier::_verify_liveness_complete:
-          Atomic::add(&_ld[obj_reg->index()], (uint) obj->forward_safe_size(), memory_order_relaxed);
+          Atomic::add(&_ld[obj_reg->index()], (uint) ShenandoahForwarding::size(obj), memory_order_relaxed);
           // fallthrough for fast failure for un-live regions:
         case ShenandoahVerifier::_verify_liveness_conservative:
           check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
@@ -209,7 +209,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
       HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
       check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
              "Forwardee start should be within the region");
-      check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->forward_safe_size()) <= fwd_reg->top(),
+      check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + ShenandoahForwarding::size(fwd)) <= fwd_reg->top(),
              "Forwardee end should be within the region");
 
       oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
@@ -327,7 +327,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
    */
   void verify_oops_from(oop obj) {
     _loc = obj;
-    Klass* klass = obj->forward_safe_klass();
+    Klass* klass = ShenandoahForwarding::klass(obj);
     obj->oop_iterate_backwards(this, klass);
     _loc = nullptr;
   }
@@ -592,7 +592,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask {
 
       while (addr < limit) {
         verify_and_follow(addr, stack, cl, &processed);
-        addr += cast_to_oop(addr)->forward_safe_size();
+        addr += ShenandoahForwarding::size(cast_to_oop(addr));
       }
     }
 
@@ -608,7 +608,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask {
 
     // Verify everything reachable from that object too, hopefully realizing
     // everything was already marked, and never touching further:
-    if (!is_instance_ref_klass(obj->forward_safe_klass())) {
+    if (!is_instance_ref_klass(ShenandoahForwarding::klass(obj))) {
       cl.verify_oops_from(obj);
       (*processed)++;
     }

src/hotspot/share/oops/oop.hpp

@@ -119,20 +119,6 @@ class oopDesc {
   // to be able to figure out the size of an object knowing its klass.
   inline size_t size_given_klass(Klass* klass);
 
-  // The following set of methods is used to access the mark-word and related
-  // properties when the object may be forwarded. Be careful where and when
-  // using this method. It assumes that the forwardee is installed in
-  // the header as a plain pointer (or self-forwarded). In particular,
-  // those methods can not deal with the encoded forwarding that is used
-  // in Serial, Parallel, G1 and Shenandoah full-GCs.
-private:
-  inline Klass*   forward_safe_klass_impl(markWord m) const;
-public:
-  inline Klass*   forward_safe_klass() const;
-  inline Klass*   forward_safe_klass(markWord m) const;
-  inline size_t   forward_safe_size();
-  inline void     forward_safe_init_mark();
-
   // type test operations (inlined in oop.inline.hpp)
   inline bool is_instance()    const;
   inline bool is_instanceRef() const;

src/hotspot/share/oops/oop.inline.hpp

@@ -91,11 +91,7 @@ markWord oopDesc::prototype_mark() const {
 }
 
 void oopDesc::init_mark() {
-  if (UseCompactObjectHeaders) {
-    set_mark(prototype_mark());
-  } else {
-    set_mark(markWord::prototype());
-  }
+  set_mark(prototype_mark());
 }
 
 Klass* oopDesc::klass() const {
@@ -225,53 +221,6 @@ size_t oopDesc::size_given_klass(Klass* klass)  {
   return s;
 }
 
-#ifdef _LP64
-Klass* oopDesc::forward_safe_klass_impl(markWord m) const {
-  assert(UseCompactObjectHeaders, "Only get here with compact headers");
-  if (m.is_marked()) {
-    oop fwd = forwardee(m);
-    markWord m2 = fwd->mark();
-    assert(!m2.is_marked() || m2.is_self_forwarded(), "no double forwarding: this: " PTR_FORMAT " (" INTPTR_FORMAT "), fwd: " PTR_FORMAT " (" INTPTR_FORMAT ")", p2i(this), m.value(), p2i(fwd), m2.value());
-    m = m2;
-  }
-  return m.klass();
-}
-#endif
-
-Klass* oopDesc::forward_safe_klass(markWord m) const {
-#ifdef _LP64
-  if (UseCompactObjectHeaders) {
-    return forward_safe_klass_impl(m);
-  } else
-#endif
-  {
-    return klass();
-  }
-}
-
-Klass* oopDesc::forward_safe_klass() const {
-#ifdef _LP64
-  if (UseCompactObjectHeaders) {
-    return forward_safe_klass_impl(mark());
-  } else
-#endif
-  {
-    return klass();
-  }
-}
-
-size_t oopDesc::forward_safe_size() {
-  return size_given_klass(forward_safe_klass());
-}
-
-void oopDesc::forward_safe_init_mark() {
-  if (UseCompactObjectHeaders) {
-    set_mark(forward_safe_klass()->prototype_header());
-  } else {
-    set_mark(markWord::prototype());
-  }
-}
-
 bool oopDesc::is_instance()    const { return klass()->is_instance_klass();             }
 bool oopDesc::is_instanceRef() const { return klass()->is_reference_instance_klass();   }
 bool oopDesc::is_stackChunk()  const { return klass()->is_stack_chunk_instance_klass(); }

src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/Oop.java

@@ -81,15 +81,10 @@ private static synchronized void initialize(TypeDataBase db) throws WrongTypeExc
   // Accessors for declared fields
   public Mark  getMark()   { return new Mark(getHandle()); }
 
-  private static Klass getKlass(Mark mark) {
-    assert(VM.getVM().isCompactObjectHeadersEnabled());
-    return mark.getKlass();
-  }
-
   public Klass getKlass() {
     if (VM.getVM().isCompactObjectHeadersEnabled()) {
       assert(VM.getVM().isCompressedKlassPointersEnabled());
-      return getKlass(getMark());
+      return getMark().getKlass();
     } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
       return (Klass)compressedKlass.getValue(getHandle());
     } else {
@@ -224,7 +219,7 @@ public static Klass getKlassForOopHandle(OopHandle handle) {
     }
     if (VM.getVM().isCompactObjectHeadersEnabled()) {
       Mark mark = new Mark(handle);
-      return getKlass(mark);
+      return mark.getKlass();
     } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
       return (Klass)Metadata.instantiateWrapperFor(handle.getCompKlassAddressAt(compressedKlass.getOffset()));
     } else {

test/hotspot/gtest/gc/shared/test_preservedMarks.cpp

@@ -37,7 +37,7 @@ TEST_VM(PreservedMarks, iterate_and_restore) {
 
   HeapWord fakeheap[32] = { nullptr };
   HeapWord* heap = align_up(fakeheap, 8 * sizeof(HeapWord));
-  GCForwarding::initialize(MemRegion(&heap[0], &heap[16]));
+  GCForwarding::initialize();
 
   oop o1 = cast_to_oop(&heap[0]); o1->set_mark(originalMark());
   oop o2 = cast_to_oop(&heap[2]); o2->set_mark(originalMark());