8305895: Implement JEP 450: Compact Object Headers (Experimental)

make/autoconf/jdk-options.m4

@@ -692,7 +692,8 @@ AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH],
              test "x$OPENJDK_TARGET_CPU" != "xaarch64" &&
              test "x$OPENJDK_TARGET_CPU" != "xppc64" &&
              test "x$OPENJDK_TARGET_CPU" != "xppc64le" &&
-             test "x$OPENJDK_TARGET_CPU" != "xriscv64"; then
+             test "x$OPENJDK_TARGET_CPU" != "xriscv64" &&
+             test "x$OPENJDK_TARGET_CPU" != "xs390x"; then
           AC_MSG_RESULT([no (compact object headers not supported for this platform)])
           AVAILABLE=false
         else

src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp

@@ -2029,8 +2029,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
 
   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
-  Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
-  Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
 
   // Length and pos's are all sign extended at this point on 64bit.
 
@@ -2094,13 +2092,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     // We don't know the array types are compatible.
     if (basic_type != T_OBJECT) {
       // Simple test for basic type arrays.
-      if (UseCompressedClassPointers) {
-        __ z_l(tmp, src_klass_addr);
-        __ z_c(tmp, dst_klass_addr);
-      } else {
-        __ z_lg(tmp, src_klass_addr);
-        __ z_cg(tmp, dst_klass_addr);
-      }
+      __ cmp_klasses_from_objects(src, dst, tmp, Z_R1_scratch);
       __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
     } else {
       // For object arrays, if src is a sub class of dst then we can
@@ -2234,15 +2226,13 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     }
 
     if (basic_type != T_OBJECT) {
-      if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
-      else                                    { __ z_cg(tmp, dst_klass_addr); }
+      __ cmp_klass(tmp, dst, Z_R1_scratch);
       __ branch_optimized(Assembler::bcondNotEqual, halt);
-      if (UseCompressedClassPointers)         { __ z_c (tmp, src_klass_addr); }
-      else                                    { __ z_cg(tmp, src_klass_addr); }
+
+      __ cmp_klass(tmp, src, Z_R1_scratch);
       __ branch_optimized(Assembler::bcondEqual, known_ok);
     } else {
-      if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
-      else                                    { __ z_cg(tmp, dst_klass_addr); }
+      __ cmp_klass(tmp, dst, Z_R1_scratch);
       __ branch_optimized(Assembler::bcondEqual, known_ok);
       __ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok);
     }
@@ -2737,12 +2727,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
     add_debug_info_for_null_check_here(info);
   }
 
-  if (UseCompressedClassPointers) {
-    __ z_llgf(result, Address(obj, oopDesc::klass_offset_in_bytes()));
-    __ decode_klass_not_null(result);
-  } else {
-    __ z_lg(result, Address(obj, oopDesc::klass_offset_in_bytes()));
-  }
+  __ load_klass(result, obj);
 }
 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
   ciMethod* method = op->profiled_method();

src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp

@@ -173,17 +173,21 @@ void C1_MacroAssembler::try_allocate(
 
 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register Rzero, Register t1) {
   assert_different_registers(obj, klass, len, t1, Rzero);
-  // This assumes that all prototype bits fit in an int32_t.
-  load_const_optimized(t1, (intx)markWord::prototype().value());
-  z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
+  if (UseCompactObjectHeaders) {
+    z_lg(t1, Address(klass, in_bytes(Klass::prototype_header_offset())));
+    z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
+  } else {
+    load_const_optimized(t1, (intx)markWord::prototype().value());
+    z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
+    store_klass(klass, obj, t1);
+  }
 
   if (len->is_valid()) {
     // Length will be in the klass gap, if one exists.
     z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
-  } else if (UseCompressedClassPointers) {
+  } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
     store_klass_gap(Rzero, obj);  // Zero klass gap for compressed oops.
   }
-  store_klass(klass, obj, t1);
 }
 
 void C1_MacroAssembler::initialize_body(Register objectFields, Register len_in_bytes, Register Rzero) {

src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp

@@ -42,6 +42,13 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi
   compiler_fast_unlock_lightweight_object(obj, box, temp1, temp2);
 }
 
+void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) {
+  // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+  // obj-start, so that we can load from the object's mark-word instead.
+  z_lg(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes()));
+  z_srlg(dst, dst, markWord::klass_shift); // TODO: could be z_sra
+}
+
 //------------------------------------------------------
 //   Special String Intrinsics. Implementation
 //------------------------------------------------------

src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp

@@ -33,6 +33,8 @@
   void fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2);
   void fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2);
 
+  void load_narrow_klass_compact_c2(Register dst, Address src);
+
   //-------------------------------------------
   // Special String Intrinsics Implementation.
   //-------------------------------------------

src/hotspot/cpu/s390/macroAssembler_s390.cpp

@@ -2160,7 +2160,16 @@ void MacroAssembler::call_VM_leaf_base(address entry_point) {
 }
 
 int MacroAssembler::ic_check_size() {
-  return 30 + (ImplicitNullChecks ? 0 : 6);
+  int ic_size = 24;
+  if (!ImplicitNullChecks) {
+    ic_size += 6;
+  }
+  if (UseCompactObjectHeaders) {
+    ic_size += 12;
+  } else {
+    ic_size += 6; // either z_llgf or z_lg
+  }
+  return ic_size;
 }
 
 int MacroAssembler::ic_check(int end_alignment) {
@@ -2181,7 +2190,9 @@ int MacroAssembler::ic_check(int end_alignment) {
     z_cgij(R2_receiver, 0, Assembler::bcondEqual, failure);
   }
 
-  if (UseCompressedClassPointers) {
+  if (UseCompactObjectHeaders) {
+    load_narrow_klass_compact(R1_scratch, R2_receiver);
+  } else if (UseCompressedClassPointers) {
     z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
   } else {
     z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
@@ -4064,17 +4075,66 @@ void MacroAssembler::load_klass(Register klass, Address mem) {
   }
 }
 
+// Loads the obj's Klass* into dst.
+// Input:
+// src - the oop we want to load the klass from.
+// dst - output nklass.
+void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
+  BLOCK_COMMENT("load_narrow_klass_compact {");
+  assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
+  z_lg(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+  z_srlg(dst, dst, markWord::klass_shift);
+  BLOCK_COMMENT("} load_narrow_klass_compact");
+}
+
+void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
+  BLOCK_COMMENT("cmp_klass {");
+  assert_different_registers(obj, klass, tmp);
+  if (UseCompactObjectHeaders) {
+    assert(tmp != noreg, "required");
+    assert_different_registers(klass, obj, tmp);
+    load_narrow_klass_compact(tmp, obj);
+    z_cr(klass, tmp);
+  } else if (UseCompressedClassPointers) {
+    z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+  } else {
+    z_cg(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+  }
+  BLOCK_COMMENT("} cmp_klass");
+}
+
+void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
+  BLOCK_COMMENT("cmp_klasses_from_objects {");
+  if (UseCompactObjectHeaders) {
+    assert(tmp1 != noreg && tmp2 != noreg, "required");
+    assert_different_registers(obj1, obj2, tmp1, tmp2);
+    load_narrow_klass_compact(tmp1, obj1);
+    load_narrow_klass_compact(tmp2, obj2);
+    z_cr(tmp1, tmp2);
+  } else if (UseCompressedClassPointers) {
+    z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
+    z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
+  } else {
+    z_lg(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
+    z_cg(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
+  }
+  BLOCK_COMMENT("} cmp_klasses_from_objects");
+}
+
 void MacroAssembler::load_klass(Register klass, Register src_oop) {
-  if (UseCompressedClassPointers) {
+  if (UseCompactObjectHeaders) {
+    load_narrow_klass_compact(klass, src_oop);
+    decode_klass_not_null(klass);
+  } else if (UseCompressedClassPointers) {
     z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop);
-    // Attention: no null check here!
     decode_klass_not_null(klass);
   } else {
     z_lg(klass, oopDesc::klass_offset_in_bytes(), src_oop);
   }
 }
 
 void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
+  assert(!UseCompactObjectHeaders, "Don't use with compact headers");
   if (UseCompressedClassPointers) {
     assert_different_registers(dst_oop, klass, Z_R0);
     if (ck == noreg) ck = klass;
@@ -4086,6 +4146,7 @@ void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck)
 }
 
 void MacroAssembler::store_klass_gap(Register s, Register d) {
+  assert(!UseCompactObjectHeaders, "Don't use with compact headers");
   if (UseCompressedClassPointers) {
     assert(s != d, "not enough registers");
     // Support s = noreg.

src/hotspot/cpu/s390/macroAssembler_s390.hpp

@@ -803,6 +803,13 @@ class MacroAssembler: public Assembler {
   void load_klass(Register klass, Register src_oop);
   void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
   void store_klass_gap(Register s, Register dst_oop);
+  void load_narrow_klass_compact(Register dst, Register src);
+  // Compares the Klass pointer of an object to a given Klass (which might be narrow,
+  // depending on UseCompressedClassPointers).
+  void cmp_klass(Register klass, Register obj, Register tmp);
+  // Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
+  // Uses tmp1 and tmp2 as temporary registers.
+  void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2);
 
   // This function calculates the size of the code generated by
   //   decode_klass_not_null(register dst)

src/hotspot/cpu/s390/s390.ad

@@ -4410,6 +4410,7 @@ instruct loadN(iRegN dst, memory mem) %{
 
 // Load narrow Klass Pointer
 instruct loadNKlass(iRegN dst, memory mem) %{
+  predicate(!UseCompactObjectHeaders);
   match(Set dst (LoadNKlass mem));
   ins_cost(MEMORY_REF_COST);
   size(Z_DISP3_SIZE);
@@ -4419,6 +4420,21 @@ instruct loadNKlass(iRegN dst, memory mem) %{
   ins_pipe(pipe_class_dummy);
 %}
 
+instruct loadNKlassCompactHeaders(iRegN dst, memory mem, flagsReg cr) %{
+  match(Set dst (LoadNKlass mem));
+  predicate(UseCompactObjectHeaders);
+  effect(KILL cr);
+  ins_cost(MEMORY_REF_COST);
+  format %{ "load_narrow_klass_compact $dst,$mem \t# compressed class ptr" %}
+  // TODO: size()
+  ins_encode %{
+    __ block_comment("load_narrow_klass_compact_c2 {");
+    __ load_narrow_klass_compact_c2($dst$$Register, $mem$$Address);
+    __ block_comment("} load_narrow_klass_compact");
+  %}
+  ins_pipe(pipe_class_dummy);
+%}
+
 // Load constant Compressed Pointer
 
 instruct loadConN(iRegN dst, immN src) %{

src/hotspot/cpu/s390/templateTable_s390.cpp

@@ -3952,7 +3952,12 @@ void TemplateTable::_new() {
     if (!ZeroTLAB) {
       // The object is initialized before the header. If the object size is
       // zero, go directly to the header initialization.
-      __ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC.
+      if (UseCompactObjectHeaders) {
+        assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
+        __ z_aghi(Rsize, (int)-oopDesc::base_offset_in_bytes());
+      } else {
+        __ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC.
+      }
       __ z_bre(initialize_header);             // Jump if size of fields is zero.
 
       // Initialize object fields.
@@ -3964,17 +3969,25 @@ void TemplateTable::_new() {
 
       // Set Rzero to 0 and use it as src length, then mvcle will copy nothing
       // and fill the object with the padding value 0.
-      __ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject);
+      if (UseCompactObjectHeaders) {
+        __ add2reg(RobjectFields, oopDesc::base_offset_in_bytes(), RallocatedObject);
+      } else {
+        __ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject);
+      }
       __ move_long_ext(RobjectFields, as_Register(Rzero->encoding() - 1), 0);
     }
 
     // Initialize object header only.
     __ bind(initialize_header);
-    __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
-                   (long)markWord::prototype().value());
-
-    __ store_klass_gap(Rzero, RallocatedObject);  // Zero klass gap for compressed oops.
-    __ store_klass(iklass, RallocatedObject);     // Store klass last.
+    if (UseCompactObjectHeaders) {
+      __ z_lg(tmp, Address(iklass, in_bytes(Klass::prototype_header_offset())));
+      __ z_stg(tmp, Address(RallocatedObject, oopDesc::mark_offset_in_bytes()));
+    } else {
+      __ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
+                     (long) markWord::prototype().value());
+      __ store_klass_gap(Rzero, RallocatedObject);  // Zero klass gap for compressed oops.
+      __ store_klass(iklass, RallocatedObject);     // Store klass last.
+    }
 
     if (DTraceAllocProbes) {
       // Trigger dtrace event for fastpath.