8343468: GenShen: Enable relocation of remembered set card tables

Reviewed-by: shade, kdnilsen, wkemper

Author: Cesar Soares Lucas

src/hotspot/cpu/aarch64/aarch64.ad

@@ -4541,6 +4541,7 @@ operand immByteMapBase()
 %{
   // Get base of card map
   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
+            !BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&
             (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
   match(ConP);
 

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -377,7 +377,8 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
 
   assert(CardTable::dirty_card_val() == 0, "must be");
 
-  __ load_byte_map_base(rscratch1);
+  Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
+  __ ldr(rscratch1, curr_ct_holder_addr);
 
   if (UseCondCardMark) {
     Label L_already_dirty;
@@ -624,7 +625,8 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb
   // number of bytes to copy
   __ sub(count, end, start);
 
-  __ load_byte_map_base(scratch);
+  Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
+  __ ldr(scratch, curr_ct_holder_addr);
   __ add(start, start, scratch);
   __ bind(L_loop);
   __ strb(zr, Address(start, count));

src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp

@@ -587,9 +587,6 @@ void ShenandoahBarrierSetAssembler::load_at(
 
 void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
   assert(ShenandoahCardBarrier, "Should have been checked by caller");
-
-  ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
-  CardTable* ct = ctbs->card_table();
   assert_different_registers(base, tmp, R0);
 
   if (ind_or_offs.is_constant()) {
@@ -598,7 +595,7 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register b
     __ add(base, ind_or_offs.as_register(), base);
   }
 
-  __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
+  __ ld(tmp, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread); /* tmp = *[R16_thread + card_table_offset] */
   __ srdi(base, base, CardTable::card_shift());
   __ li(R0, CardTable::dirty_card_val());
   __ stbx(R0, tmp, base);
@@ -797,7 +794,8 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb
   __ srdi(addr, addr, CardTable::card_shift());
   __ srdi(count, count, CardTable::card_shift());
   __ subf(count, addr, count);
-  __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
+  __ ld(R0, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
+  __ add(addr, addr, R0);
   __ addi(count, count, 1);
   __ li(R0, 0);
   __ mtctr(count);

src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp

@@ -396,7 +396,8 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
 
   assert(CardTable::dirty_card_val() == 0, "must be");
 
-  __ load_byte_map_base(t1);
+  Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
+  __ ld(t1, curr_ct_holder_addr);
   __ add(t1, obj, t1);
 
   if (UseCondCardMark) {
@@ -568,7 +569,8 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb
   // number of bytes to copy
   __ sub(count, end, start);
 
-  __ load_byte_map_base(tmp);
+  Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
+  __ ld(tmp, curr_ct_holder_addr);
   __ add(start, start, tmp);
 
   __ bind(L_loop);

src/hotspot/cpu/riscv/riscv.ad

@@ -2916,6 +2916,7 @@ operand immByteMapBase()
 %{
   // Get base of card map
   predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
+            !BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&
             (CardTable::CardValue*)n->get_ptr() ==
              ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
   match(ConP);

src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp

@@ -47,6 +47,7 @@ void CardTableBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembl
   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
   CardTable* ct = ctbs->card_table();
   intptr_t disp = (intptr_t) ct->byte_map_base();
+  SHENANDOAHGC_ONLY(assert(!UseShenandoahGC, "Shenandoah byte_map_base is not constant.");)
 
   Label L_loop, L_done;
   const Register end = count;

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@@ -614,32 +614,32 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
 
   // Does a store check for the oop in register obj. The content of
   // register obj is destroyed afterwards.
-
-  ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
-  CardTable* ct = ctbs->card_table();
-
   __ shrptr(obj, CardTable::card_shift());
 
-  Address card_addr;
+  // We'll use this register as the TLS base address and also later on
+  // to hold the byte_map_base.
+  Register thread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
+  Register tmp = LP64_ONLY(rscratch1) NOT_LP64(rdx);
 
-  // The calculation for byte_map_base is as follows:
-  // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
-  // So this essentially converts an address to a displacement and it will
-  // never need to be relocated. On 64-bit however the value may be too
-  // large for a 32-bit displacement.
-  intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
-  if (__ is_simm32(byte_map_base)) {
-    card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
-  } else {
-    // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
-    // displacement and done in a single instruction given favorable mapping and a
-    // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
-    // entry and that entry is not properly handled by the relocation code.
-    AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
-    Address index(noreg, obj, Address::times_1);
-    card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+#ifndef _LP64
+  // The next two ifs are just to get temporary registers to use for TLS and card table base.
+  if (thread == obj) {
+    thread = rdx;
+    tmp = rsi;
+  }
+  if (tmp == obj) {
+    tmp = rsi;
   }
 
+  __ push(thread);
+  __ push(tmp);
+  __ get_thread(thread);
+#endif
+
+  Address curr_ct_holder_addr(thread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
+  __ movptr(tmp, curr_ct_holder_addr);
+  Address card_addr(tmp, obj, Address::times_1);
+
   int dirty = CardTable::dirty_card_val();
   if (UseCondCardMark) {
     Label L_already_dirty;
@@ -650,6 +650,11 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o
   } else {
     __ movb(card_addr, dirty);
   }
+
+#ifndef _LP64
+  __ pop(tmp);
+  __ pop(thread);
+#endif
 }
 
 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -905,10 +910,6 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb
                                                                      Register tmp) {
   assert(ShenandoahCardBarrier, "Should have been checked by caller");
 
-  ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
-  CardTable* ct = bs->card_table();
-  intptr_t disp = (intptr_t) ct->byte_map_base();
-
   Label L_loop, L_done;
   const Register end = count;
   assert_different_registers(addr, end);
@@ -918,27 +919,38 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb
   __ jccb(Assembler::zero, L_done);
 
 #ifdef _LP64
+  const Register thread = r15_thread;
+  Address curr_ct_holder_addr(thread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
+  __ movptr(tmp, curr_ct_holder_addr);
+
   __ leaq(end, Address(addr, count, TIMES_OOP, 0));  // end == addr+count*oop_size
   __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
   __ shrptr(addr, CardTable::card_shift());
   __ shrptr(end, CardTable::card_shift());
   __ subptr(end, addr); // end --> cards count
 
-  __ mov64(tmp, disp);
   __ addptr(addr, tmp);
 
   __ BIND(L_loop);
   __ movb(Address(addr, count, Address::times_1), 0);
   __ decrement(count);
   __ jccb(Assembler::greaterEqual, L_loop);
 #else
+  const Register thread = tmp;
+  __ get_thread(thread);
+
+  Address curr_ct_holder_addr(thread, in_bytes(ShenandoahThreadLocalData::byte_map_base_offset()));
+  __ movptr(tmp, curr_ct_holder_addr);
+
   __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
   __ shrptr(addr, CardTable::card_shift());
   __ shrptr(end,  CardTable::card_shift());
   __ subptr(end, addr); // end --> count
 
+  __ addptr(addr, tmp);
+
   __ BIND(L_loop);
-  Address cardtable(addr, count, Address::times_1, disp);
+  Address cardtable(addr, count, Address::times_1, 0);
   __ movb(cardtable, 0);
   __ decrement(count);
   __ jccb(Assembler::greaterEqual, L_loop);

src/hotspot/share/ci/ciUtilities.cpp

@@ -26,6 +26,7 @@
 #include "gc/shared/cardTableBarrierSet.hpp"
 #include "gc/shared/cardTable.hpp"
 #include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/gc_globals.hpp"
 
 // ciUtilities
 //
@@ -45,5 +46,6 @@ CardTable::CardValue* ci_card_table_address() {
   BarrierSet* bs = BarrierSet::barrier_set();
   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
   CardTable* ct = ctbs->card_table();
+  assert(!UseShenandoahGC, "Shenandoah byte_map_base is not constant.");
   return ct->byte_map_base();
 }

src/hotspot/share/gc/shared/c1/cardTableBarrierSetC1.cpp

@@ -46,6 +46,8 @@ void CardTableBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_Op
   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
   CardTable* ct = ctbs->card_table();
   LIR_Const* card_table_base = new LIR_Const(ct->byte_map_base());
+  SHENANDOAHGC_ONLY(assert(!UseShenandoahGC, "Shenandoah byte_map_base is not constant.");)
+
   if (addr->is_address()) {
     LIR_Address* address = addr->as_address_ptr();
     // ptr cannot be an object because we use this barrier for array card marks

src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp

@@ -309,10 +309,12 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_O
     return;
   }
 
-  BarrierSet* bs = BarrierSet::barrier_set();
-  ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(bs);
-  CardTable* ct = ctbs->card_table();
-  LIR_Const* card_table_base = new LIR_Const(ct->byte_map_base());
+  LIR_Opr thrd = gen->getThreadPointer();
+  const int curr_ct_holder_offset = in_bytes(ShenandoahThreadLocalData::card_table_offset());
+  LIR_Address* curr_ct_holder_addr = new LIR_Address(thrd, curr_ct_holder_offset, T_ADDRESS);
+  LIR_Opr curr_ct_holder_ptr_reg = gen->new_register(T_ADDRESS);
+  __ move(curr_ct_holder_addr, curr_ct_holder_ptr_reg);
+
   if (addr->is_address()) {
     LIR_Address* address = addr->as_address_ptr();
     // ptr cannot be an object because we use this barrier for array card marks
@@ -336,13 +338,7 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_Opr addr, LIR_O
     __ unsigned_shift_right(addr, CardTable::card_shift(), tmp);
   }
 
-  LIR_Address* card_addr;
-  if (gen->can_inline_as_constant(card_table_base)) {
-    card_addr = new LIR_Address(tmp, card_table_base->as_jint(), T_BYTE);
-  } else {
-    card_addr = new LIR_Address(tmp, gen->load_constant(card_table_base), T_BYTE);
-  }
-
+  LIR_Address* card_addr = new LIR_Address(curr_ct_holder_ptr_reg, tmp, T_BYTE);
   LIR_Opr dirty = LIR_OprFact::intConst(CardTable::dirty_card_val());
   if (UseCondCardMark) {
     LIR_Opr cur_value = gen->new_register(T_INT);