8255762: Shenandoah: Consolidate/streamline interpreter LRBs

Reviewed-by: shade

Author: rkennke

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -43,8 +43,6 @@
 
 #define __ masm->
 
-address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
-
 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
                                                        Register src, Register dst, Register count, RegSet saved_regs) {
   if (is_oop) {
@@ -227,18 +225,18 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssemb
   }
 }
 
-void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address load_addr) {
+void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, bool weak) {
   assert(ShenandoahLoadRefBarrier, "Should be enabled");
   assert(dst != rscratch2, "need rscratch2");
   assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
 
-  Label done;
+  Label heap_stable, not_cset;
   __ enter();
   Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
   __ ldrb(rscratch2, gc_state);
 
   // Check for heap stability
-  __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
+  __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
 
   // use r1 for load address
   Register result_dst = dst;
@@ -253,51 +251,36 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembl
   __ lea(r1, load_addr);
   __ mov(r0, dst);
 
-  __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
-
-  __ mov(result_dst, r0);
-  __ pop(to_save, sp);
-
-  __ bind(done);
-  __ leave();
-}
-
-void ShenandoahBarrierSetAssembler::load_reference_barrier_weak(MacroAssembler* masm, Register dst, Address load_addr) {
-  if (!ShenandoahLoadRefBarrier) {
-    return;
+  // Test for in-cset
+  if (!weak) {
+    __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+    __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+    __ ldrb(rscratch2, Address(rscratch2, rscratch1));
+    __ tbz(rscratch2, 0, not_cset);
   }
 
-  assert(dst != rscratch2, "need rscratch2");
-
-  Label is_null;
-  Label done;
-
-  __ block_comment("load_reference_barrier_weak { ");
-
-  __ cbz(dst, is_null);
-
-  __ enter();
-
-  Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
-  __ ldrb(rscratch2, gc_state);
-
-  // Check for heap in evacuation phase
-  __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
-
-  __ mov(rscratch2, dst);
   __ push_call_clobbered_registers();
-  __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
-  __ lea(r1, load_addr);
-  __ mov(r0, rscratch2);
+  if (weak) {
+    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
+  } else {
+    if (UseCompressedOops) {
+      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
+    } else {
+      __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+    }
+  }
   __ blr(lr);
-  __ mov(rscratch2, r0);
+  __ mov(rscratch1, r0);
   __ pop_call_clobbered_registers();
-  __ mov(dst, rscratch2);
+  __ mov(r0, rscratch1);
 
-  __ bind(done);
+  __ bind(not_cset);
+
+  __ mov(result_dst, r0);
+  __ pop(to_save, sp);
+
+  __ bind(heap_stable);
   __ leave();
-  __ bind(is_null);
-  __ block_comment("} load_reference_barrier_weak");
 }
 
 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
@@ -308,15 +291,6 @@ void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Regis
   }
 }
 
-void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr) {
-  if (ShenandoahLoadRefBarrier) {
-    Label is_null;
-    __ cbz(dst, is_null);
-    load_reference_barrier_not_null(masm, dst, load_addr);
-    __ bind(is_null);
-  }
-}
-
 //
 // Arguments:
 //
@@ -352,11 +326,8 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
 
     BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
 
-    if (ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type)) {
-      load_reference_barrier_weak(masm, dst, src);
-    } else {
-      load_reference_barrier(masm, dst, src);
-    }
+    bool weak = ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type);
+    load_reference_barrier(masm, dst, src, weak);
 
     if (dst != result_dst) {
       __ mov(result_dst, dst);
@@ -754,67 +725,3 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
 #undef __
 
 #endif // COMPILER1
-
-address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
-  assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
-  return _shenandoah_lrb;
-}
-
-#define __ cgen->assembler()->
-
-// Shenandoah load reference barrier.
-//
-// Input:
-//   r0: OOP to evacuate.  Not null.
-//   r1: load address
-//
-// Output:
-//   r0: Pointer to evacuated OOP.
-//
-// Trash rscratch1, rscratch2.  Preserve everything else.
-address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
-
-  __ align(6);
-  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
-  address start = __ pc();
-
-  Label slow_path;
-  __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
-  __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
-  __ ldrb(rscratch2, Address(rscratch2, rscratch1));
-  __ tbnz(rscratch2, 0, slow_path);
-  __ ret(lr);
-
-  __ bind(slow_path);
-  __ enter(); // required for proper stackwalking of RuntimeStub frame
-
-  __ push_call_clobbered_registers();
-
-  if (UseCompressedOops) {
-    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
-  } else {
-    __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
-  }
-  __ blr(lr);
-  __ mov(rscratch1, r0);
-  __ pop_call_clobbered_registers();
-  __ mov(r0, rscratch1);
-
-  __ leave(); // required for proper stackwalking of RuntimeStub frame
-  __ ret(lr);
-
-  return start;
-}
-
-#undef __
-
-void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
-  if (ShenandoahLoadRefBarrier) {
-    int stub_code_size = 2048;
-    ResourceMark rm;
-    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
-    CodeBuffer buf(bb);
-    StubCodeGenerator cgen(&buf);
-    _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
-  }
-}

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp

@@ -38,8 +38,6 @@ class StubCodeGenerator;
 class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
 private:
 
-  static address _shenandoah_lrb;
-
   void satb_write_barrier_pre(MacroAssembler* masm,
                               Register obj,
                               Register pre_val,
@@ -57,14 +55,9 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
 
   void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
   void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
-  void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr);
-  void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address load_addr);
-  void load_reference_barrier_weak(MacroAssembler* masm, Register dst, Address load_addr);
-
-  address generate_shenandoah_lrb(StubCodeGenerator* cgen);
+  void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, bool weak);
 
 public:
-  static address shenandoah_lrb();
 
   void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
 
@@ -85,8 +78,6 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
                                              Register obj, Register tmp, Label& slowpath);
   void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
                    bool acquire, bool release, bool is_cae, Register result);
-
-  virtual void barrier_stubs_init();
 };
 
 #endif // CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP