8302209: [Lilliput] Optimize fix-anon monitor owner path

Reviewed-by: stuefe

Author: rkennke

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3934,13 +3934,16 @@ encode %{
     __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
 
     if (UseFastLocking) {
-      // If the owner is anonymous, we need to fix it -- in the slow-path.
-      __ ldr(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
-      // We cannot use tbnz here: tbnz would leave the condition flags untouched,
-      // but we want to carry-over the NE condition to the exit at the cont label,
-      // in order to take the slow-path.
-      __ tst(disp_hdr, (uint64_t)(intptr_t) ANONYMOUS_OWNER);
-      __ br(Assembler::NE, no_count);
+      // If the owner is anonymous, we need to fix it -- in an outline stub.
+      Register tmp2 = disp_hdr;
+      __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
+      // We cannot use tbnz here, the target might be too far away and cannot
+      // be encoded.
+      __ tst(tmp2, (uint64_t)(intptr_t) ANONYMOUS_OWNER);
+      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
+      Compile::current()->output()->add_stub(stub);
+      __ br(Assembler::NE, stub->entry());
+      __ bind(stub->continuation());
     }
 
     __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));

src/hotspot/cpu/aarch64/c2_CodeStubs_aarch64.cpp

@@ -75,6 +75,27 @@ void C2CheckLockStackStub::emit(C2_MacroAssembler& masm) {
   __ b(continuation());
 }
 
+int C2HandleAnonOMOwnerStub::max_size() const {
+  return 20;
+}
+
+void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register mon = monitor();
+  Register t = tmp();
+  assert(t != noreg, "need tmp register");
+
+  // Fix owner to be the current thread.
+  __ str(rthread, Address(mon, ObjectMonitor::owner_offset_in_bytes()));
+
+  // Pop owner object from lock-stack.
+  __ ldr(t, Address(rthread, JavaThread::lock_stack_current_offset()));
+  __ sub(t, t, oopSize);
+  __ str(t, Address(rthread, JavaThread::lock_stack_current_offset()));
+
+  __ b(continuation());
+}
+
 int C2LoadNKlassStub::max_size() const {
   return 8;
 }

src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp

@@ -85,6 +85,18 @@ void C2CheckLockStackStub::emit(C2_MacroAssembler& masm) {
 }
 
 #ifdef _LP64
+int C2HandleAnonOMOwnerStub::max_size() const {
+  return 17;
+}
+
+void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
+  __ bind(entry());
+  Register mon = monitor();
+  __ movptr(Address(mon, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), r15_thread);
+  __ subptr(Address(r15_thread, JavaThread::lock_stack_current_offset()), oopSize);
+  __ jmp(continuation());
+}
+
 int C2LoadNKlassStub::max_size() const {
   return 10;
 }

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -812,9 +812,18 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
 #endif
     jccb(Assembler::zero, Stacked);
     if (UseFastLocking) {
-      // If the owner is ANONYMOUS, we need to fix it - in the slow-path.
+      // If the owner is ANONYMOUS, we need to fix it.
       testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t) (intptr_t) ANONYMOUS_OWNER);
+#ifdef _LP64
+      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg);
+      Compile::current()->output()->add_stub(stub);
+      jcc(Assembler::notEqual, stub->entry());
+      bind(stub->continuation());
+#else
+      // We can't easily implement this optimization on 32 bit because we don't have a thread register.
+      // Call the slow-path instead.
       jcc(Assembler::notEqual, NO_COUNT);
+#endif
     }
   }
 

src/hotspot/share/opto/c2_CodeStubs.hpp

@@ -95,6 +95,19 @@ class C2CheckLockStackStub : public C2CodeStub {
 };
 
 #ifdef _LP64
+class C2HandleAnonOMOwnerStub : public C2CodeStub {
+private:
+  Register _monitor;
+  Register _tmp;
+public:
+  C2HandleAnonOMOwnerStub(Register monitor, Register tmp = noreg) : C2CodeStub(),
+    _monitor(monitor), _tmp(tmp) {}
+  Register monitor() { return _monitor; }
+  Register tmp()     { return _tmp; }
+  int max_size() const;
+  void emit(C2_MacroAssembler& masm);
+};
+
 class C2LoadNKlassStub : public C2CodeStub {
 private:
   Register _dst;