8316181: Move the fast locking implementation out of the .ad files

Backport-of: b48dbf6bfa652ef031c35f0a85a409142563aa72

Author: shipilev

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3809,202 +3809,6 @@ encode %{
     __ br(target_reg);
   %}
 
-  enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
-    C2_MacroAssembler _masm(&cbuf);
-    Register oop = as_Register($object$$reg);
-    Register box = as_Register($box$$reg);
-    Register disp_hdr = as_Register($tmp$$reg);
-    Register tmp = as_Register($tmp2$$reg);
-    Label cont;
-    Label object_has_monitor;
-    Label count, no_count;
-
-    assert_different_registers(oop, box, tmp, disp_hdr);
-
-    // Load markWord from object into displaced_header.
-    __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
-
-    if (DiagnoseSyncOnValueBasedClasses != 0) {
-      __ load_klass(tmp, oop);
-      __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
-      __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
-      __ br(Assembler::NE, cont);
-    }
-
-    // Check for existing monitor
-    __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
-
-    if (LockingMode == LM_MONITOR) {
-      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
-      __ b(cont);
-    } else if (LockingMode == LM_LEGACY) {
-      // Set tmp to be (markWord of object | UNLOCK_VALUE).
-      __ orr(tmp, disp_hdr, markWord::unlocked_value);
-
-      // Initialize the box. (Must happen before we update the object mark!)
-      __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-
-      // Compare object markWord with an unlocked value (tmp) and if
-      // equal exchange the stack address of our box with object markWord.
-      // On failure disp_hdr contains the possibly locked markWord.
-      __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
-                 /*release*/ true, /*weak*/ false, disp_hdr);
-      __ br(Assembler::EQ, cont);
-
-      assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-      // If the compare-and-exchange succeeded, then we found an unlocked
-      // object, will have now locked it will continue at label cont
-
-      // Check if the owner is self by comparing the value in the
-      // markWord of object (disp_hdr) with the stack pointer.
-      __ mov(rscratch1, sp);
-      __ sub(disp_hdr, disp_hdr, rscratch1);
-      __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
-      // If condition is true we are cont and hence we can store 0 as the
-      // displaced header in the box, which indicates that it is a recursive lock.
-      __ ands(tmp/*==0?*/, disp_hdr, tmp);   // Sets flags for result
-      __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-      __ b(cont);
-    } else {
-      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-      __ lightweight_lock(oop, disp_hdr, tmp, rscratch1, no_count);
-      __ b(count);
-    }
-
-    // Handle existing monitor.
-    __ bind(object_has_monitor);
-
-    // The object's monitor m is unlocked iff m->owner == NULL,
-    // otherwise m->owner may contain a thread or a stack address.
-    //
-    // Try to CAS m->owner from NULL to current thread.
-    __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
-    __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
-               /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
-
-    if (LockingMode != LM_LIGHTWEIGHT) {
-      // Store a non-null value into the box to avoid looking like a re-entrant
-      // lock. The fast-path monitor unlock code checks for
-      // markWord::monitor_value so use markWord::unused_mark which has the
-      // relevant bit set, and also matches ObjectSynchronizer::enter.
-      __ mov(tmp, (address)markWord::unused_mark().value());
-      __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-    }
-    __ br(Assembler::EQ, cont); // CAS success means locking succeeded
-
-    __ cmp(rscratch1, rthread);
-    __ br(Assembler::NE, cont); // Check for recursive locking
-
-    // Recursive lock case
-    __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
-    // flag == EQ still from the cmp above, checking if this is a reentrant lock
-
-    __ bind(cont);
-    // flag == EQ indicates success
-    // flag == NE indicates failure
-    __ br(Assembler::NE, no_count);
-
-    __ bind(count);
-    __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
-
-    __ bind(no_count);
-  %}
-
-  enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
-    C2_MacroAssembler _masm(&cbuf);
-    Register oop = as_Register($object$$reg);
-    Register box = as_Register($box$$reg);
-    Register disp_hdr = as_Register($tmp$$reg);
-    Register tmp = as_Register($tmp2$$reg);
-    Label cont;
-    Label object_has_monitor;
-    Label count, no_count;
-
-    assert_different_registers(oop, box, tmp, disp_hdr);
-
-    if (LockingMode == LM_LEGACY) {
-      // Find the lock address and load the displaced header from the stack.
-      __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
-
-      // If the displaced header is 0, we have a recursive unlock.
-      __ cmp(disp_hdr, zr);
-      __ br(Assembler::EQ, cont);
-    }
-
-    // Handle existing monitor.
-    __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
-    __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
-
-    if (LockingMode == LM_MONITOR) {
-      __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
-      __ b(cont);
-    } else if (LockingMode == LM_LEGACY) {
-      // Check if it is still a light weight lock, this is is true if we
-      // see the stack address of the basicLock in the markWord of the
-      // object.
-
-      __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
-                 /*release*/ true, /*weak*/ false, tmp);
-      __ b(cont);
-    } else {
-      assert(LockingMode == LM_LIGHTWEIGHT, "must be");
-      __ lightweight_unlock(oop, tmp, box, disp_hdr, no_count);
-      __ b(count);
-    }
-
-    assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
-
-    // Handle existing monitor.
-    __ bind(object_has_monitor);
-    STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
-    __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
-
-    if (LockingMode == LM_LIGHTWEIGHT) {
-      // If the owner is anonymous, we need to fix it -- in an outline stub.
-      Register tmp2 = disp_hdr;
-      __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
-      // We cannot use tbnz here, the target might be too far away and cannot
-      // be encoded.
-      __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
-      C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
-      Compile::current()->output()->add_stub(stub);
-      __ br(Assembler::NE, stub->entry());
-      __ bind(stub->continuation());
-    }
-
-    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
-
-    Label notRecursive;
-    __ cbz(disp_hdr, notRecursive);
-
-    // Recursive lock
-    __ sub(disp_hdr, disp_hdr, 1u);
-    __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
-    __ cmp(disp_hdr, disp_hdr); // Sets flags for result
-    __ b(cont);
-
-    __ bind(notRecursive);
-    __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
-    __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
-    __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
-    __ cmp(rscratch1, zr); // Sets flags for result
-    __ cbnz(rscratch1, cont);
-    // need a release store here
-    __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
-    __ stlr(zr, tmp); // set unowned
-
-    __ bind(cont);
-    // flag == EQ indicates success
-    // flag == NE indicates failure
-    __ br(Assembler::NE, no_count);
-
-    __ bind(count);
-    __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
-
-    __ bind(no_count);
-  %}
-
 %}
 
 //----------FRAME--------------------------------------------------------------
@@ -16619,7 +16423,9 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegP
   ins_cost(5 * INSN_COST);
   format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
 
-  ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
+  ins_encode %{
+    __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+  %}
 
   ins_pipe(pipe_serial);
 %}
@@ -16632,7 +16438,9 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRe
   ins_cost(5 * INSN_COST);
   format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
 
-  ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
+  ins_encode %{
+    __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+  %}
 
   ins_pipe(pipe_serial);
 %}