dolphin-emu#11191 Merge jitarm64-no-checked-entry into dev+

Source/Core/Core/PowerPC/CachedInterpreter/CachedInterpreter.cpp

@@ -260,7 +260,6 @@ void CachedInterpreter::Jit(u32 address)
   js.numFloatingPointInst = 0;
   js.curBlock = b;
 
-  b->checkedEntry = GetCodePtr();
   b->normalEntry = GetCodePtr();
 
   for (u32 i = 0; i < code_block.m_num_instructions; i++)
@@ -322,7 +321,7 @@ void CachedInterpreter::Jit(u32 address)
   }
   m_code.emplace_back();
 
-  b->codeSize = (u32)(GetCodePtr() - b->checkedEntry);
+  b->codeSize = (u32)(GetCodePtr() - b->normalEntry);
   b->originalSize = code_block.m_num_instructions;
 
   m_block_cache.FinalizeBlock(*b, jo.enableBlocklink, code_block.m_physical_addresses);

Source/Core/Core/PowerPC/Jit64/Jit.cpp

@@ -918,7 +918,6 @@ bool Jit64::DoJit(u32 em_address, JitBlock* b, u32 nextPC)
 
   // TODO: Test if this or AlignCode16 make a difference from GetCodePtr
   u8* const start = AlignCode4();
-  b->checkedEntry = start;
   b->normalEntry = start;
 
   // Used to get a trace of the last few blocks before a crash, sometimes VERY useful

Source/Core/Core/PowerPC/Jit64Common/BlockCache.cpp

@@ -14,8 +14,7 @@ JitBlockCache::JitBlockCache(JitBase& jit) : JitBaseBlockCache{jit}
 void JitBlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest)
 {
   u8* location = source.exitPtrs;
-  const u8* address =
-      dest ? dest->checkedEntry : m_jit.GetAsmRoutines()->dispatcher_no_timing_check;
+  const u8* address = dest ? dest->normalEntry : m_jit.GetAsmRoutines()->dispatcher_no_timing_check;
   if (source.call)
   {
     Gen::XEmitter emit(location, location + 5);
@@ -42,11 +41,9 @@ void JitBlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBl
 
 void JitBlockCache::WriteDestroyBlock(const JitBlock& block)
 {
-  // Only clear the entry points as we might still be within this block.
-  Gen::XEmitter emit(block.checkedEntry, block.checkedEntry + 1);
+  // Only clear the entry point as we might still be within this block.
+  Gen::XEmitter emit(block.normalEntry, block.normalEntry + 1);
   emit.INT3();
-  Gen::XEmitter emit2(block.normalEntry, block.normalEntry + 1);
-  emit2.INT3();
 }
 
 void JitBlockCache::Init()

Source/Core/Core/PowerPC/JitArm64/Jit.cpp

@@ -401,18 +401,35 @@ void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return
 
   if (LK)
   {
-    // Push {ARM_PC+20; PPC_PC} on the stack
+    // Push {ARM_PC; PPC_PC} on the stack
     MOVI2R(ARM64Reg::X1, exit_address_after_return);
-    ADR(ARM64Reg::X0, 20);
+    ADR(ARM64Reg::X0, JitArm64BlockCache::BLOCK_LINK_FAST_BL_OFFSET + 12);
     STP(IndexType::Pre, ARM64Reg::X0, ARM64Reg::X1, ARM64Reg::SP, -16);
   }
 
+  constexpr size_t farcode_size = 3 * sizeof(u32);
+  const bool switch_to_far_code = !IsInFarCode();
+  const u8* farcode_1_addr;
+  if (switch_to_far_code)
+  {
+    SwitchToFarCode();
+    farcode_1_addr = GetCodePtr();
+    SwitchToNearCode();
+  }
+  else
+  {
+    farcode_1_addr = GetCodePtr() + JitArm64BlockCache::BLOCK_LINK_SIZE +
+                     (LK ? JitArm64BlockCache::BLOCK_LINK_SIZE : 0);
+  }
+  const u8* farcode_2_addr = farcode_1_addr + farcode_size;
+
   JitBlock* b = js.curBlock;
   JitBlock::LinkData linkData;
   linkData.exitAddress = destination;
   linkData.exitPtrs = GetWritableCodePtr();
   linkData.linkStatus = false;
   linkData.call = LK;
+  linkData.exitFarcode = farcode_1_addr;
   b->linkData.push_back(linkData);
 
   blocks.WriteLinkBlock(*this, linkData);
@@ -424,10 +441,32 @@ void JitArm64::WriteExit(u32 destination, bool LK, u32 exit_address_after_return
     linkData.exitPtrs = GetWritableCodePtr();
     linkData.linkStatus = false;
     linkData.call = false;
+    linkData.exitFarcode = farcode_2_addr;
     b->linkData.push_back(linkData);
 
     blocks.WriteLinkBlock(*this, linkData);
   }
+
+  if (switch_to_far_code)
+    SwitchToFarCode();
+  ASSERT(GetCodePtr() == farcode_1_addr);
+  MOVI2R(DISPATCHER_PC, destination);
+  if (LK)
+    BL(GetAsmRoutines()->do_timing);
+  else
+    B(GetAsmRoutines()->do_timing);
+
+  if (LK)
+  {
+    if (GetCodePtr() == farcode_2_addr - sizeof(u32))
+      BRK(101);
+    ASSERT(GetCodePtr() == farcode_2_addr);
+    MOVI2R(DISPATCHER_PC, exit_address_after_return);
+    B(GetAsmRoutines()->do_timing);
+  }
+
+  if (switch_to_far_code)
+    SwitchToNearCode();
 }
 
 void JitArm64::WriteExit(Arm64Gen::ARM64Reg dest, bool LK, u32 exit_address_after_return)
@@ -461,9 +500,28 @@ void JitArm64::WriteExit(Arm64Gen::ARM64Reg dest, bool LK, u32 exit_address_afte
     linkData.exitPtrs = GetWritableCodePtr();
     linkData.linkStatus = false;
     linkData.call = false;
+    const bool switch_to_far_code = !IsInFarCode();
+    if (switch_to_far_code)
+    {
+      SwitchToFarCode();
+      linkData.exitFarcode = GetCodePtr();
+      SwitchToNearCode();
+    }
+    else
+    {
+      linkData.exitFarcode = GetCodePtr() + JitArm64BlockCache::BLOCK_LINK_SIZE;
+    }
     b->linkData.push_back(linkData);
 
     blocks.WriteLinkBlock(*this, linkData);
+
+    if (switch_to_far_code)
+      SwitchToFarCode();
+    linkData.exitFarcode = GetCodePtr();
+    MOVI2R(DISPATCHER_PC, exit_address_after_return);
+    B(GetAsmRoutines()->do_timing);
+    if (switch_to_far_code)
+      SwitchToNearCode();
   }
 }
 
@@ -492,10 +550,29 @@ void JitArm64::FakeLKExit(u32 exit_address_after_return)
   linkData.exitPtrs = GetWritableCodePtr();
   linkData.linkStatus = false;
   linkData.call = false;
+  const bool switch_to_far_code = !IsInFarCode();
+  if (switch_to_far_code)
+  {
+    SwitchToFarCode();
+    linkData.exitFarcode = GetCodePtr();
+    SwitchToNearCode();
+  }
+  else
+  {
+    linkData.exitFarcode = GetCodePtr() + JitArm64BlockCache::BLOCK_LINK_SIZE;
+  }
   b->linkData.push_back(linkData);
 
   blocks.WriteLinkBlock(*this, linkData);
 
+  if (switch_to_far_code)
+    SwitchToFarCode();
+  linkData.exitFarcode = GetCodePtr();
+  MOVI2R(DISPATCHER_PC, exit_address_after_return);
+  B(GetAsmRoutines()->do_timing);
+  if (switch_to_far_code)
+    SwitchToNearCode();
+
   SetJumpTarget(skip_exit);
 }
 
@@ -870,18 +947,7 @@ bool JitArm64::DoJit(u32 em_address, JitBlock* b, u32 nextPC)
   js.numFloatingPointInst = 0;
 
   u8* const start = GetWritableCodePtr();
-  b->checkedEntry = start;
-
-  // Downcount flag check, Only valid for linked blocks
-  {
-    FixupBranch bail = B(CC_PL);
-    MOVI2R(DISPATCHER_PC, js.blockStart);
-    B(do_timing);
-    SetJumpTarget(bail);
-  }
-
-  // Normal entry doesn't need to check for downcount.
-  b->normalEntry = GetWritableCodePtr();
+  b->normalEntry = start;
 
   // Conditionally add profiling code.
   if (jo.profile_blocks)

Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.cpp

@@ -22,64 +22,75 @@ void JitArm64BlockCache::Init()
 void JitArm64BlockCache::WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit,
                                         const JitBlock::LinkData& source, const JitBlock* dest)
 {
+  const u8* start = emit.GetCodePtr();
+
   if (!dest)
   {
-    // Use a fixed amount of instructions, so we can assume to use 3 instructions on patching.
-    emit.MOVZ(DISPATCHER_PC, source.exitAddress & 0xFFFF, ShiftAmount::Shift0);
-    emit.MOVK(DISPATCHER_PC, source.exitAddress >> 16, ShiftAmount::Shift16);
-
+    emit.MOVI2R(DISPATCHER_PC, source.exitAddress);
     if (source.call)
+    {
+      while (emit.GetCodePtr() < start + BLOCK_LINK_FAST_BL_OFFSET)
+        emit.NOP();
       emit.BL(m_jit.GetAsmRoutines()->dispatcher);
+    }
     else
+    {
       emit.B(m_jit.GetAsmRoutines()->dispatcher);
-    return;
+    }
   }
-
-  if (source.call)
+  else
   {
-    // The "fast" BL must be the third instruction. So just use the former two to inline the
-    // downcount check here. It's better to do this near jump before the long jump to the other
-    // block.
-    FixupBranch fast_link = emit.B(CC_GT);
-    emit.BL(dest->checkedEntry);
-    emit.SetJumpTarget(fast_link);
-    emit.BL(dest->normalEntry);
-    return;
+    if (source.call)
+    {
+      // The "fast" BL should be the last instruction, so that the return address matches the
+      // address that was pushed onto the stack by the function that called WriteLinkBlock
+      FixupBranch fast = emit.B(CC_GT);
+      emit.B(source.exitFarcode);
+      emit.SetJumpTarget(fast);
+      emit.BL(dest->normalEntry);
+    }
+    else
+    {
+      // Are we able to jump directly to the block?
+      s64 block_distance = ((s64)dest->normalEntry - (s64)emit.GetCodePtr()) >> 2;
+      if (block_distance >= -0x40000 && block_distance <= 0x3FFFF)
+      {
+        emit.B(CC_GT, dest->normalEntry);
+        emit.B(source.exitFarcode);
+      }
+      else
+      {
+        FixupBranch slow = emit.B(CC_LE);
+        emit.B(dest->normalEntry);
+        emit.SetJumpTarget(slow);
+        emit.B(source.exitFarcode);
+      }
+    }
   }
 
-  // Are we able to jump directly to the normal entry?
-  s64 distance = ((s64)dest->normalEntry - (s64)emit.GetCodePtr()) >> 2;
-  if (distance >= -0x40000 && distance <= 0x3FFFF)
-  {
-    emit.B(CC_GT, dest->normalEntry);
-    emit.B(dest->checkedEntry);
+  // Use a fixed number of instructions so we have enough room for any patching needed later.
+  const u8* end = start + BLOCK_LINK_SIZE;
+  while (emit.GetCodePtr() < end)
     emit.BRK(101);
-    return;
-  }
-
-  FixupBranch fast_link = emit.B(CC_GT);
-  emit.B(dest->checkedEntry);
-  emit.SetJumpTarget(fast_link);
-  emit.B(dest->normalEntry);
+  ASSERT(emit.GetCodePtr() == end);
 }
 
 void JitArm64BlockCache::WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest)
 {
   const Common::ScopedJITPageWriteAndNoExecute enable_jit_page_writes;
   u8* location = source.exitPtrs;
-  ARM64XEmitter emit(location, location + 12);
+  ARM64XEmitter emit(location, location + BLOCK_LINK_SIZE);
 
   WriteLinkBlock(emit, source, dest);
   emit.FlushIcache();
 }
 
 void JitArm64BlockCache::WriteDestroyBlock(const JitBlock& block)
 {
-  // Only clear the entry points as we might still be within this block.
-  ARM64XEmitter emit(block.checkedEntry, block.normalEntry + 4);
+  // Only clear the entry point as we might still be within this block.
+  ARM64XEmitter emit(block.normalEntry, block.normalEntry + 4);
   const Common::ScopedJITPageWriteAndNoExecute enable_jit_page_writes;
-  while (emit.GetWritableCodePtr() <= block.normalEntry)
-    emit.BRK(0x123);
+  emit.BRK(0x123);
   emit.FlushIcache();
 }
 

Source/Core/Core/PowerPC/JitArm64/JitArm64Cache.h

@@ -29,6 +29,9 @@ class JitArm64BlockCache : public JitBaseBlockCache
   void WriteLinkBlock(Arm64Gen::ARM64XEmitter& emit, const JitBlock::LinkData& source,
                       const JitBlock* dest = nullptr);
 
+  static constexpr size_t BLOCK_LINK_SIZE = 3 * sizeof(u32);
+  static constexpr size_t BLOCK_LINK_FAST_BL_OFFSET = 2 * sizeof(u32);
+
 private:
   void WriteLinkBlock(const JitBlock::LinkData& source, const JitBlock* dest) override;
   void WriteDestroyBlock(const JitBlock& block) override;

Source/Core/Core/PowerPC/JitCommon/JitCache.cpp

@@ -132,12 +132,12 @@ void JitBaseBlockCache::FinalizeBlock(JitBlock& block, bool block_link,
   if (JitRegister::IsEnabled() &&
       (symbol = g_symbolDB.GetSymbolFromAddr(block.effectiveAddress)) != nullptr)
   {
-    JitRegister::Register(block.checkedEntry, block.codeSize, "JIT_PPC_{}_{:08x}",
+    JitRegister::Register(block.normalEntry, block.codeSize, "JIT_PPC_{}_{:08x}",
                           symbol->function_name.c_str(), block.physicalAddress);
   }
   else
   {
-    JitRegister::Register(block.checkedEntry, block.codeSize, "JIT_PPC_{:08x}",
+    JitRegister::Register(block.normalEntry, block.codeSize, "JIT_PPC_{:08x}",
                           block.physicalAddress);
   }
 }

Source/Core/Core/PowerPC/JitCommon/JitCache.h

@@ -29,9 +29,6 @@ struct JitBlockData
   u8* far_begin;
   u8* far_end;
 
-  // A special entry point for block linking; usually used to check the
-  // downcount.
-  u8* checkedEntry;
   // The normal entry point for the block, returned by Dispatch().
   u8* normalEntry;
 
@@ -72,6 +69,9 @@ struct JitBlock : public JitBlockData
   struct LinkData
   {
     u8* exitPtrs;  // to be able to rewrite the exit jump
+#ifdef _M_ARM_64
+    const u8* exitFarcode;
+#endif
     u32 exitAddress;
     bool linkStatus;  // is it already linked?
     bool call;

Source/Core/Core/PowerPC/JitInterface.cpp

@@ -178,7 +178,7 @@ std::variant<GetHostCodeError, GetHostCodeResult> GetHostCode(u32 address)
   }
 
   GetHostCodeResult result;
-  result.code = block->checkedEntry;
+  result.code = block->normalEntry;
   result.code_size = block->codeSize;
   result.entry_address = block->effectiveAddress;
   return result;