diff --git a/Utilities/JIT.cpp b/Utilities/JIT.cpp index 0c5653fe6b9f..72b98a558f22 100644 --- a/Utilities/JIT.cpp +++ b/Utilities/JIT.cpp @@ -10,6 +10,7 @@ #include "StrFmt.h" #include "File.h" #include "Log.h" +#include "VirtualMemory.h" #ifdef _MSC_VER #pragma warning(push, 0) @@ -25,12 +26,6 @@ #ifdef _WIN32 #include -#else -#include -#include -#include -#include -#include #endif #include "JIT.h" @@ -44,19 +39,15 @@ static const u64 s_memory_size = 0x20000000; // Try to reserve a portion of virtual memory in the first 2 GB address space beforehand, if possible. static void* const s_memory = []() -> void* { -#ifdef _WIN32 - for (u64 addr = 0x10000000; addr <= 0x60000000; addr += 0x1000000) + for (u64 addr = 0x10000000; addr <= 0x80000000 - s_memory_size; addr += 0x1000000) { - if (VirtualAlloc((void*)addr, s_memory_size, MEM_RESERVE, PAGE_NOACCESS)) + if (auto ptr = utils::memory_reserve(s_memory_size, (void*)addr)) { - return (void*)addr; + return ptr; } } - return VirtualAlloc(NULL, s_memory_size, MEM_RESERVE, PAGE_NOACCESS); -#else - return ::mmap((void*)0x10000000, s_memory_size, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0); -#endif + return utils::memory_reserve(s_memory_size); }(); // Code section @@ -76,8 +67,12 @@ struct MemoryManager final : llvm::RTDyldMemoryManager { std::unordered_map& m_link; + std::array* m_tramps; + MemoryManager(std::unordered_map& table) : m_link(table) + , m_next(s_memory) + , m_tramps(nullptr) { } @@ -88,23 +83,53 @@ struct MemoryManager final : llvm::RTDyldMemoryManager virtual u64 getSymbolAddress(const std::string& name) override { - const auto found = m_link.find(name); + auto& addr = m_link[name]; - if (found != m_link.end()) + // Find function address + if (!addr) { - return found->second; + addr = RTDyldMemoryManager::getSymbolAddress(name); + + if (addr) + { + LOG_WARNING(GENERAL, "LLVM: Symbol requested: %s -> 0x%016llx", name, addr); + } + else + { + // It's fine if some function is never called, for example. + LOG_ERROR(GENERAL, "LLVM: Linkage failed: %s", name); + addr = (u64)null; + } } - if (u64 addr = RTDyldMemoryManager::getSymbolAddress(name)) + // Verify address for small code model + if ((u64)s_memory > 0x80000000 - s_memory_size ? (u64)addr - (u64)s_memory >= s_memory_size : addr >= 0x80000000) { - // This may be bad if LLVM requests some built-in functions like fma. - LOG_ERROR(GENERAL, "LLVM: Symbol requested: %s -> 0x%016llx", name, addr); - return addr; + // Allocate memory for trampolines + if (!m_tramps) + { + m_tramps = reinterpret_cast(m_next); + utils::memory_commit(m_next, 4096, utils::protection::wx); + m_next = (u8*)((u64)m_next + 4096); + } + + // Create a trampoline + auto& data = *m_tramps++; + data[0x0] = 0x48; // MOV rax, imm64 + data[0x1] = 0xb8; + std::memcpy(data.data() + 2, &addr, 8); + data[0xa] = 0xff; // JMP rax + data[0xb] = 0xe0; + addr = (u64)&data; + + // Reset pointer (memory page exhausted) + if (((u64)m_tramps % 4096) == 0) + { + m_tramps = nullptr; + } } - // It's fine if some function is never called, for example. - LOG_ERROR(GENERAL, "LLVM: Symbol not found: %s", name); - return (u64)null; + return addr; } virtual u8* allocateCodeSection(std::uintptr_t size, uint align, uint sec_id, llvm::StringRef sec_name) override @@ -118,16 +143,7 @@ struct MemoryManager final : llvm::RTDyldMemoryManager return nullptr; } -#ifdef _WIN32 - if (!VirtualAlloc(m_next, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE)) -#else - if (::mprotect(m_next, size, PROT_READ | PROT_WRITE | PROT_EXEC)) -#endif - { - LOG_FATAL(GENERAL, "LLVM: Failed to allocate memory at %p", m_next); - return nullptr; - } - + utils::memory_commit(m_next, size, utils::protection::wx); s_code_addr = (u8*)m_next; s_code_size = size; @@ -151,15 +167,7 @@ struct MemoryManager final : llvm::RTDyldMemoryManager LOG_ERROR(GENERAL, "LLVM: Writeable data section not supported!"); } -#ifdef _WIN32 - if (!VirtualAlloc(m_next, size, MEM_COMMIT, PAGE_READWRITE)) -#else - if (::mprotect(m_next, size, PROT_READ | PROT_WRITE)) -#endif - { - LOG_FATAL(GENERAL, "LLVM: Failed to allocate memory at %p", m_next); - return nullptr; - } + utils::memory_commit(m_next, size); LOG_NOTICE(GENERAL, "LLVM: Data section %u '%s' allocated -> %p (size=0x%llx, aligned 0x%x, %s)", sec_id, sec_name.data(), m_next, size, align, is_ro ? "ro" : "rw"); return (u8*)std::exchange(m_next, (void*)next); @@ -206,23 +214,15 @@ struct MemoryManager final : llvm::RTDyldMemoryManager } s_unwind.clear(); - - if (!VirtualFree(s_memory, 0, MEM_DECOMMIT)) - { - LOG_FATAL(GENERAL, "VirtualFree(%p) failed! Error %u", s_memory, GetLastError()); - } #else - if (!::mmap(s_memory, s_memory_size, PROT_NONE, MAP_FIXED | MAP_ANON | MAP_PRIVATE, -1, 0)) - { - LOG_FATAL(GENERAL, "mmap(%p) failed! Error %d", s_memory, errno); - } - // TODO: unregister EH frames if necessary #endif + + utils::memory_decommit(s_memory, s_memory_size); } private: - void* m_next = s_memory; + void* m_next; }; // Helper class @@ -242,18 +242,10 @@ struct EventListener final : llvm::JITEventListener static EventListener s_listener; -static void dummy() -{ -} - jit_compiler::jit_compiler(std::unordered_map init_linkage_info, std::string _cpu) : m_link(std::move(init_linkage_info)) , m_cpu(std::move(_cpu)) { -#ifdef _MSC_VER - m_link.emplace("__chkstk", (u64)&dummy); -#endif - verify(HERE), s_memory; // Initialization @@ -277,7 +269,7 @@ jit_compiler::jit_compiler(std::unordered_map init_ .setErrorStr(&result) .setMCJITMemoryManager(std::make_unique(m_link)) .setOptLevel(llvm::CodeGenOpt::Aggressive) - .setCodeModel((u64)s_memory <= 0x60000000 ? llvm::CodeModel::Small : llvm::CodeModel::Large) // TODO + .setCodeModel(llvm::CodeModel::Small) .setMCPU(m_cpu) .create()); @@ -405,6 +397,8 @@ void jit_compiler::init() } s_unwind.emplace_back(std::move(unwind)); +#else + // TODO: register EH frames if necessary #endif } diff --git a/Utilities/VirtualMemory.cpp b/Utilities/VirtualMemory.cpp index ff6558a0eb8c..c52539ef7f19 100644 --- a/Utilities/VirtualMemory.cpp +++ b/Utilities/VirtualMemory.cpp @@ -13,21 +13,49 @@ namespace utils { - void* memory_reserve(std::size_t size) + // Convert memory protection (internal) + static auto operator +(protection prot) { #ifdef _WIN32 - return verify("reserve_memory" HERE, ::VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_NOACCESS)); + DWORD _prot = PAGE_NOACCESS; + switch (prot) + { + case protection::rw: _prot = PAGE_READWRITE; break; + case protection::ro: _prot = PAGE_READONLY; break; + case protection::no: break; + case protection::wx: _prot = PAGE_EXECUTE_READWRITE; break; + case protection::rx: _prot = PAGE_EXECUTE_READ; break; + } #else - return verify("reserve_memory" HERE, ::mmap(nullptr, size, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0)); + int _prot = PROT_NONE; + switch (prot) + { + case protection::rw: _prot = PROT_READ | PROT_WRITE; break; + case protection::ro: _prot = PROT_READ; break; + case protection::no: break; + case protection::wx: _prot = PROT_READ | PROT_WRITE | PROT_EXEC; break; + case protection::rx: _prot = PROT_READ | PROT_EXEC; break; + } #endif + + return _prot; } - void memory_commit(void* pointer, std::size_t size) + void* memory_reserve(std::size_t size, void* use_addr) { #ifdef _WIN32 - verify(HERE), ::VirtualAlloc(pointer, size, MEM_COMMIT, PAGE_READWRITE); + return verify("reserve_memory" HERE, ::VirtualAlloc(use_addr, size, MEM_RESERVE, PAGE_NOACCESS)); #else - verify(HERE), ::mprotect((void*)((u64)pointer & -4096), ::align(size, 4096), PROT_READ | PROT_WRITE) != -1; + return verify("reserve_memory" HERE, ::mmap(use_addr, size, PROT_NONE, MAP_ANON | MAP_PRIVATE | (use_addr ? MAP_FIXED : 0), -1, 0)); +#endif + } + + void memory_commit(void* pointer, std::size_t size, protection prot) + { +#ifdef _WIN32 + verify(HERE), ::VirtualAlloc(pointer, size, MEM_COMMIT, +prot); +#else + verify(HERE), ::mprotect((void*)((u64)pointer & -4096), ::align(size, 4096), +prot) != -1; #endif } @@ -43,30 +71,10 @@ namespace utils void memory_protect(void* pointer, std::size_t size, protection prot) { #ifdef _WIN32 - DWORD _prot = PAGE_NOACCESS; - switch (prot) - { - case protection::rw: _prot = PAGE_READWRITE; break; - case protection::ro: _prot = PAGE_READONLY; break; - case protection::no: break; - case protection::wx: _prot = PAGE_EXECUTE_READWRITE; break; - case protection::rx: _prot = PAGE_EXECUTE_READ; break; - } - DWORD old; - verify(HERE), ::VirtualProtect(pointer, size, _prot, &old); + verify(HERE), ::VirtualProtect(pointer, size, +prot, &old); #else - int _prot = PROT_NONE; - switch (prot) - { - case protection::rw: _prot = PROT_READ | PROT_WRITE; break; - case protection::ro: _prot = PROT_READ; break; - case protection::no: break; - case protection::wx: _prot = PROT_READ | PROT_WRITE | PROT_EXEC; break; - case protection::rx: _prot = PROT_READ | PROT_EXEC; break; - } - - verify(HERE), ::mprotect((void*)((u64)pointer & -4096), ::align(size, 4096), _prot) != -1; + verify(HERE), ::mprotect((void*)((u64)pointer & -4096), ::align(size, 4096), +prot) != -1; #endif } } diff --git a/Utilities/VirtualMemory.h b/Utilities/VirtualMemory.h index e8381f8c8ed2..74f441a598dc 100644 --- a/Utilities/VirtualMemory.h +++ b/Utilities/VirtualMemory.h @@ -2,33 +2,34 @@ namespace utils { + // Memory protection type + enum class protection + { + rw, // Read + write (default) + ro, // Read only + no, // No access + wx, // Read + write + execute + rx, // Read + execute + }; + /** * Reserve `size` bytes of virtual memory and returns it. * The memory should be commited before usage. */ - void* memory_reserve(std::size_t size); + void* memory_reserve(std::size_t size, void* use_addr = nullptr); /** * Commit `size` bytes of virtual memory starting at pointer. * That is, bake reserved memory with physical memory. * pointer should belong to a range of reserved memory. */ - void memory_commit(void* pointer, std::size_t size); + void memory_commit(void* pointer, std::size_t size, protection prot = protection::rw); /** * Decommit all memory committed via commit_page_memory. */ void memory_decommit(void* pointer, std::size_t size); - // Memory protection type - enum class protection - { - rw, // Read + write (default) - ro, // Read only - no, // No access - wx, // Read + write + execute - rx, // Read + execute - }; - + // Set memory protection void memory_protect(void* pointer, std::size_t size, protection prot); } diff --git a/rpcs3/Emu/Cell/PPUTranslator.cpp b/rpcs3/Emu/Cell/PPUTranslator.cpp index 9447c8d56834..36b7f4afd7bf 100644 --- a/rpcs3/Emu/Cell/PPUTranslator.cpp +++ b/rpcs3/Emu/Cell/PPUTranslator.cpp @@ -244,7 +244,7 @@ Function* PPUTranslator::TranslateToIR(const ppu_function& info, be_t* bin, m_ir->SetInsertPoint(m_blocks.at(block.first)); // Bloat the beginning of each block: check state - const auto vstate = m_ir->CreateLoad(m_ir->CreateConstGEP2_32(nullptr, m_thread, 0, 1)); + const auto vstate = m_ir->CreateLoad(m_ir->CreateConstGEP2_32(nullptr, m_thread, 0, 1), true); const auto vblock = BasicBlock::Create(m_context, fmt::format("l0c_%llx", block.first), m_function); const auto vcheck = BasicBlock::Create(m_context, fmt::format("lcc_%llx", block.first), m_function); @@ -3703,7 +3703,7 @@ void PPUTranslator::FCTIW(ppu_opcode_t op) void PPUTranslator::FCTIWZ(ppu_opcode_t op) { const auto b = GetFpr(op.frb); - SetFpr(op.frd, m_ir->CreateFPToSI(b, GetType())); + SetFpr(op.frd, Call(GetType(), "llvm.x86.sse2.cvttsd2si", m_ir->CreateInsertElement(GetUndef(), b, uint64_t{0}))); } void PPUTranslator::FDIV(ppu_opcode_t op) @@ -3949,7 +3949,7 @@ void PPUTranslator::FCTID(ppu_opcode_t op) void PPUTranslator::FCTIDZ(ppu_opcode_t op) { const auto b = GetFpr(op.frb); - SetFpr(op.frd, m_ir->CreateFPToSI(b, GetType())); + SetFpr(op.frd, Call(GetType(), "llvm.x86.sse2.cvttsd2si64", m_ir->CreateInsertElement(GetUndef(), b, uint64_t{0}))); } void PPUTranslator::FCFID(ppu_opcode_t op)