diff --git a/Source/JavaScriptCore/assembler/AssemblerBuffer.cpp b/Source/JavaScriptCore/assembler/AssemblerBuffer.cpp index 462d0cd2d910f..564f9fd4328ab 100644 --- a/Source/JavaScriptCore/assembler/AssemblerBuffer.cpp +++ b/Source/JavaScriptCore/assembler/AssemblerBuffer.cpp @@ -45,7 +45,7 @@ ThreadSpecificAssemblerData& threadSpecificAssemblerData() return *threadSpecificAssemblerDataPtr; } -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) static ThreadSpecificAssemblerHashes* threadSpecificAssemblerHashesPtr; ThreadSpecificAssemblerHashes& threadSpecificAssemblerHashes() { @@ -57,7 +57,7 @@ ThreadSpecificAssemblerHashes& threadSpecificAssemblerHashes() }); return *threadSpecificAssemblerHashesPtr; } -#endif // ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#endif // ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) #endif // ENABLE(ASSEMBLER) diff --git a/Source/JavaScriptCore/assembler/AssemblerBuffer.h b/Source/JavaScriptCore/assembler/AssemblerBuffer.h index cfb38c866a7b3..5bc9e32749b22 100644 --- a/Source/JavaScriptCore/assembler/AssemblerBuffer.h +++ b/Source/JavaScriptCore/assembler/AssemblerBuffer.h @@ -29,18 +29,20 @@ #include "ExecutableAllocator.h" #include "JITCompilationEffort.h" -#include "SecureARM64EHashPinsInlines.h" #include "stdint.h" #include #include #include -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) -#include -#endif +#include #include #include #include +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) && CPU(ARM64E) +#include "SecureARM64EHashPinsInlines.h" +#include +#endif + namespace JSC { enum class AssemblerDataType : uint8_t { Code, Hashes }; template @@ -50,7 +52,7 @@ namespace JSC { using ThreadSpecificAssemblerData = ThreadSpecific; JS_EXPORT_PRIVATE ThreadSpecificAssemblerData& threadSpecificAssemblerData(); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) using AssemblerHashes = AssemblerDataImpl; using ThreadSpecificAssemblerHashes = ThreadSpecific; JS_EXPORT_PRIVATE ThreadSpecificAssemblerHashes& threadSpecificAssemblerHashes(); @@ -80,7 +82,7 @@ namespace JSC { inline uint32_t offset() const { -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) && CPU(ARM64E) return static_cast(untagInt(m_offset, bitwise_cast(this))); #else return m_offset; @@ -90,14 +92,14 @@ namespace JSC { private: inline void setOffset(uint32_t offset) { -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) && CPU(ARM64E) m_offset = tagInt(static_cast(offset), bitwise_cast(this)); #else m_offset = offset; #endif } -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) && CPU(ARM64E) uint64_t m_offset; #else uint32_t m_offset; @@ -115,7 +117,7 @@ namespace JSC { { if constexpr (type == AssemblerDataType::Code) takeBufferIfLarger(*threadSpecificAssemblerData()); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) if constexpr (type == AssemblerDataType::Hashes) takeBufferIfLarger(*threadSpecificAssemblerHashes()); #else @@ -177,7 +179,7 @@ namespace JSC { { if constexpr (type == AssemblerDataType::Code) threadSpecificAssemblerData()->takeBufferIfLarger(*this); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) if constexpr (type == AssemblerDataType::Hashes) threadSpecificAssemblerHashes()->takeBufferIfLarger(*this); #else @@ -216,27 +218,56 @@ namespace JSC { unsigned m_capacity; }; -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) enum class ShouldSign : bool { No, Yes }; + enum class HashType { Insecure, Pac }; + +#if CPU(ARM64E) + template +#else + template +#endif + class BufferChecksum; + template - class ARM64EHash { - WTF_MAKE_NONCOPYABLE(ARM64EHash); + class BufferChecksum { + WTF_MAKE_NONCOPYABLE(BufferChecksum); public: - ARM64EHash() + BufferChecksum() { allocatePinForCurrentThreadAndInitializeHash(); } - ~ARM64EHash() + ~BufferChecksum() { deallocatePinForCurrentThread(); } + ALWAYS_INLINE uint32_t update(uint32_t instruction, uint32_t byteIdx) + { + uint32_t instrIdx = byteIdx / sizeof(instruction); + uint32_t currentHash = this->currentHash(instrIdx); + uint64_t nextIdx = instrIdx + 1; + uint32_t output = nextWordHash(instruction, nextIdx, currentHash); + setUpdatedHash(output, nextIdx); + return output; + } + + void earlyCleanup() + { + deallocatePinForCurrentThread(); + } + + private: + static constexpr uint8_t initializationNamespace = 0x11; + ALWAYS_INLINE void allocatePinForCurrentThreadAndInitializeHash() { if constexpr (shouldSign == ShouldSign::Yes) { m_initializedPin = true; +#if CPU(ARM64E) g_jscConfig.arm64eHashPins.allocatePinForCurrentThread(); +#endif setUpdatedHash(0, 0); } else m_hash = 0; @@ -245,52 +276,46 @@ namespace JSC { void deallocatePinForCurrentThread() { if (m_initializedPin) { +#if CPU(ARM64E) g_jscConfig.arm64eHashPins.deallocatePinForCurrentThread(); +#endif m_initializedPin = false; } } - ALWAYS_INLINE uint32_t update(uint32_t instruction, uint32_t index) + static ALWAYS_INLINE PtrTag makeDiversifier(uint8_t namespaceTag, uint64_t instrIdx, uint32_t value) { - uint32_t currentHash = this->currentHash(index); - uint64_t nextIndex = index + 1; - uint32_t output = nextValue(instruction, nextIndex, currentHash); - setUpdatedHash(output, nextIndex); - return output; + // + return static_cast((static_cast(namespaceTag) << 56) + ((instrIdx & 0xFFFFFF) << 32) + value); } - private: - static constexpr uint8_t initializationNamespace = 0x11; - - static ALWAYS_INLINE PtrTag makeDiversifier(uint8_t namespaceTag, uint64_t index, uint32_t value) - { - // - return static_cast((static_cast(namespaceTag) << 56) + ((index & 0xFFFFFF) << 32) + value); - } - - static ALWAYS_INLINE uint32_t nextValue(uint64_t instruction, uint64_t index, uint32_t currentValue) + static ALWAYS_INLINE uint32_t nextWordHash(uint64_t instruction, uint64_t instrIdx, uint32_t currentValue) { - uint64_t a = tagInt(instruction, makeDiversifier(0x12, index, currentValue)); - uint64_t b = tagInt(instruction, makeDiversifier(0x13, index, currentValue)); + uint64_t a = tagInt(instruction, makeDiversifier(0x12, instrIdx, currentValue)); + uint64_t b = tagInt(instruction, makeDiversifier(0x13, instrIdx, currentValue)); return (a >> 39) ^ (b >> 23); } static ALWAYS_INLINE uint32_t pin() { +#if CPU(ARM64E) return g_jscConfig.arm64eHashPins.pinForCurrentThread(); +#else + return 1; +#endif } - ALWAYS_INLINE uint32_t currentHash(uint32_t index) + ALWAYS_INLINE uint32_t currentHash(uint32_t instrIdx) { if constexpr (shouldSign == ShouldSign::Yes) - return untagInt(m_hash, makeDiversifier(initializationNamespace, index, pin())); + return untagInt(m_hash, makeDiversifier(initializationNamespace, instrIdx, pin())); return m_hash; } - ALWAYS_INLINE void setUpdatedHash(uint32_t value, uint32_t index) + ALWAYS_INLINE void setUpdatedHash(uint32_t value, uint32_t instrIdx) { if constexpr (shouldSign == ShouldSign::Yes) - m_hash = tagInt(static_cast(value), makeDiversifier(initializationNamespace, index, pin())); + m_hash = tagInt(static_cast(value), makeDiversifier(initializationNamespace, instrIdx, pin())); else m_hash = static_cast(value); } @@ -298,26 +323,59 @@ namespace JSC { uint64_t m_hash; bool m_initializedPin { false }; }; -#endif // ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) + + template + class BufferChecksum { + WTF_MAKE_NONCOPYABLE(BufferChecksum); + public: + BufferChecksum() = default; + + template + ALWAYS_INLINE IntegralType update(IntegralType instruction, uint32_t byteIdx) + { + IntegralType result { 0 }; + for (size_t i = 0; i < sizeof(instruction); i++) { + uint8_t currentInstrByte = (instruction >> (8 * i)) & 0xFF; + + byteIdx++; + m_hash = nextByteHash(currentInstrByte, byteIdx, m_hash); + result = (result << 8) | m_hash; + } + return result; + } + + void earlyCleanup() { } + + private: + static ALWAYS_INLINE uint8_t nextByteHash(uint8_t byte, uint32_t byteIdx, uint8_t currentValue) + { + // ignore top 16b of index + uint8_t idx_hash = (byteIdx ^ (byteIdx >> 8)) & 0xFF; + return currentValue ^ byte ^ idx_hash; + } + + uint8_t m_hash { 0 }; + }; +#endif // ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) class AssemblerBuffer { public: AssemblerBuffer() : m_storage() , m_index(0) -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) , m_hash() , m_hashes() #endif { -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) ASSERT(m_storage.capacity() == m_hashes.capacity()); #endif } ~AssemblerBuffer() { -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) ASSERT(m_storage.capacity() == m_hashes.capacity()); #endif } @@ -377,7 +435,7 @@ namespace JSC { return WTFMove(m_storage); } -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) AssemblerHashes&& releaseAssemblerHashes() { return WTFMove(m_hashes); @@ -440,8 +498,8 @@ namespace JSC { void* data() const { return m_storage.buffer(); } #endif -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) - ARM64EHash& arm64eHash() { return m_hash; } +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + BufferChecksum& checksum() { return m_hash; } #endif protected: @@ -459,10 +517,10 @@ namespace JSC { { #if CPU(ARM64) static_assert(sizeof(value) == 4); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) - uint32_t hash = m_hash.update(value, m_index / sizeof(IntegralType)); - WTF::unalignedStore(m_hashes.buffer() + m_index, hash); #endif +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + auto hash = m_hash.update(value, m_index); + WTF::unalignedStore(m_hashes.buffer() + m_index, hash); #endif ASSERT(isAvailable(sizeof(IntegralType))); WTF::unalignedStore(m_storage.buffer() + m_index, value); @@ -473,7 +531,7 @@ namespace JSC { void grow(int extraCapacity = 0) { m_storage.grow(extraCapacity); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) m_hashes.grow(extraCapacity); #endif } @@ -481,7 +539,7 @@ namespace JSC { NEVER_INLINE void outOfLineGrow() { m_storage.grow(); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) m_hashes.grow(); #endif } @@ -493,8 +551,8 @@ namespace JSC { AssemblerData m_storage; unsigned m_index; -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) - ARM64EHash m_hash; +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + BufferChecksum m_hash; AssemblerHashes m_hashes; #endif }; diff --git a/Source/JavaScriptCore/assembler/LinkBuffer.cpp b/Source/JavaScriptCore/assembler/LinkBuffer.cpp index 9091f2ea1b63a..61316088db947 100644 --- a/Source/JavaScriptCore/assembler/LinkBuffer.cpp +++ b/Source/JavaScriptCore/assembler/LinkBuffer.cpp @@ -297,8 +297,8 @@ void LinkBuffer::copyCompactAndLinkCode(MacroAssembler& macroAssembler, JITCompi auto& jumpsToLink = macroAssembler.jumpsToLink(); m_assemblerStorage = macroAssembler.m_assembler.buffer().releaseAssemblerData(); uint8_t* inData = bitwise_cast(m_assemblerStorage.buffer()); -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) - ARM64EHash verifyUncompactedHash; +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + BufferChecksum verifyUncompactedHash; m_assemblerHashesStorage = macroAssembler.m_assembler.buffer().releaseAssemblerHashes(); uint32_t* inHashes = bitwise_cast(m_assemblerHashesStorage.buffer()); #endif @@ -319,10 +319,10 @@ void LinkBuffer::copyCompactAndLinkCode(MacroAssembler& macroAssembler, JITCompi auto read = [&](const InstructionType* ptr) -> InstructionType { InstructionType value = *ptr; -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) - unsigned index = (bitwise_cast(ptr) - inData) / 4; +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + unsigned index = bitwise_cast(ptr) - inData; uint32_t hash = verifyUncompactedHash.update(value, index); - RELEASE_ASSERT(inHashes[index] == hash); + RELEASE_ASSERT(inHashes[index / 4] == hash); #endif return value; }; @@ -482,6 +482,17 @@ void LinkBuffer::linkCode(MacroAssembler& macroAssembler, JITCompilationEffort e ASSERT(m_code); AssemblerBuffer& buffer = macroAssembler.m_assembler.buffer(); void* code = m_code.dataLocation(); +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + BufferChecksum verifyUncompactedHash; + m_assemblerHashesStorage = macroAssembler.m_assembler.buffer().releaseAssemblerHashes(); + uint32_t* inHashes = bitwise_cast(m_assemblerHashesStorage.buffer()); + uint32_t* inCode = bitwise_cast(buffer.data()); + // ignore the tail on variable-length ISAs for performance reasons + for (size_t i = 0; i < buffer.codeSize(); i += sizeof(uint32_t)) { + uint32_t hash = verifyUncompactedHash.update(inCode[i], i); + RELEASE_ASSERT(inHashes[i] == hash); + } +#endif #if CPU(ARM64) RELEASE_ASSERT(roundUpToMultipleOf(code) == code); #endif @@ -519,8 +530,8 @@ void LinkBuffer::allocate(MacroAssembler& macroAssembler, JITCompilationEffort e initialSize = macroAssembler.m_assembler.codeSize(); } -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) - macroAssembler.m_assembler.buffer().arm64eHash().deallocatePinForCurrentThread(); +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) + macroAssembler.m_assembler.buffer().checksum().earlyCleanup(); #endif m_executableMemory = ExecutableAllocator::singleton().allocate(initialSize, effort); diff --git a/Source/JavaScriptCore/assembler/LinkBuffer.h b/Source/JavaScriptCore/assembler/LinkBuffer.h index c8b088f536b9d..b85c131fde112 100644 --- a/Source/JavaScriptCore/assembler/LinkBuffer.h +++ b/Source/JavaScriptCore/assembler/LinkBuffer.h @@ -388,11 +388,11 @@ ALLOW_NONLITERAL_FORMAT_END RefPtr m_executableMemory; size_t m_size { 0 }; void* m_ownerUID { nullptr }; -#if ENABLE(BRANCH_COMPACTION) - AssemblerData m_assemblerStorage; -#if ENABLE(JIT_SIGN_ASSEMBLER_BUFFER) +#if ENABLE(JIT_CHECKSUM_ASSEMBLER_BUFFER) AssemblerHashes m_assemblerHashesStorage; #endif +#if ENABLE(BRANCH_COMPACTION) + AssemblerData m_assemblerStorage; bool m_shouldPerformBranchCompaction { true }; #endif bool m_didAllocate { false }; diff --git a/Source/WTF/wtf/PlatformEnable.h b/Source/WTF/wtf/PlatformEnable.h index 32aae7e93a8ba..9e1d8dc61daff 100644 --- a/Source/WTF/wtf/PlatformEnable.h +++ b/Source/WTF/wtf/PlatformEnable.h @@ -918,8 +918,8 @@ #define ENABLE_JIT_OPERATION_DISASSEMBLY 1 #endif -#if CPU(ARM64E) -#define ENABLE_JIT_SIGN_ASSEMBLER_BUFFER 1 +#if !defined(ENABLE_JIT_CHECKSUM_ASSEMBLER_BUFFER) && (CPU(ARM64E) || CPU(X86_64)) +#define ENABLE_JIT_CHECKSUM_ASSEMBLER_BUFFER 1 #endif #if !defined(ENABLE_BINDING_INTEGRITY) && !OS(WINDOWS)