diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 36403fab9efbb..2cebc03310e17 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -45,11 +45,166 @@ enum EdgeKind_aarch64 : Edge::Kind { /// only const char *getEdgeKindName(Edge::Kind K); -bool isLoadStoreImm12(uint32_t Instr); +// Returns whether the Instr is LD/ST (imm12) +inline bool isLoadStoreImm12(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + return (Instr & LoadStoreImm12Mask) == 0x39000000; +} -unsigned getPageOffset12Shift(uint32_t Instr); +// Returns the amount the address operand of LD/ST (imm12) +// should be shifted right by. +// +// The shift value varies by the data size of LD/ST instruction. +// For instance, LDH instructoin needs the address to be shifted +// right by 1. +inline unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t Vec128Mask = 0x04800000; + + if (isLoadStoreImm12(Instr)) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; +} + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast(Value) & 0x3) + return make_error("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (!isInt<33>(PageDelta)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return makeTargetOutOfRangeError(G, B, E); + + uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + case TLVPage21: + case GOTPage21: + case TLVPageOffset12: + case GOTPageOffset12: + case PointerToGOT: { + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); + } + default: + return make_error( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + "unsupported edge kind" + getEdgeKindName(E.getKind())); + } -Error applyFixup(LinkGraph &G, Block &B, const Edge &E); + return Error::success(); +} /// AArch64 null pointer content. extern const uint8_t NullGOTEntryContent[8]; diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 697c31b65a0b2..1397a45df4c2f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,159 +18,6 @@ namespace llvm { namespace jitlink { namespace aarch64 { -bool isLoadStoreImm12(uint32_t Instr) { - constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; - return (Instr & LoadStoreImm12Mask) == 0x39000000; -} - -unsigned getPageOffset12Shift(uint32_t Instr) { - constexpr uint32_t Vec128Mask = 0x04800000; - - if (isLoadStoreImm12(Instr)) { - uint32_t ImplicitShift = Instr >> 30; - if (ImplicitShift == 0) - if ((Instr & Vec128Mask) == Vec128Mask) - ImplicitShift = 4; - - return ImplicitShift; - } - - return 0; -} - -Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { - using namespace support; - - char *BlockWorkingMem = B.getAlreadyMutableContent().data(); - char *FixupPtr = BlockWorkingMem + E.getOffset(); - orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch26: { - assert((FixupAddress.getValue() & 0x3) == 0 && - "Branch-inst is not 32-bit aligned"); - - int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - - if (static_cast(Value) & 0x3) - return make_error("Branch26 target is not 32-bit " - "aligned"); - - if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(little32_t *)FixupPtr; - assert((RawInstr & 0x7fffffff) == 0x14000000 && - "RawInstr isn't a B or BR immediate instruction"); - uint32_t Imm = (static_cast(Value) & ((1 << 28) - 1)) >> 2; - uint32_t FixedInstr = RawInstr | Imm; - *(little32_t *)FixupPtr = FixedInstr; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - if (Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case Page21: { - assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && - "GOTPAGE21 with non-zero addend"); - uint64_t TargetPage = - (E.getTarget().getAddress().getValue() + E.getAddend()) & - ~static_cast(4096 - 1); - uint64_t PCPage = - FixupAddress.getValue() & ~static_cast(4096 - 1); - - int64_t PageDelta = TargetPage - PCPage; - if (!isInt<33>(PageDelta)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert((RawInstr & 0xffffffe0) == 0x90000000 && - "RawInstr isn't an ADRP instruction"); - uint32_t ImmLo = (static_cast(PageDelta) >> 12) & 0x3; - uint32_t ImmHi = (static_cast(PageDelta) >> 14) & 0x7ffff; - uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case PageOffset12: { - uint64_t TargetOffset = - (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; - - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - unsigned ImmShift = getPageOffset12Shift(RawInstr); - - if (TargetOffset & ((1 << ImmShift) - 1)) - return make_error("PAGEOFF12 target is not aligned"); - - uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case LDRLiteral19: { - assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned"); - assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); - uint32_t RawInstr = *(ulittle32_t *)FixupPtr; - assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); - int64_t Delta = E.getTarget().getAddress() - FixupAddress; - if (Delta & 0x3) - return make_error("LDR literal target is not 32-bit " - "aligned"); - if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) - return makeTargetOutOfRangeError(G, B, E); - - uint32_t EncodedImm = ((static_cast(Delta) >> 2) & 0x7ffff) << 5; - uint32_t FixedInstr = RawInstr | EncodedImm; - *(ulittle32_t *)FixupPtr = FixedInstr; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return makeTargetOutOfRangeError(G, B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - case TLVPage21: - case GOTPage21: - case TLVPageOffset12: - case GOTPageOffset12: - case PointerToGOT: { - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind())); - } - default: - return make_error( - "In graph " + G.getName() + ", section " + B.getSection().getName() + - "unsupported edge kind" + getEdgeKindName(E.getKind())); - } - - return Error::success(); -} - const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};