diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 84fbd5661fd0a..40e7da06666fc 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -65,6 +65,8 @@ class DWARFUnit; namespace bolt { +struct BranchLivenessInfo; + using InputOffsetToAddressMapTy = std::unordered_multimap; /// Types of macro-fusion alignment corrections. @@ -2463,7 +2465,7 @@ class BinaryFunction { /// while the second successor - false/fall-through branch. /// /// When we reverse the branch condition, the CFG is updated accordingly. - void fixBranches(); + void fixBranches(const BranchLivenessInfo *BLI = nullptr); /// Mark function as finalized. No further optimizations are permitted. void setFinalized() { CurrentState = State::CFG_Finalized; } diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 84b44a9ab5483..d0e76032c7222 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -18,6 +18,7 @@ #include "bolt/Core/Relocation.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/MC/MCAsmBackend.h" @@ -52,6 +53,7 @@ namespace bolt { class BinaryBasicBlock; class BinaryContext; class BinaryFunction; +class DataflowInfoManager; /// Different types of indirect branches encountered during disassembly. enum class IndirectBranchType : char { @@ -73,6 +75,14 @@ enum BTIKind { JC /// Accepting both. }; +struct BranchLivenessInfo { + DenseSet FlagsDead; + + bool mustPreserveFlags(const MCInst &Inst) const { + return !FlagsDead.count(&Inst); + } +}; + class MCPlusBuilder { public: using AllocatorIdTy = uint16_t; @@ -474,8 +484,16 @@ class MCPlusBuilder { return false; } + /// Return liveness info required for branch transformations. + virtual BranchLivenessInfo + createBranchLivenessInfo(BinaryFunction &BF, DataflowInfoManager &DIM) const { + return BranchLivenessInfo(); + } + /// Check whether this conditional branch can be reversed - virtual bool isReversibleBranch(const MCInst &Inst) const { + virtual bool + isReversibleBranch(const MCInst &Inst, + const BranchLivenessInfo *BLI = nullptr) const { assert(!isUnsupportedInstruction(Inst) && isConditionalBranch(Inst) && "Instruction is not known conditional branch"); @@ -2141,8 +2159,12 @@ class MCPlusBuilder { } /// Reverses the branch condition in Inst and update its taken target to TBB. - virtual void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, - MCContext *Ctx) const { + /// Assumes that the branch is reversible. It may replace Inst with a longer + /// instruction sequence on some targets. + virtual void + reverseBranchCondition(BinaryBasicBlock *Parent, MCInst &Inst, + const MCSymbol *TBB, MCContext *Ctx, + const BranchLivenessInfo *BLI = nullptr) const { llvm_unreachable("not implemented"); } diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h index 4633d30104d43..90e10943c0f70 100644 --- a/bolt/include/bolt/Passes/LongJmp.h +++ b/bolt/include/bolt/Passes/LongJmp.h @@ -14,6 +14,8 @@ namespace llvm { namespace bolt { +struct BranchLivenessInfo; + /// LongJmp is veneer-insertion pass originally written for AArch64 that /// compensates for its short-range branches, typically done during linking. We /// pull this pass inside BOLT because here we can do a better job at stub @@ -74,7 +76,7 @@ class LongJmpPass : public BinaryFunctionPass { /// Relax all internal function branches including those between fragments. /// Assume that fragments are placed in different sections but are within /// 128MB of each other. - void relaxLocalBranches(BinaryFunction &BF); + void relaxLocalBranches(BinaryFunction &BF, const BranchLivenessInfo *BLI); /// -- Layout estimation methods -- /// Try to do layout before running the emitter, by looking at BinaryFunctions diff --git a/bolt/include/bolt/Utils/CommandLineOpts.h b/bolt/include/bolt/Utils/CommandLineOpts.h index e11b18d3489cf..88b56217f9512 100644 --- a/bolt/include/bolt/Utils/CommandLineOpts.h +++ b/bolt/include/bolt/Utils/CommandLineOpts.h @@ -132,6 +132,10 @@ extern llvm::cl::opt UpdateDebugSections; // dbgs() for output within DEBUG(). extern llvm::cl::opt Verbosity; +// Option to control whether liveness analysis should be used by +// FixupBranches and LongJmpPass. Needed for branch inversion on AArch64. +extern llvm::cl::opt LivenessAnalysis; + /// Return true if we should process all functions in the binary. bool processAllFunctions(); diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index 200e286d8e80e..4fa0290af8b3b 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -3650,7 +3650,7 @@ bool BinaryFunction::validateCFG() const { return true; } -void BinaryFunction::fixBranches() { +void BinaryFunction::fixBranches(const BranchLivenessInfo *BLI) { assert(isSimple() && "Expected function with valid CFG."); auto &MIB = BC.MIB; @@ -3709,7 +3709,7 @@ void BinaryFunction::fixBranches() { // Reverse branch condition and swap successors. auto swapSuccessors = [&]() { - if (!MIB->isReversibleBranch(*CondBranch)) { + if (!MIB->isReversibleBranch(*CondBranch, BLI)) { if (opts::Verbosity) { BC.outs() << "BOLT-INFO: unable to swap successors in " << *this << '\n'; @@ -3719,7 +3719,8 @@ void BinaryFunction::fixBranches() { std::swap(TSuccessor, FSuccessor); BB->swapConditionalSuccessors(); auto L = BC.scopeLock(); - MIB->reverseBranchCondition(*CondBranch, TSuccessor->getLabel(), Ctx); + MIB->reverseBranchCondition(BB, *CondBranch, TSuccessor->getLabel(), + Ctx, BLI); return true; }; diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp index adf2bbae52d11..95ebbd8258159 100644 --- a/bolt/lib/Passes/BinaryPasses.cpp +++ b/bolt/lib/Passes/BinaryPasses.cpp @@ -11,8 +11,10 @@ //===----------------------------------------------------------------------===// #include "bolt/Passes/BinaryPasses.h" +#include "bolt/Core/BinaryFunctionCallGraph.h" #include "bolt/Core/FunctionLayout.h" #include "bolt/Core/ParallelUtilities.h" +#include "bolt/Passes/DataflowInfoManager.h" #include "bolt/Passes/ReorderAlgorithm.h" #include "bolt/Passes/ReorderFunctions.h" #include "bolt/Utils/CommandLineOpts.h" @@ -545,13 +547,25 @@ bool ReorderBasicBlocks::modifyFunctionLayout(BinaryFunction &BF, } Error FixupBranches::runOnFunctions(BinaryContext &BC) { - for (auto &It : BC.getBinaryFunctions()) { - BinaryFunction &Function = It.second; - if (!BC.shouldEmit(Function) || !Function.isSimple()) - continue; + auto forEachFunction = [&](auto &&Apply) { + for (auto &It : BC.getBinaryFunctions()) { + BinaryFunction &Function = It.second; + if (!BC.shouldEmit(Function) || !Function.isSimple()) + continue; + Apply(Function); + } + }; - Function.fixBranches(); - } + if (opts::LivenessAnalysis) { + BinaryFunctionCallGraph CG = buildCallGraph(BC); + RegAnalysis RA(BC, &BC.getBinaryFunctions(), &CG); + forEachFunction([&](BinaryFunction &BF) { + DataflowInfoManager DIM(BF, &RA, nullptr); + BranchLivenessInfo Info = BC.MIB->createBranchLivenessInfo(BF, DIM); + BF.fixBranches(&Info); + }); + } else + forEachFunction([&](BinaryFunction &BF) { BF.fixBranches(nullptr); }); return Error::success(); } @@ -961,7 +975,7 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) { uint64_t Count = 0; if (CondSucc != BB) { // Patch the new target address into the conditional branch. - MIB->reverseBranchCondition(*CondBranch, CalleeSymbol, Ctx); + MIB->reverseBranchCondition(PredBB, *CondBranch, CalleeSymbol, Ctx); // Since we reversed the condition on the branch we need to change // the target for the unconditional branch or add a unconditional // branch to the old target. This has to be done manually since diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp index b771e6a8b120a..0e8903697612d 100644 --- a/bolt/lib/Passes/LongJmp.cpp +++ b/bolt/lib/Passes/LongJmp.cpp @@ -11,7 +11,9 @@ //===----------------------------------------------------------------------===// #include "bolt/Passes/LongJmp.h" +#include "bolt/Core/BinaryFunctionCallGraph.h" #include "bolt/Core/ParallelUtilities.h" +#include "bolt/Passes/DataflowInfoManager.h" #include "bolt/Utils/CommandLineOpts.h" #include "llvm/Support/MathExtras.h" @@ -662,7 +664,8 @@ Error LongJmpPass::relax(BinaryFunction &Func, bool &Modified) { return Error::success(); } -void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { +void LongJmpPass::relaxLocalBranches(BinaryFunction &BF, + const BranchLivenessInfo *BLI) { BinaryContext &BC = BF.getBinaryContext(); auto &MIB = BC.MIB; @@ -708,14 +711,20 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { DenseMap FragmentTrampolines; // Create a trampoline code after \p BB or at the end of the fragment if BB - // is nullptr. If \p UpdateOffsets is true, update FragmentSize and offsets - // for basic blocks affected by the insertion of the trampoline. + // is nullptr. \p Offset reflects the size delta of BB caused by splitting + // unconditional branches, or replacing a branch with a longer instruction + // sequence. It is used to update the output addresses of basic blocks + // following the trampoline. auto addTrampolineAfter = [&](BinaryBasicBlock *BB, BinaryBasicBlock *TargetBB, uint64_t Count, - bool UpdateOffsets = true) { + uint64_t Offset = 0) { FunctionTrampolines.emplace_back(BB ? BB : FF.back(), BF.createBasicBlock()); BinaryBasicBlock *TrampolineBB = FunctionTrampolines.back().second.get(); + const uint64_t OldBBEnd = BB ? BB->getOutputEndAddress() : 0; + if (BB && Offset) + BB->setOutputEndAddress(OldBBEnd + Offset); + Offset += TrampolineSize; MCInst Inst; { @@ -731,13 +740,23 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { TrampolineBB->setOutputEndAddress(TrampolineAddress + TrampolineSize); TrampolineBB->setFragmentNum(FF.getFragmentNum()); + // Shift the fragment-local output address range for blocks at or after + // the old end address. + auto adjustBasicBlockAddress = [](BinaryBasicBlock *BB, uint64_t Address, + uint64_t Offset) { + if (BB->getOutputStartAddress() < Address) + return; + BB->setOutputStartAddress(BB->getOutputStartAddress() + Offset); + BB->setOutputEndAddress(BB->getOutputEndAddress() + Offset); + }; + if (!FragmentTrampolines.lookup(TargetBB)) FragmentTrampolines[TargetBB] = TrampolineBB; - if (!UpdateOffsets) + if (!Offset) return TrampolineBB; - FragmentSize += TrampolineSize; + FragmentSize += Offset; // If the trampoline was added at the end of the fragment, offsets of // other fragments should stay intact. @@ -745,13 +764,8 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { return TrampolineBB; // Update offsets for blocks after BB. - for (BinaryBasicBlock *IBB : FF) { - if (IBB->getOutputStartAddress() >= TrampolineAddress) { - IBB->setOutputStartAddress(IBB->getOutputStartAddress() + - TrampolineSize); - IBB->setOutputEndAddress(IBB->getOutputEndAddress() + TrampolineSize); - } - } + for (BinaryBasicBlock *IBB : FF) + adjustBasicBlockAddress(IBB, OldBBEnd, Offset); // Update offsets for trampolines in this fragment that are placed after // the new trampoline. Note that trampoline blocks are not part of the @@ -763,11 +777,7 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { continue; if (IBB == TrampolineBB) continue; - if (IBB->getOutputStartAddress() >= TrampolineAddress) { - IBB->setOutputStartAddress(IBB->getOutputStartAddress() + - TrampolineSize); - IBB->setOutputEndAddress(IBB->getOutputEndAddress() + TrampolineSize); - } + adjustBasicBlockAddress(IBB, OldBBEnd, Offset); } return TrampolineBB; @@ -788,7 +798,7 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { BinaryBasicBlock *TargetBB = BB->getSuccessor(TargetSymbol, BI); BinaryBasicBlock *TrampolineBB = - addTrampolineAfter(BB, TargetBB, BI.Count, /*UpdateOffsets*/ false); + addTrampolineAfter(BB, TargetBB, BI.Count, /*Offset=*/-4); BB->replaceSuccessor(TargetBB, TrampolineBB, BI.Count); } @@ -832,7 +842,7 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { // If the other successor is a fall-through, invert the condition code. BinaryBasicBlock *NextBB = BF->getLayout().getBasicBlockAfter(BB, /*IgnoreSplits*/ false); - bool IsReversibleBranch = MIB->isReversibleBranch(Inst); + bool IsReversibleBranch = MIB->isReversibleBranch(Inst, BLI); bool ShouldReverseBranch = BB->getConditionalSuccessor(false) == NextBB; // Create a trampoline basic block for the fall-through target of the @@ -844,14 +854,22 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { BB->replaceSuccessor(NextBB, FallThrough, NextCount); } - // Create a trampoline basic block for the taken target of the branch. - TrampolineBB = addTrampolineAfter(BB, TargetBB, Count); - if (ShouldReverseBranch && IsReversibleBranch) { + const uint64_t OldBBSize = BB->estimateSize(); BB->swapConditionalSuccessors(); - auto L = BC.scopeLock(); - MIB->reverseBranchCondition(Inst, NextBB->getLabel(), BC.Ctx.get()); + { + auto L = BC.scopeLock(); + MIB->reverseBranchCondition(BB, Inst, NextBB->getLabel(), + BC.Ctx.get(), BLI); + } + const uint64_t NewBBSize = BB->estimateSize(); + + // Create a trampoline basic block for the original taken target. + TrampolineBB = + addTrampolineAfter(BB, TargetBB, Count, NewBBSize - OldBBSize); } else { + // Create a trampoline basic block for the taken target of the branch. + TrampolineBB = addTrampolineAfter(BB, TargetBB, Count); auto L = BC.scopeLock(); MIB->replaceBranchTarget(Inst, TrampolineBB->getLabel(), BC.Ctx.get()); } @@ -866,7 +884,10 @@ void LongJmpPass::relaxLocalBranches(BinaryFunction &BF) { for (auto BBI = FF.begin(); BBI != FF.end(); ++BBI) { BinaryBasicBlock *BB = *BBI; uint64_t NextInstOffset = BB->getOutputStartAddress(); - for (MCInst &Inst : *BB) { + // Branch reversal may replace the current instruction with a sequence. + // Use an index so the next instruction is reloaded after the mutation. + for (size_t I = 0; I < BB->size(); ++I) { + MCInst &Inst = *(BB->begin() + I); const size_t InstAddress = NextInstOffset; if (!MIB->isPseudo(Inst)) NextInstOffset += 4; @@ -935,19 +956,37 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) { opts::SplitStrategy != opts::SplitFunctionsStrategy::CDSplit) && "LongJmp cannot work with functions split in more than two fragments"); + DenseMap BranchLiveness; + if (opts::LivenessAnalysis) { + BinaryFunctionCallGraph CG = buildCallGraph(BC); + RegAnalysis RA(BC, &BC.getBinaryFunctions(), &CG); + for (auto &It : BC.getBinaryFunctions()) { + BinaryFunction &BF = It.second; + if (!BC.shouldEmit(BF) || !BF.isSimple()) + continue; + DataflowInfoManager DIM(BF, &RA, nullptr); + BranchLiveness[&BF] = BC.MIB->createBranchLivenessInfo(BF, DIM); + } + } + auto getBranchLiveness = + [&](BinaryFunction &BF) -> const BranchLivenessInfo * { + auto It = BranchLiveness.find(&BF); + return It == BranchLiveness.end() ? nullptr : &It->second; + }; + if (opts::CompactCodeModel) { BC.outs() << "BOLT-INFO: relaxing branches for compact code model (<128MB)\n"; - ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) { - relaxLocalBranches(BF); - }; - ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) { return !BC.shouldEmit(BF) || !BF.isSimple(); }; + ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) { + relaxLocalBranches(BF, getBranchLiveness(BF)); + }; + ParallelUtilities::runOnEachFunction( BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun, SkipPredicate, "RelaxLocalBranches"); @@ -970,7 +1009,7 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) { // Don't ruin non-simple functions, they can't afford to have the layout // changed. if (Modified && Func->isSimple()) - Func->fixBranches(); + Func->fixBranches(getBranchLiveness(*Func)); } } while (Modified); BC.outs() << "BOLT-INFO: Inserted " << NumHotStubs diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index b4a9dff9d25b2..3503cd7df4750 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -22,6 +22,7 @@ #include "bolt/Core/BinaryFunction.h" #include "bolt/Core/MCInstUtils.h" #include "bolt/Core/MCPlusBuilder.h" +#include "bolt/Passes/DataflowInfoManager.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" @@ -2045,6 +2046,25 @@ class AArch64MCPlusBuilder : public MCPlusBuilder { exit(1); } + unsigned getInvertedCC(unsigned Opcode) const { + // clang-format off + switch (Opcode) { + default: + llvm_unreachable("Failed to invert condition code"); + return Opcode; + // Compare register with immediate and branch. + case AArch64::CBGTWri: return AArch64CC::LE; + case AArch64::CBGTXri: return AArch64CC::LE; + case AArch64::CBLTWri: return AArch64CC::GE; + case AArch64::CBLTXri: return AArch64CC::GE; + case AArch64::CBHIWri: return AArch64CC::LS; + case AArch64::CBHIXri: return AArch64CC::LS; + case AArch64::CBLOWri: return AArch64CC::HS; + case AArch64::CBLOXri: return AArch64CC::HS; + } + // clang-format on + } + unsigned getInvertedBranchOpcode(unsigned Opcode) const { // clang-format off switch (Opcode) { @@ -2171,38 +2191,95 @@ class AArch64MCPlusBuilder : public MCPlusBuilder { } } - bool isReversibleBranch(const MCInst &Inst) const override { + BranchLivenessInfo + createBranchLivenessInfo(BinaryFunction &BF, + DataflowInfoManager &DIM) const override { + SmallVector CompAndBranchInsts; + for (BinaryBasicBlock &BB : BF) + for (MCInst &Inst : BB) + if (isCompAndBranch(Inst)) + CompAndBranchInsts.push_back(&Inst); + if (CompAndBranchInsts.empty()) + return {}; + + BranchLivenessInfo Info; + LivenessAnalysis &LA = DIM.getLivenessAnalysis(); + for (MCInst *Inst : CompAndBranchInsts) + if (!LA.getLiveIn(*Inst).test(getFlagsReg())) + Info.FlagsDead.insert(Inst); + return Info; + } + + bool + isReversibleBranch(const MCInst &Inst, + const BranchLivenessInfo *BLI = nullptr) const override { if (isCompAndBranch(Inst)) { + bool MustPreserveFlags = BLI ? BLI->mustPreserveFlags(Inst) : true; unsigned InvertedOpcode = getInvertedBranchOpcode(Inst.getOpcode()); - if (needsImmDec(InvertedOpcode) && Inst.getOperand(1).getImm() == 0) + if (needsImmDec(InvertedOpcode) && Inst.getOperand(1).getImm() == 0 && + MustPreserveFlags) return false; - if (needsImmInc(InvertedOpcode) && Inst.getOperand(1).getImm() == 63) + if (needsImmInc(InvertedOpcode) && Inst.getOperand(1).getImm() == 63 && + MustPreserveFlags) return false; } return MCPlusBuilder::isReversibleBranch(Inst); } - void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, - MCContext *Ctx) const override { - if (!isReversibleBranch(Inst)) { - errs() << "BOLT-ERROR: Cannot reverse branch " << Inst << "\n"; - exit(1); - } + void reverseBranchCondition( + BinaryBasicBlock *Parent, MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx, const BranchLivenessInfo *BLI = nullptr) const override { + assert(isReversibleBranch(Inst, BLI) && "Irreversible branch"); if (isTB(Inst) || isCB(Inst) || isCompAndBranch(Inst)) { + bool ImmediateOutOfBounds = false; unsigned InvertedOpcode = getInvertedBranchOpcode(Inst.getOpcode()); - Inst.setOpcode(InvertedOpcode); - assert(Inst.getOpcode() != 0 && "Invalid branch instruction"); + assert(InvertedOpcode != 0 && "Invalid branch instruction"); // The FEAT_CMPBR compare-and-branch instructions cannot encode all // the possible condition codes, therefore we either have to adjust // the immediate value by +-1, or to swap the register operands // when reversing the branch condition. if (needsRegSwap(InvertedOpcode)) std::swap(Inst.getOperand(0), Inst.getOperand(1)); - else if (needsImmDec(InvertedOpcode)) - Inst.getOperand(1).setImm(Inst.getOperand(1).getImm() - 1); - else if (needsImmInc(InvertedOpcode)) - Inst.getOperand(1).setImm(Inst.getOperand(1).getImm() + 1); + else if (needsImmDec(InvertedOpcode)) { + if (Inst.getOperand(1).getImm() == 0) + ImmediateOutOfBounds = true; + else + Inst.getOperand(1).setImm(Inst.getOperand(1).getImm() - 1); + } else if (needsImmInc(InvertedOpcode)) { + if (Inst.getOperand(1).getImm() == 63) + ImmediateOutOfBounds = true; + else + Inst.getOperand(1).setImm(Inst.getOperand(1).getImm() + 1); + } + if (ImmediateOutOfBounds) { + auto is32BitVariant = [](unsigned Opcode) { + switch (Opcode) { + default: + return false; + case AArch64::CBGTWri: + case AArch64::CBLTWri: + case AArch64::CBHIWri: + case AArch64::CBLOWri: + return true; + } + }; + InstructionListType Code; + MCInstBuilder Cmp = + is32BitVariant(InvertedOpcode) + ? MCInstBuilder(AArch64::SUBSWri).addReg(AArch64::WZR) + : MCInstBuilder(AArch64::SUBSXri).addReg(AArch64::XZR); + Cmp.addReg(Inst.getOperand(0).getReg()) + .addImm(Inst.getOperand(1).getImm()) + .addImm(0); + Code.emplace_back(std::move(Cmp)); + Code.emplace_back(MCInstBuilder(AArch64::Bcc) + .addImm(getInvertedCC(Inst.getOpcode())) + .addExpr(MCSymbolRefExpr::create(TBB, *Ctx))); + Parent->replaceInstruction(Parent->findInstruction(&Inst), Code); + return; + } + Inst.setOpcode(InvertedOpcode); } else if (Inst.getOpcode() == AArch64::Bcc) { Inst.getOperand(0).setImm(AArch64CC::getInvertedCondCode( static_cast(Inst.getOperand(0).getImm()))); diff --git a/bolt/lib/Target/AArch64/CMakeLists.txt b/bolt/lib/Target/AArch64/CMakeLists.txt index 1e171748aece6..e28ed0bd66ba1 100644 --- a/bolt/lib/Target/AArch64/CMakeLists.txt +++ b/bolt/lib/Target/AArch64/CMakeLists.txt @@ -29,7 +29,11 @@ add_llvm_library(LLVMBOLTTargetAArch64 AArch64CommonTableGen ) -target_link_libraries(LLVMBOLTTargetAArch64 PRIVATE LLVMBOLTCore LLVMBOLTUtils) +target_link_libraries(LLVMBOLTTargetAArch64 PRIVATE + LLVMBOLTCore + LLVMBOLTPasses + LLVMBOLTUtils + ) include_directories( ${LLVM_MAIN_SRC_DIR}/lib/Target/AArch64 diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp index d1a0572277874..34e8392f59743 100644 --- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -162,8 +162,9 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { } } - void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, - MCContext *Ctx) const override { + void reverseBranchCondition( + BinaryBasicBlock *Parent, MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx, const BranchLivenessInfo *BLI = nullptr) const override { auto Opcode = getInvertedBranchOpcode(Inst.getOpcode()); Inst.setOpcode(Opcode); replaceBranchTarget(Inst, TBB, Ctx); diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index 11a297f514530..98b9eb8fdd4cb 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -2811,8 +2811,9 @@ class X86MCPlusBuilder : public MCPlusBuilder { Inst.addOperand(MCOperand::createImm(CC)); } - void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, - MCContext *Ctx) const override { + void reverseBranchCondition( + BinaryBasicBlock *Parent, MCInst &Inst, const MCSymbol *TBB, + MCContext *Ctx, const BranchLivenessInfo *BLI = nullptr) const override { unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); diff --git a/bolt/lib/Utils/CommandLineOpts.cpp b/bolt/lib/Utils/CommandLineOpts.cpp index 20b24c3b4acc5..82fde5788397a 100644 --- a/bolt/lib/Utils/CommandLineOpts.cpp +++ b/bolt/lib/Utils/CommandLineOpts.cpp @@ -365,6 +365,12 @@ cl::opt cl::init(0), cl::ZeroOrMore, cl::cat(BoltCategory), cl::sub(cl::SubCommand::getAll())); +cl::opt LivenessAnalysis( + "liveness-analysis", + cl::desc("use liveness analysis in FixupBranches and LongJmpPass" + "(needed for branch inversion on AArch64)"), + cl::init(false), cl::cat(BoltCategory)); + bool processAllFunctions() { if (opts::AggregateOnly) return false; diff --git a/bolt/test/AArch64/compare-and-branch-inversion.S b/bolt/test/AArch64/compare-and-branch-inversion.S index 28167416c31cb..0ea084e904101 100644 --- a/bolt/test/AArch64/compare-and-branch-inversion.S +++ b/bolt/test/AArch64/compare-and-branch-inversion.S @@ -1,18 +1,24 @@ # This test checks that branch inversion works when reordering blocks which # contain short range conditional branches. Handles edge cases, like when # the immediate value is the upper or lower allowed value in which case the -# transformation bails. +# transformation bails. If liveness analysis proves that the condition flags +# are dead we can replace the branch with cmp + b.cc # REQUIRES: system-linux, asserts # RUN: %clang %cflags -march=armv9-a+cmpbr -Wl,-q %s -o %t # RUN: link_fdata --no-lbr %s %t %t.fdata # RUN: llvm-strip --strip-unneeded %t +# # RUN: llvm-bolt -v=1 %t -o %t.bolt --data %t.fdata --reorder-blocks=ext-tsp --compact-code-model \ -# RUN: | FileCheck %s --check-prefix=BOLT-INFO -# RUN: llvm-objdump -d %t.bolt | FileCheck %s +# RUN: | FileCheck %s --check-prefix=BOLT-INFO-NO-LIVENESS +# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=COMMON --check-prefix=NO-LIVENESS +# +# RUN: llvm-bolt -v=1 %t -o %t.bolt --data %t.fdata --reorder-blocks=ext-tsp --compact-code-model \ +# RUN: --liveness-analysis | FileCheck %s --check-prefix=BOLT-INFO-LIVENESS +# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=COMMON --check-prefix=LIVENESS -# CHECK: Disassembly of section .text: +# COMMON: Disassembly of section .text: .globl immediate_increment .type immediate_increment, %function @@ -29,12 +35,12 @@ immediate_increment: mov x0, #2 ret -# CHECK: : -# CHECK-NEXT: {{.*}} cblt x0, #0x1, 0x[[ADDR0:[0-9a-f]+]] <{{.*}}> -# CHECK-NEXT: {{.*}} mov x0, #0x2 // =2 -# CHECK-NEXT: {{.*}} ret -# CHECK-NEXT: [[ADDR0]]: {{.*}} mov x0, #0x1 // =1 -# CHECK-NEXT: {{.*}} ret +# COMMON: : +# COMMON-NEXT: {{.*}} cblt x0, #0x1, 0x[[ADDR0:[0-9a-f]+]] <{{.*}}> +# COMMON-NEXT: {{.*}} mov x0, #0x2 // =2 +# COMMON-NEXT: {{.*}} ret +# COMMON-NEXT: [[ADDR0]]: {{.*}} mov x0, #0x1 // =1 +# COMMON-NEXT: {{.*}} ret .globl immediate_decrement .type immediate_decrement, %function @@ -51,12 +57,12 @@ immediate_decrement: mov x0, #2 ret -# CHECK: : -# CHECK-NEXT: {{.*}} cbhi x0, #0x0, 0x[[ADDR1:[0-9a-f]+]] <{{.*}}> -# CHECK-NEXT: {{.*}} mov x0, #0x2 // =2 -# CHECK-NEXT: {{.*}} ret -# CHECK-NEXT: [[ADDR1]]: {{.*}} mov x0, #0x1 // =1 -# CHECK-NEXT: {{.*}} ret +# COMMON: : +# COMMON-NEXT: {{.*}} cbhi x0, #0x0, 0x[[ADDR1:[0-9a-f]+]] <{{.*}}> +# COMMON-NEXT: {{.*}} mov x0, #0x2 // =2 +# COMMON-NEXT: {{.*}} ret +# COMMON-NEXT: [[ADDR1]]: {{.*}} mov x0, #0x1 // =1 +# COMMON-NEXT: {{.*}} ret .globl register_swap .type register_swap, %function @@ -73,37 +79,77 @@ register_swap: mov x0, #2 ret -# CHECK: : -# CHECK-NEXT: {{.*}} cbgt x1, x0, 0x[[ADDR2:[0-9a-f]+]] <{{.*}}> -# CHECK-NEXT: {{.*}} mov x0, #0x2 // =2 -# CHECK-NEXT: {{.*}} ret -# CHECK-NEXT: [[ADDR2]]: {{.*}} mov x0, #0x1 // =1 -# CHECK-NEXT: {{.*}} ret +# COMMON: : +# COMMON-NEXT: {{.*}} cbgt x1, x0, 0x[[ADDR2:[0-9a-f]+]] <{{.*}}> +# COMMON-NEXT: {{.*}} mov x0, #0x2 // =2 +# COMMON-NEXT: {{.*}} ret +# COMMON-NEXT: [[ADDR2]]: {{.*}} mov x0, #0x1 // =1 +# COMMON-NEXT: {{.*}} ret - .globl irreversible - .type irreversible, %function -irreversible: + .globl immediate_overflow + .type immediate_overflow, %function +immediate_overflow: .entry3: -# FDATA: 1 irreversible #.entry3# 10 +# FDATA: 1 immediate_overflow #.entry3# 10 cbgt x0, #63, .exit3 .cold3: -# FDATA: 1 irreversible #.cold3# 1 +# FDATA: 1 immediate_overflow #.cold3# 1 mov x0, #1 ret .exit3: -# FDATA: 1 irreversible #.exit3# 10 +# FDATA: 1 immediate_overflow #.exit3# 10 + mov x0, #2 + ret + +# BOLT-INFO-NO-LIVENESS: unable to swap successors in immediate_overflow +# +# Without liveness the blocks get reordered, but since the branch is +# irreversible an additional unconditional branch is emitted. +# This codegen is suboptimal yet correct. +# +# NO-LIVENESS: : +# NO-LIVENESS-NEXT: {{.*}} cbgt x0, #0x3f, 0x[[ADDR3:[0-9a-f]+]] <{{.*}}> +# NO-LIVENESS-NEXT: {{.*}} b 0x[[ADDR4:[0-9a-f]+]] <{{.*}}> +# NO-LIVENESS-NEXT: [[ADDR3]]: {{.*}} mov x0, #0x2 // =2 +# NO-LIVENESS-NEXT: {{.*}} ret +# NO-LIVENESS-NEXT: [[ADDR4]]: {{.*}} mov x0, #0x1 // =1 +# NO-LIVENESS-NEXT: {{.*}} ret + +# LIVENESS: : +# LIVENESS-NEXT: {{.*}} cmp x0, #0x3f +# LIVENESS-NEXT: {{.*}} b.le 0x[[ADDR5:[0-9a-f]+]] <{{.*}}> +# LIVENESS-NEXT: {{.*}} mov x0, #0x2 // =2 +# LIVENESS-NEXT: {{.*}} ret +# LIVENESS-NEXT: [[ADDR5]]: {{.*}} mov x0, #0x1 // =1 +# LIVENESS-NEXT: {{.*}} ret + + .globl irreversible + .type irreversible, %function +irreversible: +.entry4: +# FDATA: 1 irreversible #.entry4# 10 + cmp x0, #63 + cbgt x0, #63, .exit4 +.cold4: +# FDATA: 1 irreversible #.cold4# 1 + csel x0, x1, x2, le + ret +.exit4: +# FDATA: 1 irreversible #.exit4# 10 mov x0, #2 ret -# BOLT-INFO: unable to swap successors in irreversible +# BOLT-INFO-NO-LIVENESS: unable to swap successors in irreversible +# BOLT-INFO-LIVENESS: unable to swap successors in irreversible -# CHECK: : -# CHECK-NEXT: {{.*}} cbgt x0, #0x3f, 0x[[ADDR3:[0-9a-f]+]] <{{.*}}> -# CHECK-NEXT: {{.*}} b 0x[[ADDR4:[0-9a-f]+]] <{{.*}}> -# CHECK-NEXT: [[ADDR3]]: {{.*}} mov x0, #0x2 // =2 -# CHECK-NEXT: {{.*}} ret -# CHECK-NEXT: [[ADDR4]]: {{.*}} mov x0, #0x1 // =1 -# CHECK-NEXT: {{.*}} ret +# COMMON: : +# COMMON-NEXT: {{.*}} cmp x0, #0x3f +# COMMON-NEXT: {{.*}} cbgt x0, #0x3f, 0x[[ADDR6:[0-9a-f]+]] <{{.*}}> +# COMMON-NEXT: {{.*}} b 0x[[ADDR7:[0-9a-f]+]] <{{.*}}> +# COMMON-NEXT: [[ADDR6]]: {{.*}} mov x0, #0x2 // =2 +# COMMON-NEXT: {{.*}} ret +# COMMON-NEXT: [[ADDR7]]: {{.*}} csel x0, x1, x2, le +# COMMON-NEXT: {{.*}} ret ## Force relocation mode. .reloc 0, R_AARCH64_NONE diff --git a/bolt/unittests/Core/CMakeLists.txt b/bolt/unittests/Core/CMakeLists.txt index 297dec7449202..b755afc8a66da 100644 --- a/bolt/unittests/Core/CMakeLists.txt +++ b/bolt/unittests/Core/CMakeLists.txt @@ -24,6 +24,7 @@ target_link_libraries(CoreTests PRIVATE LLVMBOLTCore LLVMBOLTRewrite + LLVMBOLTPasses LLVMBOLTProfile LLVMBOLTUtils ) diff --git a/bolt/unittests/Core/MCPlusBuilder.cpp b/bolt/unittests/Core/MCPlusBuilder.cpp index e67460fe2a6a6..5fdaf47094770 100644 --- a/bolt/unittests/Core/MCPlusBuilder.cpp +++ b/bolt/unittests/Core/MCPlusBuilder.cpp @@ -18,6 +18,9 @@ #include "bolt/Core/BinaryBasicBlock.h" #include "bolt/Core/BinaryFunction.h" +#include "bolt/Core/BinaryFunctionCallGraph.h" +#include "bolt/Passes/BinaryPasses.h" +#include "bolt/Passes/DataflowInfoManager.h" #include "bolt/Rewrite/RewriteInstance.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" @@ -233,8 +236,8 @@ TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch) { .addExpr(MCSymbolRefExpr::create( TargetBB->getLabel(), *BC->Ctx.get())); ASSERT_TRUE(BC->MIB->isReversibleBranch(NeedsImmInc)); - BC->MIB->reverseBranchCondition(NeedsImmInc, TargetBB->getLabel(), - BC->Ctx.get()); + BC->MIB->reverseBranchCondition(/*ParentBB*/ nullptr, NeedsImmInc, + TargetBB->getLabel(), BC->Ctx.get()); ASSERT_EQ(NeedsImmInc.getOpcode(), AArch64::CBLTXri); ASSERT_EQ(NeedsImmInc.getOperand(1).getImm(), 1); @@ -247,8 +250,8 @@ TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch) { .addExpr(MCSymbolRefExpr::create( TargetBB->getLabel(), *BC->Ctx.get())); ASSERT_TRUE(BC->MIB->isReversibleBranch(NeedsImmDec)); - BC->MIB->reverseBranchCondition(NeedsImmDec, TargetBB->getLabel(), - BC->Ctx.get()); + BC->MIB->reverseBranchCondition(/*ParentBB*/ nullptr, NeedsImmDec, + TargetBB->getLabel(), BC->Ctx.get()); ASSERT_EQ(NeedsImmDec.getOpcode(), AArch64::CBHIXri); ASSERT_EQ(NeedsImmDec.getOperand(1).getImm(), 0); @@ -261,8 +264,8 @@ TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch) { .addExpr(MCSymbolRefExpr::create( TargetBB->getLabel(), *BC->Ctx.get())); ASSERT_TRUE(BC->MIB->isReversibleBranch(CompRegNeedsRegSwap)); - BC->MIB->reverseBranchCondition(CompRegNeedsRegSwap, TargetBB->getLabel(), - BC->Ctx.get()); + BC->MIB->reverseBranchCondition(/*ParentBB*/ nullptr, CompRegNeedsRegSwap, + TargetBB->getLabel(), BC->Ctx.get()); ASSERT_EQ(CompRegNeedsRegSwap.getOpcode(), AArch64::CBGTXrr); ASSERT_EQ(CompRegNeedsRegSwap.getOperand(0).getReg(), AArch64::X1); ASSERT_EQ(CompRegNeedsRegSwap.getOperand(1).getReg(), AArch64::X0); @@ -276,8 +279,8 @@ TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch) { .addExpr(MCSymbolRefExpr::create( TargetBB->getLabel(), *BC->Ctx.get())); ASSERT_TRUE(BC->MIB->isReversibleBranch(CompByteNeedsRegSwap)); - BC->MIB->reverseBranchCondition(CompByteNeedsRegSwap, TargetBB->getLabel(), - BC->Ctx.get()); + BC->MIB->reverseBranchCondition(/*ParentBB*/ nullptr, CompByteNeedsRegSwap, + TargetBB->getLabel(), BC->Ctx.get()); ASSERT_EQ(CompByteNeedsRegSwap.getOpcode(), AArch64::CBBHSWrr); ASSERT_EQ(CompByteNeedsRegSwap.getOperand(0).getReg(), AArch64::W1); ASSERT_EQ(CompByteNeedsRegSwap.getOperand(1).getReg(), AArch64::W0); @@ -291,8 +294,8 @@ TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch) { .addExpr(MCSymbolRefExpr::create( TargetBB->getLabel(), *BC->Ctx.get())); ASSERT_TRUE(BC->MIB->isReversibleBranch(CompHalfNeedsRegSwap)); - BC->MIB->reverseBranchCondition(CompHalfNeedsRegSwap, TargetBB->getLabel(), - BC->Ctx.get()); + BC->MIB->reverseBranchCondition(/*ParentBB*/ nullptr, CompHalfNeedsRegSwap, + TargetBB->getLabel(), BC->Ctx.get()); ASSERT_EQ(CompHalfNeedsRegSwap.getOpcode(), AArch64::CBHHIWrr); ASSERT_EQ(CompHalfNeedsRegSwap.getOperand(0).getReg(), AArch64::W1); ASSERT_EQ(CompHalfNeedsRegSwap.getOperand(1).getReg(), AArch64::W0); @@ -318,6 +321,129 @@ TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch) { ASSERT_FALSE(BC->MIB->isReversibleBranch(Overflows)); } +TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch_Underflows) { + if (GetParam() != Triple::aarch64) + GTEST_SKIP(); + + BinaryFunction *BF = BC->createInjectedBinaryFunction("BF", true); + BinaryBasicBlock *EntryBB = BF->addBasicBlock(); + BinaryBasicBlock *FallThroughBB = BF->addBasicBlock(); + BinaryBasicBlock *TargetBB = BF->addBasicBlock(); + BF->addEntryPoint(*EntryBB); + EntryBB->addSuccessor(TargetBB); + EntryBB->addSuccessor(FallThroughBB); + + // Inversion requires expansion, immediate value underflows. + // cblt x0, #0, target ~> cmp x0, #0 + // b.ge target + auto I = + EntryBB->addInstruction(MCInstBuilder(AArch64::CBLTXri) + .addReg(AArch64::X0) + .addImm(0) + .addExpr(MCSymbolRefExpr::create( + TargetBB->getLabel(), *BC->Ctx.get()))); + BinaryFunctionCallGraph CG(buildCallGraph(*BC.get())); + RegAnalysis RA(*BC.get(), &BC->getBinaryFunctions(), &CG); + DataflowInfoManager DIM(*BF, &RA, nullptr); + BranchLivenessInfo BranchLiveness = + BC->MIB->createBranchLivenessInfo(*BF, DIM); + + ASSERT_TRUE(BC->MIB->isReversibleBranch(*I, &BranchLiveness)); + BC->MIB->reverseBranchCondition(EntryBB, *I, TargetBB->getLabel(), + BC->Ctx.get(), &BranchLiveness); + I = EntryBB->begin(); + ASSERT_EQ(I->getOpcode(), AArch64::SUBSXri); + ASSERT_EQ(I->getOperand(0).getReg(), AArch64::XZR); + ASSERT_EQ(I->getOperand(1).getReg(), AArch64::X0); + ASSERT_EQ(I->getOperand(2).getImm(), 0); + ASSERT_EQ(I->getOperand(3).getImm(), 0); + I++; + ASSERT_EQ(I->getOpcode(), AArch64::Bcc); + ASSERT_EQ(I->getOperand(0).getImm(), AArch64CC::GE); +} + +TEST_P(MCPlusBuilderTester, AArch64_ReverseCompAndBranch_Overflows) { + if (GetParam() != Triple::aarch64) + GTEST_SKIP(); + + BinaryFunction *BF = BC->createInjectedBinaryFunction("BF", true); + BinaryBasicBlock *EntryBB = BF->addBasicBlock(); + BinaryBasicBlock *FallThroughBB = BF->addBasicBlock(); + BinaryBasicBlock *TargetBB = BF->addBasicBlock(); + BF->addEntryPoint(*EntryBB); + EntryBB->addSuccessor(TargetBB); + EntryBB->addSuccessor(FallThroughBB); + + // Inversion requires expansion, immediate value overflows. + // cbhi w0, #63, target ~> cmp w0, #63 + // b.ls target + auto I = + EntryBB->addInstruction(MCInstBuilder(AArch64::CBHIWri) + .addReg(AArch64::W0) + .addImm(63) + .addExpr(MCSymbolRefExpr::create( + TargetBB->getLabel(), *BC->Ctx.get()))); + BinaryFunctionCallGraph CG(buildCallGraph(*BC.get())); + RegAnalysis RA(*BC.get(), &BC->getBinaryFunctions(), &CG); + DataflowInfoManager DIM(*BF, &RA, nullptr); + BranchLivenessInfo BranchLiveness = + BC->MIB->createBranchLivenessInfo(*BF, DIM); + + ASSERT_TRUE(BC->MIB->isReversibleBranch(*I, &BranchLiveness)); + BC->MIB->reverseBranchCondition(EntryBB, *I, TargetBB->getLabel(), + BC->Ctx.get(), &BranchLiveness); + I = EntryBB->begin(); + ASSERT_EQ(I->getOpcode(), AArch64::SUBSWri); + ASSERT_EQ(I->getOperand(0).getReg(), AArch64::WZR); + ASSERT_EQ(I->getOperand(1).getReg(), AArch64::W0); + ASSERT_EQ(I->getOperand(2).getImm(), 63); + ASSERT_EQ(I->getOperand(3).getImm(), 0); + I++; + ASSERT_EQ(I->getOpcode(), AArch64::Bcc); + ASSERT_EQ(I->getOperand(0).getImm(), AArch64CC::LS); +} + +TEST_P(MCPlusBuilderTester, AArch64_IsReversibleBranch_LiveCondFlags) { + if (GetParam() != Triple::aarch64) + GTEST_SKIP(); + + BinaryFunction *BF = BC->createInjectedBinaryFunction("BF", true); + BinaryBasicBlock *EntryBB = BF->addBasicBlock(); + BinaryBasicBlock *FallThroughBB = BF->addBasicBlock(); + BinaryBasicBlock *TargetBB = BF->addBasicBlock(); + BF->addEntryPoint(*EntryBB); + EntryBB->addSuccessor(TargetBB); + EntryBB->addSuccessor(FallThroughBB); + + // cmp x0, #63 + EntryBB->addInstruction(MCInstBuilder(AArch64::SUBSXri) + .addReg(AArch64::XZR) + .addReg(AArch64::X0) + .addImm(63) + .addImm(0)); + // cbgt x0, #63, target + auto I = + EntryBB->addInstruction(MCInstBuilder(AArch64::CBGTXri) + .addReg(AArch64::X0) + .addImm(63) + .addExpr(MCSymbolRefExpr::create( + TargetBB->getLabel(), *BC->Ctx.get()))); + // csel x0, x1, x2, le + FallThroughBB->addInstruction(MCInstBuilder(AArch64::CSELXr) + .addReg(AArch64::X0) + .addReg(AArch64::X1) + .addReg(AArch64::X2) + .addImm(13)); + + BinaryFunctionCallGraph CG(buildCallGraph(*BC.get())); + RegAnalysis RA(*BC.get(), &BC->getBinaryFunctions(), &CG); + DataflowInfoManager DIM(*BF, &RA, nullptr); + BranchLivenessInfo BranchLiveness = + BC->MIB->createBranchLivenessInfo(*BF, DIM); + + ASSERT_FALSE(BC->MIB->isReversibleBranch(*I, &BranchLiveness)); +} + TEST_P(MCPlusBuilderTester, AArch64_CmpJE) { if (GetParam() != Triple::aarch64) GTEST_SKIP();