diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index 82684dcafc0f4..383b265586674 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -121,37 +121,10 @@ struct GVNOptions { /// FIXME: We should have a good summary of the GVN algorithm implemented by /// this particular pass here. class GVNPass : public OptionalPassInfoMixin { - GVNOptions Options; - public: struct Expression; struct AvailableValue; struct AvailableValueInBlock; - - GVNPass(GVNOptions Options = {}) : Options(Options) {} - - /// Run the pass over the function. - LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); - - LLVM_ABI void - printPipeline(raw_ostream &OS, - function_ref MapClassName2PassName); - - /// This removes the specified instruction from - /// our various maps and marks it for deletion. - LLVM_ABI void salvageAndRemoveInstruction(Instruction *I); - - DominatorTree &getDominatorTree() const { return *DT; } - AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); } - MemoryDependenceResults &getMemDep() const { return *MD; } - - LLVM_ABI bool isScalarPREEnabled() const; - LLVM_ABI bool isLoadPREEnabled() const; - LLVM_ABI bool isLoadInLoopPREEnabled() const; - LLVM_ABI bool isLoadPRESplitBackedgeEnabled() const; - LLVM_ABI bool isMemDepEnabled() const; - LLVM_ABI bool isMemorySSAEnabled() const; - /// This class holds the mapping between values and value numbers. It is used /// as an efficient mechanism to determine the expression-wise equivalence of /// two values. @@ -211,6 +184,7 @@ class GVNPass : public OptionalPassInfoMixin { LLVM_ABI ~ValueTable(); LLVM_ABI ValueTable &operator=(const ValueTable &Arg); + LLVM_ABI void add(Value *V, uint32_t Num); LLVM_ABI uint32_t lookupOrAdd(MemoryAccess *MA); LLVM_ABI uint32_t lookupOrAdd(Value *V); LLVM_ABI uint32_t lookup(Value *V, bool Verify = true) const; @@ -223,7 +197,6 @@ class GVNPass : public OptionalPassInfoMixin { LLVM_ABI void eraseTranslateCacheEntry(uint32_t Num, const BasicBlock &CurrBlock); LLVM_ABI bool exists(Value *V) const; - LLVM_ABI void add(Value *V, uint32_t Num); LLVM_ABI void clear(); LLVM_ABI void erase(Value *V); void setAliasAnalysis(AAResults *A) { AA = A; } @@ -245,6 +218,7 @@ class GVNPass : public OptionalPassInfoMixin { friend class GVNLegacyPass; friend struct DenseMapInfo; + GVNOptions Options; MemoryDependenceResults *MD = nullptr; DominatorTree *DT = nullptr; const TargetLibraryInfo *TLI = nullptr; @@ -255,7 +229,6 @@ class GVNPass : public OptionalPassInfoMixin { LoopInfo *LI = nullptr; AAResults *AA = nullptr; MemorySSAUpdater *MSSAU = nullptr; - ValueTable VN; /// A mapping from value numbers to lists of Value*'s that @@ -347,18 +320,35 @@ class GVNPass : public OptionalPassInfoMixin { // of BlockRPONumber prior to accessing the contents of BlockRPONumber. bool InvalidBlockRPONumbers = true; + // List of critical edges to be split between iterations. + SmallVector, 4> ToSplit; + +public: + GVNPass(GVNOptions Options = {}) : Options(Options) {} + + /// Run the pass over the function. + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + LLVM_ABI void + printPipeline(raw_ostream &OS, + function_ref MapClassName2PassName); + +private: + DominatorTree &getDominatorTree() const { return *DT; } + AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); } + MemoryDependenceResults &getMemDep() const { return *MD; } + + bool isScalarPREEnabled() const; + bool isLoadPREEnabled() const; + bool isLoadInLoopPREEnabled() const; + bool isLoadPRESplitBackedgeEnabled() const; + bool isMemDepEnabled() const; + bool isMemorySSAEnabled() const; + using LoadDepVect = SmallVector; using AvailValInBlkVect = SmallVector; using UnavailBlkVect = SmallVector; - bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, - const TargetLibraryInfo &RunTLI, AAResults &RunAA, - MemoryDependenceResults *RunMD, LoopInfo &LI, - OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr); - - // List of critical edges to be split between iterations. - SmallVector, 4> ToSplit; - enum class DepKind { Other = 0, // Unknown value. Def, // Exactly overlapping locations. @@ -417,6 +407,41 @@ class GVNPass : public OptionalPassInfoMixin { using DependencyBlockSet = DenseMap; + /// Given a select-dependency for the load (the load address is a select of + /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a + /// value is available by finding dominating values for both addresses. If + /// so, the load can be rematerialized as a select of those two values. + std::optional + analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr, + Value *FalseAddr, Instruction *From); + + /// Given a local dependency (Def or Clobber) determine if a value is + /// available for the load. + std::optional + analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep, + Value *Address); + + /// Given a list of non-local dependencies, determine if a value is + /// available for the load in each specified block. If it is, add it to + /// ValuesPerBlock. If not, add it to UnavailableBlocks. + void analyzeLoadAvailability(LoadInst *Load, + SmallVectorImpl &Deps, + AvailValInBlkVect &ValuesPerBlock, + UnavailBlkVect &UnavailableBlocks); + + /// Given a critical edge from Pred to LoadBB, find a load instruction + /// which is identical to Load from another successor of Pred. + LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB, + LoadInst *Load); + + /// Eliminates partially redundant \p Load, replacing it with \p + /// AvailableLoads (connected by Phis if needed). + void eliminatePartiallyRedundantLoad( + LoadInst *Load, AvailValInBlkVect &ValuesPerBlock, + MapVector &AvailableLoads, + MapVector *CriticalEdgePredAndLoad); + + // Helper functions for d etermining load dependencies. std::optional scanMemoryAccessesUsers( const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB, const SmallVectorImpl &ClobbersList, MemorySSA &MSSA, @@ -439,40 +464,6 @@ class GVNPass : public OptionalPassInfoMixin { SmallVectorImpl &Values, MemorySSA &MSSA, AAResults &AA); - // Helper functions of redundant load elimination. - bool processLoad(LoadInst *L); - bool processMaskedLoad(IntrinsicInst *I); - bool processNonLocalLoad(LoadInst *L); - bool processNonLocalLoad(LoadInst *L, SmallVectorImpl &Deps); - bool processAssumeIntrinsic(AssumeInst *II); - - /// Given a local dependency (Def or Clobber) determine if a value is - /// available for the load. - std::optional - analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep, - Value *Address); - - /// Given a select-dependency for the load (the load address is a select of - /// \p TrueAddr and \p FalseAddr guarded by \p Cond), determine whether a - /// value is available by finding dominating values for both addresses. If - /// so, the load can be rematerialized as a select of those two values. - std::optional - analyzeSelectAvailability(LoadInst *Load, Value *Cond, Value *TrueAddr, - Value *FalseAddr, Instruction *From); - - /// Given a list of non-local dependencies, determine if a value is - /// available for the load in each specified block. If it is, add it to - /// ValuesPerBlock. If not, add it to UnavailableBlocks. - void analyzeLoadAvailability(LoadInst *Load, - SmallVectorImpl &Deps, - AvailValInBlkVect &ValuesPerBlock, - UnavailBlkVect &UnavailableBlocks); - - /// Given a critical edge from Pred to LoadBB, find a load instruction - /// which is identical to Load from another successor of Pred. - LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB, - LoadInst *Load); - bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock, UnavailBlkVect &UnavailableBlocks); @@ -482,31 +473,63 @@ class GVNPass : public OptionalPassInfoMixin { bool performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock, UnavailBlkVect &UnavailableBlocks); - /// Eliminates partially redundant \p Load, replacing it with \p - /// AvailableLoads (connected by Phis if needed). - void eliminatePartiallyRedundantLoad( - LoadInst *Load, AvailValInBlkVect &ValuesPerBlock, - MapVector &AvailableLoads, - MapVector *CriticalEdgePredAndLoad); + // Try to eliminate redundent loades with non-local dependencies. + bool processNonLocalLoad(LoadInst *L); + bool processNonLocalLoad(LoadInst *L, SmallVectorImpl &Deps); - // Other helper routines. + /// Add any blocks determined to be unreachable by a conditional branch with a + /// constant condition to the dead blocks. + bool processFoldableCondBr(CondBrInst *BI); + + /// Propagate equalities derived from llvm.assume intrinsics. + bool processAssumeIntrinsic(AssumeInst *II); + + /// Try to eliminate redundant loads. + bool processLoad(LoadInst *L); + + /// Try to eliminate masked loads which have loaded from + /// masked stores with the same mask. + bool processMaskedLoad(IntrinsicInst *I); + + /// Propagate value of a condition to blocks dominated by "then" and "else" + /// edges, as well as certains derived equalities. + bool + propagateEquality(Value *LHS, Value *RHS, + const std::variant &Root); + + // Pass iteration helper functions. bool processInstruction(Instruction *I); bool processBlock(BasicBlock *BB); bool iterateOnFunction(Function &F); - bool performPRE(Function &F); - bool performScalarPRE(Instruction *I); + + // Scalar PRE helper functions bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, BasicBlock *Curr, unsigned int ValNo); + bool performScalarPRE(Instruction *I); + bool performPRE(Function &F); + + /// Main entry point for the GVN pass. Also used by the GVNLegacyPass. + bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, + const TargetLibraryInfo &RunTLI, AAResults &RunAA, + MemoryDependenceResults *RunMD, LoopInfo &LI, + OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr); + + // Other helper routines. + Value *findLeader(const BasicBlock *BB, uint32_t Num); void cleanupGlobalSets(); + void removeInstruction(Instruction *I); + + /// This removes the specified instruction from + /// our various maps and marks it for deletion. + void salvageAndRemoveInstruction(Instruction *I); + void verifyRemoved(const Instruction *I) const; + bool splitCriticalEdges(); BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ); - bool - propagateEquality(Value *LHS, Value *RHS, - const std::variant &Root); - bool processFoldableCondBr(CondBrInst *BI); + void addDeadBlock(BasicBlock *BB); void assignValNumForDeadCode(); void assignBlockRPONumber(Function &F); diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index 42fd413423129..996ed2a72cdaf 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -289,6 +289,72 @@ struct llvm::GVNPass::AvailableValue { Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt) const; }; +Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load, + Instruction *InsertPt) const { + Value *Res; + Type *LoadTy = Load->getType(); + const DataLayout &DL = Load->getDataLayout(); + if (isSimpleValue()) { + Res = getSimpleValue(); + if (Res->getType() != LoadTy) { + Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction()); + + LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset + << " " << *getSimpleValue() << '\n' + << *Res << '\n' + << "\n\n\n"); + } + } else if (isCoercedLoadValue()) { + LoadInst *CoercedLoad = getCoercedLoadValue(); + if (CoercedLoad->getType() == LoadTy && Offset == 0) { + Res = CoercedLoad; + combineMetadataForCSE(CoercedLoad, Load, false); + } else { + Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt, + Load->getFunction()); + // We are adding a new user for this load, for which the original + // metadata may not hold. Additionally, the new load may have a different + // size and type, so their metadata cannot be combined in any + // straightforward way. + // Drop all metadata that is not known to cause immediate UB on violation, + // unless the load has !noundef, in which case all metadata violations + // will be promoted to UB. + // !noalias and !alias.scope are kept: the load is not moved and still + // accesses the same memory, and these are independent of the load type + // and offset, so they remain valid for the coerced result. + if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef)) + CoercedLoad->dropUnknownNonDebugMetadata( + {LLVMContext::MD_dereferenceable, + LLVMContext::MD_dereferenceable_or_null, + LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group, + LLVMContext::MD_alias_scope, LLVMContext::MD_noalias}); + LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset + << " " << *getCoercedLoadValue() << '\n' + << *Res << '\n' + << "\n\n\n"); + } + } else if (isMemIntrinValue()) { + Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy, InsertPt, + DL); + LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset + << " " << *getMemIntrinValue() << '\n' + << *Res << '\n' + << "\n\n\n"); + } else if (isSelectValue()) { + // Introduce a new value select for a load from an eligible pointer select. + Value *Cond = getSelectCondition(); + assert(V1 && V2 && "both value operands of the select must be present"); + Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator()); + // We use the DebugLoc from the original load here, as this instruction + // materializes the value that would previously have been loaded. + cast(Res)->setDebugLoc(Load->getDebugLoc()); + } else { + llvm_unreachable("Should not materialize value from dead block"); + } + assert(Res && "failed to materialize?"); + return Res; +} + /// Represents an AvailableValue which can be rematerialized at the end of /// the associated BasicBlock. struct llvm::GVNPass::AvailableValueInBlock { @@ -452,37 +518,6 @@ GVNPass::Expression GVNPass::ValueTable::createGEPExpr(GetElementPtrInst *GEP) { return E; } -//===----------------------------------------------------------------------===// -// ValueTable External Functions -//===----------------------------------------------------------------------===// - -GVNPass::ValueTable::ValueTable() = default; -GVNPass::ValueTable::ValueTable(const ValueTable &) = default; -GVNPass::ValueTable::ValueTable(ValueTable &&) = default; -GVNPass::ValueTable::~ValueTable() = default; -GVNPass::ValueTable & -GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default; - -/// add - Insert a value into the table with a specified value number. -void GVNPass::ValueTable::add(Value *V, uint32_t Num) { - ValueNumbering.insert(std::make_pair(V, Num)); - if (PHINode *PN = dyn_cast(V)) - NumberingPhi[Num] = PN; -} - -/// Include the incoming memory state into the hash of the expression for the -/// given instruction. If the incoming memory state is: -/// * LiveOnEntry, add the value number of the entry block, -/// * a MemoryPhi, add the value number of the basic block corresponding to that -/// MemoryPhi, -/// * a MemoryDef, add the value number of the memory setting instruction. -void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) { - assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA"); - assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory"); - MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I); - Exp.VarArgs.push_back(lookupOrAdd(MA)); -} - uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) { // FIXME: Currently the calls which may access the thread id may // be considered as not accessing the memory. But this is @@ -634,133 +669,326 @@ uint32_t GVNPass::ValueTable::computeLoadStoreVN(Instruction *I) { return V; } -/// Returns true if a value number exists for the specified value. -bool GVNPass::ValueTable::exists(Value *V) const { - return ValueNumbering.contains(V); -} +/// Translate value number \p Num using phis, so that it has the values of +/// the phis in BB. +uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred, + const BasicBlock *PhiBlock, + uint32_t Num, GVNPass &GVN) { + // See if we can refine the value number by looking at the PN incoming value + // for the given predecessor. + if (PHINode *PN = NumberingPhi[Num]) { + if (PN->getParent() != PhiBlock) + return Num; + for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) { + if (PN->getIncomingBlock(I) != Pred) + continue; + if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false)) + return TransVal; + } + return Num; + } -uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) { - return MSSA->isLiveOnEntryDef(MA) || isa(MA) - ? lookupOrAdd(MA->getBlock()) - : lookupOrAdd(cast(MA)->getMemoryInst()); -} + if (BasicBlock *BB = NumberingBB[Num]) { + assert(MSSA && "NumberingBB is non-empty only when using MemorySSA"); + // Value numbers of basic blocks are used to represent memory state in + // load/store instructions and read-only function calls when said state is + // set by a MemoryPhi. + if (BB != PhiBlock) + return Num; + MemoryPhi *MPhi = MSSA->getMemoryAccess(BB); + for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) { + if (MPhi->getIncomingBlock(i) != Pred) + continue; + MemoryAccess *MA = MPhi->getIncomingValue(i); + if (auto *PredPhi = dyn_cast(MA)) + return lookupOrAdd(PredPhi->getBlock()); + if (MSSA->isLiveOnEntryDef(MA)) + return lookupOrAdd(&BB->getParent()->getEntryBlock()); + return lookupOrAdd(cast(MA)->getMemoryInst()); + } + llvm_unreachable( + "CFG/MemorySSA mismatch: predecessor not found among incoming blocks"); + } -/// lookupOrAdd - Returns the value number for the specified value, assigning -/// it a new number if it did not have one before. -uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) { - auto VI = ValueNumbering.find(V); - if (VI != ValueNumbering.end()) - return VI->second; + // If there is any value related with Num is defined in a BB other than + // PhiBlock, it cannot depend on a phi in PhiBlock without going through + // a backedge. We can do an early exit in that case to save compile time. + if (!areAllValsInBB(Num, PhiBlock, GVN)) + return Num; - auto *I = dyn_cast(V); - if (!I) { - ValueNumbering[V] = NextValueNumber; - if (isa(V)) - NumberingBB[NextValueNumber] = cast(V); - return NextValueNumber++; + if (Num >= ExprIdx.size() || ExprIdx[Num] == 0) + return Num; + Expression Exp = Expressions[ExprIdx[Num]]; + + for (unsigned I = 0; I < Exp.VarArgs.size(); I++) { + // For InsertValue and ExtractValue, some varargs are index numbers + // instead of value numbers. Those index numbers should not be + // translated. + if ((I > 1 && Exp.Opcode == Instruction::InsertValue) || + (I > 0 && Exp.Opcode == Instruction::ExtractValue) || + (I > 1 && Exp.Opcode == Instruction::ShuffleVector)) + continue; + Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN); } - Expression Exp; - switch (I->getOpcode()) { - case Instruction::Call: - return lookupOrAddCall(cast(I)); - case Instruction::FNeg: - case Instruction::Add: - case Instruction::FAdd: - case Instruction::Sub: - case Instruction::FSub: - case Instruction::Mul: - case Instruction::FMul: - case Instruction::UDiv: - case Instruction::SDiv: - case Instruction::FDiv: - case Instruction::URem: - case Instruction::SRem: - case Instruction::FRem: - case Instruction::Shl: - case Instruction::LShr: - case Instruction::AShr: - case Instruction::And: - case Instruction::Or: - case Instruction::Xor: - case Instruction::ICmp: - case Instruction::FCmp: - case Instruction::Trunc: - case Instruction::ZExt: - case Instruction::SExt: - case Instruction::FPToUI: - case Instruction::FPToSI: - case Instruction::UIToFP: - case Instruction::SIToFP: - case Instruction::FPTrunc: - case Instruction::FPExt: - case Instruction::PtrToInt: - case Instruction::PtrToAddr: - case Instruction::IntToPtr: - case Instruction::AddrSpaceCast: - case Instruction::BitCast: - case Instruction::Select: - case Instruction::Freeze: - case Instruction::ExtractElement: - case Instruction::InsertElement: - case Instruction::ShuffleVector: - case Instruction::InsertValue: - Exp = createExpr(I); - break; - case Instruction::GetElementPtr: - Exp = createGEPExpr(cast(I)); - break; - case Instruction::ExtractValue: - Exp = createExtractvalueExpr(cast(I)); - break; - case Instruction::PHI: - ValueNumbering[V] = NextValueNumber; - NumberingPhi[NextValueNumber] = cast(V); - return NextValueNumber++; - case Instruction::Load: - case Instruction::Store: - return computeLoadStoreVN(I); - default: - ValueNumbering[V] = NextValueNumber; - return NextValueNumber++; + if (Exp.Commutative) { + assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!"); + if (Exp.VarArgs[0] > Exp.VarArgs[1]) { + std::swap(Exp.VarArgs[0], Exp.VarArgs[1]); + uint32_t Opcode = Exp.Opcode >> 8; + if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) + Exp.Opcode = (Opcode << 8) | + CmpInst::getSwappedPredicate( + static_cast(Exp.Opcode & 255)); + } } - uint32_t E = assignExpNewValueNum(Exp).first; - ValueNumbering[V] = E; - return E; + if (uint32_t NewNum = ExpressionNumbering[Exp]) { + if (Exp.Opcode == Instruction::Call && NewNum != Num) + return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num; + return NewNum; + } + return Num; } -/// Returns the value number of the specified value. Fails if -/// the value has not yet been numbered. -uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const { - auto VI = ValueNumbering.find(V); - if (Verify) { - assert(VI != ValueNumbering.end() && "Value not numbered?"); - return VI->second; +// Return true if the value number \p Num and NewNum have equal value. +// Return false if the result is unknown. +bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum, + const BasicBlock *Pred, + const BasicBlock *PhiBlock, + GVNPass &GVN) { + CallInst *Call = nullptr; + auto Leaders = GVN.LeaderTable.getLeaders(Num); + for (const auto &Entry : Leaders) { + Call = dyn_cast(&*Entry.Val); + if (Call && Call->getParent() == PhiBlock) + break; } - return (VI != ValueNumbering.end()) ? VI->second : 0; -} -/// Returns the value number of the given comparison, -/// assigning it a new number if it did not have one before. Useful when -/// we deduced the result of a comparison, but don't immediately have an -/// instruction realizing that comparison to hand. -uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode, - CmpInst::Predicate Predicate, - Value *LHS, Value *RHS) { - Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS); - return assignExpNewValueNum(Exp).first; -} + if (AA->doesNotAccessMemory(Call)) + return true; -/// Returns the value number of ptrtoint \p Ptr to \Ty. -uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) { - Expression Exp(Instruction::PtrToInt); - Exp.Ty = Ty; - Exp.VarArgs.push_back(lookupOrAdd(Ptr)); - return ExpressionNumbering.lookup(Exp); + if (!MD || !AA->onlyReadsMemory(Call)) + return false; + + MemDepResult LocalDep = MD->getDependency(Call); + if (!LocalDep.isNonLocal()) + return false; + + const MemoryDependenceResults::NonLocalDepInfo &Deps = + MD->getNonLocalCallDependency(Call); + + // Check to see if the Call has no function local clobber. + for (const NonLocalDepEntry &D : Deps) { + if (D.getResult().isNonFuncLocal()) + return true; + } + return false; } -/// Remove all entries from the ValueTable. +/// Return a pair the first field showing the value number of \p Exp and the +/// second field showing whether it is a value number newly created. +std::pair +GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) { + uint32_t &E = ExpressionNumbering[Exp]; + bool CreateNewValNum = !E; + if (CreateNewValNum) { + Expressions.push_back(Exp); + if (ExprIdx.size() < NextValueNumber + 1) + ExprIdx.resize(NextValueNumber * 2); + E = NextValueNumber; + ExprIdx[NextValueNumber++] = NextExprNumber++; + } + return {E, CreateNewValNum}; +} + +/// Return whether all the values related with the same \p num are +/// defined in \p BB. +bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB, + GVNPass &GVN) { + return all_of( + GVN.LeaderTable.getLeaders(Num), + [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; }); +} + +/// Include the incoming memory state into the hash of the expression for the +/// given instruction. If the incoming memory state is: +/// * LiveOnEntry, add the value number of the entry block, +/// * a MemoryPhi, add the value number of the basic block corresponding to that +/// MemoryPhi, +/// * a MemoryDef, add the value number of the memory setting instruction. +void GVNPass::ValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) { + assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA"); + assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory"); + MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I); + Exp.VarArgs.push_back(lookupOrAdd(MA)); +} + +//===----------------------------------------------------------------------===// +// ValueTable External Functions +//===----------------------------------------------------------------------===// + +GVNPass::ValueTable::ValueTable() = default; +GVNPass::ValueTable::ValueTable(const ValueTable &) = default; +GVNPass::ValueTable::ValueTable(ValueTable &&) = default; +GVNPass::ValueTable::~ValueTable() = default; +GVNPass::ValueTable & +GVNPass::ValueTable::operator=(const GVNPass::ValueTable &Arg) = default; + +/// add - Insert a value into the table with a specified value number. +void GVNPass::ValueTable::add(Value *V, uint32_t Num) { + ValueNumbering.insert(std::make_pair(V, Num)); + if (PHINode *PN = dyn_cast(V)) + NumberingPhi[Num] = PN; +} + +uint32_t GVNPass::ValueTable::lookupOrAdd(MemoryAccess *MA) { + return MSSA->isLiveOnEntryDef(MA) || isa(MA) + ? lookupOrAdd(MA->getBlock()) + : lookupOrAdd(cast(MA)->getMemoryInst()); +} + +/// lookupOrAdd - Returns the value number for the specified value, assigning +/// it a new number if it did not have one before. +uint32_t GVNPass::ValueTable::lookupOrAdd(Value *V) { + auto VI = ValueNumbering.find(V); + if (VI != ValueNumbering.end()) + return VI->second; + + auto *I = dyn_cast(V); + if (!I) { + ValueNumbering[V] = NextValueNumber; + if (isa(V)) + NumberingBB[NextValueNumber] = cast(V); + return NextValueNumber++; + } + + Expression Exp; + switch (I->getOpcode()) { + case Instruction::Call: + return lookupOrAddCall(cast(I)); + case Instruction::FNeg: + case Instruction::Add: + case Instruction::FAdd: + case Instruction::Sub: + case Instruction::FSub: + case Instruction::Mul: + case Instruction::FMul: + case Instruction::UDiv: + case Instruction::SDiv: + case Instruction::FDiv: + case Instruction::URem: + case Instruction::SRem: + case Instruction::FRem: + case Instruction::Shl: + case Instruction::LShr: + case Instruction::AShr: + case Instruction::And: + case Instruction::Or: + case Instruction::Xor: + case Instruction::ICmp: + case Instruction::FCmp: + case Instruction::Trunc: + case Instruction::ZExt: + case Instruction::SExt: + case Instruction::FPToUI: + case Instruction::FPToSI: + case Instruction::UIToFP: + case Instruction::SIToFP: + case Instruction::FPTrunc: + case Instruction::FPExt: + case Instruction::PtrToInt: + case Instruction::PtrToAddr: + case Instruction::IntToPtr: + case Instruction::AddrSpaceCast: + case Instruction::BitCast: + case Instruction::Select: + case Instruction::Freeze: + case Instruction::ExtractElement: + case Instruction::InsertElement: + case Instruction::ShuffleVector: + case Instruction::InsertValue: + Exp = createExpr(I); + break; + case Instruction::GetElementPtr: + Exp = createGEPExpr(cast(I)); + break; + case Instruction::ExtractValue: + Exp = createExtractvalueExpr(cast(I)); + break; + case Instruction::PHI: + ValueNumbering[V] = NextValueNumber; + NumberingPhi[NextValueNumber] = cast(V); + return NextValueNumber++; + case Instruction::Load: + case Instruction::Store: + return computeLoadStoreVN(I); + default: + ValueNumbering[V] = NextValueNumber; + return NextValueNumber++; + } + + uint32_t E = assignExpNewValueNum(Exp).first; + ValueNumbering[V] = E; + return E; +} + +/// Returns the value number of the specified value. Fails if +/// the value has not yet been numbered. +uint32_t GVNPass::ValueTable::lookup(Value *V, bool Verify) const { + auto VI = ValueNumbering.find(V); + if (Verify) { + assert(VI != ValueNumbering.end() && "Value not numbered?"); + return VI->second; + } + return (VI != ValueNumbering.end()) ? VI->second : 0; +} + +/// Returns the value number of the given comparison, +/// assigning it a new number if it did not have one before. Useful when +/// we deduced the result of a comparison, but don't immediately have an +/// instruction realizing that comparison to hand. +uint32_t GVNPass::ValueTable::lookupOrAddCmp(unsigned Opcode, + CmpInst::Predicate Predicate, + Value *LHS, Value *RHS) { + Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS); + return assignExpNewValueNum(Exp).first; +} + +/// Returns the value number of ptrtoint \p Ptr to \Ty. +uint32_t GVNPass::ValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) { + Expression Exp(Instruction::PtrToInt); + Exp.Ty = Ty; + Exp.VarArgs.push_back(lookupOrAdd(Ptr)); + return ExpressionNumbering.lookup(Exp); +} + +/// Wrap phiTranslateImpl to provide caching functionality. +uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred, + const BasicBlock *PhiBlock, + uint32_t Num, GVNPass &GVN) { + auto FindRes = PhiTranslateTable.find({Num, Pred}); + if (FindRes != PhiTranslateTable.end()) + return FindRes->second; + uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN); + PhiTranslateTable.insert({{Num, Pred}, NewNum}); + return NewNum; +} + +/// Erase stale entry from phiTranslate cache so phiTranslate can be computed +/// again. +void GVNPass::ValueTable::eraseTranslateCacheEntry( + uint32_t Num, const BasicBlock &CurrBlock) { + for (const BasicBlock *Pred : predecessors(&CurrBlock)) + PhiTranslateTable.erase({Num, Pred}); +} + +/// Returns true if a value number exists for the specified value. +bool GVNPass::ValueTable::exists(Value *V) const { + return ValueNumbering.contains(V); +} + +/// Remove all entries from the ValueTable. void GVNPass::ValueTable::clear() { ValueNumbering.clear(); ExpressionNumbering.clear(); @@ -851,31 +1079,6 @@ void GVNPass::LeaderMap::erase(uint32_t N, Instruction *I, // GVN Pass //===----------------------------------------------------------------------===// -bool GVNPass::isScalarPREEnabled() const { - return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE); -} - -bool GVNPass::isLoadPREEnabled() const { - return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE); -} - -bool GVNPass::isLoadInLoopPREEnabled() const { - return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE); -} - -bool GVNPass::isLoadPRESplitBackedgeEnabled() const { - return Options.AllowLoadPRESplitBackedge.value_or( - GVNEnableSplitBackedgeInLoadPRE); -} - -bool GVNPass::isMemDepEnabled() const { - return Options.AllowMemDep.value_or(GVNEnableMemDep); -} - -bool GVNPass::isMemorySSAEnabled() const { - return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA); -} - PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) { // FIXME: The order of evaluation of these 'getResult' calls is very // significant! Re-ordering these variables will cause GVN when run alone to @@ -928,10 +1131,29 @@ void GVNPass::printPipeline( OS << '>'; } -void GVNPass::salvageAndRemoveInstruction(Instruction *I) { - salvageKnowledge(I, AC); - salvageDebugInfo(*I); - removeInstruction(I); +bool GVNPass::isScalarPREEnabled() const { + return Options.AllowScalarPRE.value_or(GVNEnableScalarPRE); +} + +bool GVNPass::isLoadPREEnabled() const { + return Options.AllowLoadPRE.value_or(GVNEnableLoadPRE); +} + +bool GVNPass::isLoadInLoopPREEnabled() const { + return Options.AllowLoadInLoopPRE.value_or(GVNEnableLoadInLoopPRE); +} + +bool GVNPass::isLoadPRESplitBackedgeEnabled() const { + return Options.AllowLoadPRESplitBackedge.value_or( + GVNEnableSplitBackedgeInLoadPRE); +} + +bool GVNPass::isMemDepEnabled() const { + return Options.AllowMemDep.value_or(GVNEnableMemDep); +} + +bool GVNPass::isMemorySSAEnabled() const { + return Options.AllowMemorySSA.value_or(GVNEnableMemorySSA); } enum class AvailabilityState : char { @@ -1091,12 +1313,11 @@ static void replaceValuesPerBlockEntry( static Value * constructSSAForLoadSet(LoadInst *Load, SmallVectorImpl &ValuesPerBlock, - GVNPass &GVN) { + DominatorTree &DT) { // Check for the fully redundant, dominating load case. In this case, we can // just use the dominating value directly. if (ValuesPerBlock.size() == 1 && - GVN.getDominatorTree().properlyDominates(ValuesPerBlock[0].BB, - Load->getParent())) { + DT.properlyDominates(ValuesPerBlock[0].BB, Load->getParent())) { assert(!ValuesPerBlock[0].AV.isUndefValue() && "Dead BB dominate this block"); return ValuesPerBlock[0].MaterializeAdjustedValue(Load); @@ -1132,110 +1353,44 @@ constructSSAForLoadSet(LoadInst *Load, return SSAUpdate.GetValueInMiddleOfBlock(Load->getParent()); } -Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load, - Instruction *InsertPt) const { - Value *Res; - Type *LoadTy = Load->getType(); - const DataLayout &DL = Load->getDataLayout(); - if (isSimpleValue()) { - Res = getSimpleValue(); - if (Res->getType() != LoadTy) { - Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction()); +static bool isLifetimeStart(const Instruction *Inst) { + if (const IntrinsicInst *II = dyn_cast(Inst)) + return II->getIntrinsicID() == Intrinsic::lifetime_start; + return false; +} - LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset - << " " << *getSimpleValue() << '\n' - << *Res << '\n' - << "\n\n\n"); - } - } else if (isCoercedLoadValue()) { - LoadInst *CoercedLoad = getCoercedLoadValue(); - if (CoercedLoad->getType() == LoadTy && Offset == 0) { - Res = CoercedLoad; - combineMetadataForCSE(CoercedLoad, Load, false); - } else { - Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt, - Load->getFunction()); - // We are adding a new user for this load, for which the original - // metadata may not hold. Additionally, the new load may have a different - // size and type, so their metadata cannot be combined in any - // straightforward way. - // Drop all metadata that is not known to cause immediate UB on violation, - // unless the load has !noundef, in which case all metadata violations - // will be promoted to UB. - // !noalias and !alias.scope are kept: the load is not moved and still - // accesses the same memory, and these are independent of the load type - // and offset, so they remain valid for the coerced result. - if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef)) - CoercedLoad->dropUnknownNonDebugMetadata( - {LLVMContext::MD_dereferenceable, - LLVMContext::MD_dereferenceable_or_null, - LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group, - LLVMContext::MD_alias_scope, LLVMContext::MD_noalias}); - LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset - << " " << *getCoercedLoadValue() << '\n' - << *Res << '\n' - << "\n\n\n"); - } - } else if (isMemIntrinValue()) { - Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy, - InsertPt, DL); - LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset - << " " << *getMemIntrinValue() << '\n' - << *Res << '\n' - << "\n\n\n"); - } else if (isSelectValue()) { - // Introduce a new value select for a load from an eligible pointer select. - Value *Cond = getSelectCondition(); - assert(V1 && V2 && "both value operands of the select must be present"); - Res = SelectInst::Create(Cond, V1, V2, "", InsertPt->getIterator()); - // We use the DebugLoc from the original load here, as this instruction - // materializes the value that would previously have been loaded. - cast(Res)->setDebugLoc(Load->getDebugLoc()); - } else { - llvm_unreachable("Should not materialize value from dead block"); - } - assert(Res && "failed to materialize?"); - return Res; -} - -static bool isLifetimeStart(const Instruction *Inst) { - if (const IntrinsicInst* II = dyn_cast(Inst)) - return II->getIntrinsicID() == Intrinsic::lifetime_start; - return false; -} - -/// Assuming To can be reached from both From and Between, does Between lie on -/// every path from From to To? -static bool liesBetween(const Instruction *From, Instruction *Between, - const Instruction *To, const DominatorTree *DT) { - if (From->getParent() == Between->getParent()) - return DT->dominates(From, Between); - SmallPtrSet Exclusion; - Exclusion.insert(Between->getParent()); - return !isPotentiallyReachable(From, To, &Exclusion, DT); -} - -static const Instruction *findMayClobberedPtrAccess(LoadInst *Load, - const DominatorTree *DT) { - Value *PtrOp = Load->getPointerOperand(); - if (!PtrOp->hasUseList()) - return nullptr; - - Instruction *OtherAccess = nullptr; - - for (auto *U : PtrOp->users()) { - if (U != Load && (isa(U) || isa(U))) { - auto *I = cast(U); - if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) { - // Use the most immediately dominating value. - if (OtherAccess) { - if (DT->dominates(OtherAccess, I)) - OtherAccess = I; - else - assert(U == OtherAccess || DT->dominates(I, OtherAccess)); - } else - OtherAccess = I; - } +/// Assuming To can be reached from both From and Between, does Between lie on +/// every path from From to To? +static bool liesBetween(const Instruction *From, Instruction *Between, + const Instruction *To, const DominatorTree *DT) { + if (From->getParent() == Between->getParent()) + return DT->dominates(From, Between); + SmallPtrSet Exclusion; + Exclusion.insert(Between->getParent()); + return !isPotentiallyReachable(From, To, &Exclusion, DT); +} + +static const Instruction *findMayClobberedPtrAccess(LoadInst *Load, + const DominatorTree *DT) { + Value *PtrOp = Load->getPointerOperand(); + if (!PtrOp->hasUseList()) + return nullptr; + + Instruction *OtherAccess = nullptr; + + for (auto *U : PtrOp->users()) { + if (U != Load && (isa(U) || isa(U))) { + auto *I = cast(U); + if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) { + // Use the most immediately dominating value. + if (OtherAccess) { + if (DT->dominates(OtherAccess, I)) + OtherAccess = I; + else + assert(U == OtherAccess || DT->dominates(I, OtherAccess)); + } else + OtherAccess = I; + } } } @@ -1682,7 +1837,7 @@ void GVNPass::eliminatePartiallyRedundantLoad( } // Perform PHI construction. - Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this); + Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree()); // constructSSAForLoadSet is responsible for combining metadata. ICF->removeUsersOf(Load); Load->replaceAllUsesWith(V); @@ -1699,1076 +1854,1010 @@ void GVNPass::eliminatePartiallyRedundantLoad( salvageAndRemoveInstruction(Load); } -bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock, - UnavailBlkVect &UnavailableBlocks) { - // Okay, we have *some* definitions of the value. This means that the value - // is available in some of our (transitive) predecessors. Lets think about - // doing PRE of this load. This will involve inserting a new load into the - // predecessor when it's not available. We could do this in general, but - // prefer to not increase code size. As such, we only do this when we know - // that we only have to insert *one* load (which means we're basically moving - // the load, not inserting a new one). - - SmallPtrSet Blockers(llvm::from_range, UnavailableBlocks); +static void reportLoadElim(LoadInst *Load, Value *AvailableValue, + OptimizationRemarkEmitter *ORE) { + using namespace ore; - // Let's find the first basic block with more than one predecessor. Walk - // backwards through predecessors if needed. - BasicBlock *LoadBB = Load->getParent(); - BasicBlock *TmpBB = LoadBB; + ORE->emit([&]() { + return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load) + << "load of type " << NV("Type", Load->getType()) << " eliminated" + << setExtraArgs() << " in favor of " + << NV("InfavorOfValue", AvailableValue); + }); +} - // Check that there is no implicit control flow instructions above our load in - // its block. If there is an instruction that doesn't always pass the - // execution to the following instruction, then moving through it may become - // invalid. For example: - // - // int arr[LEN]; - // int index = ???; - // ... - // guard(0 <= index && index < LEN); - // use(arr[index]); - // - // It is illegal to move the array access to any point above the guard, - // because if the index is out of bounds we should deoptimize rather than - // access the array. - // Check that there is no guard in this block above our instruction. - bool MustEnsureSafetyOfSpeculativeExecution = - ICF->isDominatedByICFIFromSameBlock(Load); +/// If a load has !invariant.group, try to find the most-dominating instruction +/// with the same metadata and equivalent pointer (modulo bitcasts and zero +/// GEPs). If one is found that dominates the load, its value can be reused. +static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) { + Value *PointerOperand = L->getPointerOperand()->stripPointerCasts(); - while (TmpBB->getSinglePredecessor()) { - TmpBB = TmpBB->getSinglePredecessor(); - if (TmpBB == LoadBB) // Infinite (unreachable) loop. - return false; - if (Blockers.count(TmpBB)) - return false; + // It's not safe to walk the use list of a global value because function + // passes aren't allowed to look outside their functions. + // FIXME: this could be fixed by filtering instructions from outside of + // current function. + if (isa(PointerOperand)) + return nullptr; - // If any of these blocks has more than one successor (i.e. if the edge we - // just traversed was critical), then there are other paths through this - // block along which the load may not be anticipated. Hoisting the load - // above this block would be adding the load to execution paths along - // which it was not previously executed. - if (TmpBB->getTerminator()->getNumSuccessors() != 1) - return false; + // Queue to process all pointers that are equivalent to load operand. + SmallVector PointerUsesQueue; + PointerUsesQueue.push_back(PointerOperand); - // Check that there is no implicit control flow in a block above. - MustEnsureSafetyOfSpeculativeExecution = - MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB); - } + Instruction *MostDominatingInstruction = L; - assert(TmpBB); - LoadBB = TmpBB; + // FIXME: This loop is potentially O(n^2) due to repeated dominates checks. + while (!PointerUsesQueue.empty()) { + Value *Ptr = PointerUsesQueue.pop_back_val(); + assert(Ptr && !isa(Ptr) && + "Null or GlobalValue should not be inserted"); - // Check to see how many predecessors have the loaded value fully - // available. - MapVector PredLoads; - DenseMap FullyAvailableBlocks; - for (const AvailableValueInBlock &AV : ValuesPerBlock) - FullyAvailableBlocks[AV.BB] = AvailabilityState::Available; - for (BasicBlock *UnavailableBB : UnavailableBlocks) - FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable; + for (User *U : Ptr->users()) { + auto *I = dyn_cast(U); + if (!I || I == L || !DT.dominates(I, MostDominatingInstruction)) + continue; - // The edge from Pred to LoadBB is a critical edge will be splitted. - SmallVector CriticalEdgePredSplit; - // The edge from Pred to LoadBB is a critical edge, another successor of Pred - // contains a load can be moved to Pred. This data structure maps the Pred to - // the movable load. - MapVector CriticalEdgePredAndLoad; - for (BasicBlock *Pred : predecessors(LoadBB)) { - // If any predecessor block is an EH pad that does not allow non-PHI - // instructions before the terminator, we can't PRE the load. - if (Pred->getTerminator()->isEHPad()) { - LLVM_DEBUG( - dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '" - << Pred->getName() << "': " << *Load << '\n'); - return false; - } + // Add bitcasts and zero GEPs to queue. + // TODO: Should drop bitcast? + if (isa(I) || + (isa(I) && + cast(I)->hasAllZeroIndices())) { + PointerUsesQueue.push_back(I); + continue; + } - if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) { - continue; + // If we hit a load/store with an invariant.group metadata and the same + // pointer operand, we can assume that value pointed to by the pointer + // operand didn't change. + if (I->hasMetadata(LLVMContext::MD_invariant_group) && + Ptr == getLoadStorePointerOperand(I) && !I->isVolatile()) + MostDominatingInstruction = I; } + } - if (Pred->getTerminator()->getNumSuccessors() != 1) { - if (isa(Pred->getTerminator())) { - LLVM_DEBUG( - dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '" - << Pred->getName() << "': " << *Load << '\n'); - return false; - } - - if (LoadBB->isEHPad()) { - LLVM_DEBUG( - dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '" - << Pred->getName() << "': " << *Load << '\n'); - return false; - } + return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr; +} - // Do not split backedge as it will break the canonical loop form. - if (!isLoadPRESplitBackedgeEnabled()) - if (DT->dominates(LoadBB, Pred)) { - LLVM_DEBUG( - dbgs() - << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '" - << Pred->getName() << "': " << *Load << '\n'); - return false; - } +/// Return the memory location accessed by the (masked) load/store instruction +/// `I`, if the instruction could potentially provide a useful value for +/// eliminating the load. +static std::optional +maybeLoadStoreLocation(Instruction *I, bool AllowStores, + const TargetLibraryInfo *TLI) { + if (auto *LI = dyn_cast(I)) + return MemoryLocation::get(LI); - if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load)) - CriticalEdgePredAndLoad[Pred] = LI; - else - CriticalEdgePredSplit.push_back(Pred); - } else { - // Only add the predecessors that will not be split for now. - PredLoads[Pred] = nullptr; + if (auto *II = dyn_cast(I)) { + switch (II->getIntrinsicID()) { + case Intrinsic::masked_load: + return MemoryLocation::getForArgument(II, 0, TLI); + case Intrinsic::masked_store: + if (AllowStores) + return MemoryLocation::getForArgument(II, 1, TLI); + return std::nullopt; + default: + break; } } - // Decide whether PRE is profitable for this load. - unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size(); - unsigned NumUnavailablePreds = NumInsertPreds + - CriticalEdgePredAndLoad.size(); - assert(NumUnavailablePreds != 0 && - "Fully available value should already be eliminated!"); - (void)NumUnavailablePreds; - - // If we need to insert new load in multiple predecessors, reject it. - // FIXME: If we could restructure the CFG, we could make a common pred with - // all the preds that don't have an available Load and insert a new load into - // that one block. - if (NumInsertPreds > 1) - return false; - - // Now we know where we will insert load. We must ensure that it is safe - // to speculatively execute the load at that points. - if (MustEnsureSafetyOfSpeculativeExecution) { - if (CriticalEdgePredSplit.size()) - if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC, - DT)) - return false; - for (auto &PL : PredLoads) - if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC, - DT)) - return false; - for (auto &CEP : CriticalEdgePredAndLoad) - if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC, - DT)) - return false; - } - - // Split critical edges, and update the unavailable predecessors accordingly. - for (BasicBlock *OrigPred : CriticalEdgePredSplit) { - BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB); - assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!"); - PredLoads[NewPred] = nullptr; - LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->" - << LoadBB->getName() << '\n'); - } - - for (auto &CEP : CriticalEdgePredAndLoad) - PredLoads[CEP.first] = nullptr; + if (!AllowStores) + return std::nullopt; - // Check if the load can safely be moved to all the unavailable predecessors. - bool CanDoPRE = true; - const DataLayout &DL = Load->getDataLayout(); - SmallVector NewInsts; - for (auto &PredLoad : PredLoads) { - BasicBlock *UnavailablePred = PredLoad.first; + if (auto *SI = dyn_cast(I)) + return MemoryLocation::get(SI); + return std::nullopt; +} - // Do PHI translation to get its value in the predecessor if necessary. The - // returned pointer (if non-null) is guaranteed to dominate UnavailablePred. - // We do the translation for each edge we skipped by going from Load's block - // to LoadBB, otherwise we might miss pieces needing translation. +/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`, +/// looking for memory reads whose location aliases `Loc` and dominates our +/// load. +std::optional GVNPass::scanMemoryAccessesUsers( + const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB, + const SmallVectorImpl &ClobbersList, MemorySSA &MSSA, + BatchAAResults &AA, LoadInst *L) { - // If all preds have a single successor, then we know it is safe to insert - // the load on the pred (?!?), so we can insert code to materialize the - // pointer if it is not available. - Value *LoadPtr = Load->getPointerOperand(); - BasicBlock *Cur = Load->getParent(); - while (Cur != LoadBB) { - PHITransAddr Address(LoadPtr, DL, AC); - LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(), - *DT, NewInsts); - if (!LoadPtr) { - CanDoPRE = false; - break; - } - Cur = Cur->getSinglePredecessor(); + // Prefer a candidate that is closer to the load within the same block. + auto UpdateChoice = [&](std::optional &Choice, + AliasResult &AR, Instruction *Candidate) { + if (!Choice) { + if (AR == AliasResult::PartialAlias) + Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset()); + else + Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate); + return; } + if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst), + MSSA.getMemoryAccess(Candidate))) + return; - if (LoadPtr) { - PHITransAddr Address(LoadPtr, DL, AC); - LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT, - NewInsts); - } - // If we couldn't find or insert a computation of this phi translated value, - // we fail PRE. - if (!LoadPtr) { - LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: " - << *Load->getPointerOperand() << "\n"); - CanDoPRE = false; - break; + if (AR == AliasResult::PartialAlias) { + Choice->Kind = DepKind::Clobber; + Choice->Offset = AR.getOffset(); + } else { + Choice->Kind = DepKind::Def; + Choice->Offset = -1; } - PredLoad.second = LoadPtr; - } + Choice->Inst = Candidate; + Choice->Block = Candidate->getParent(); + }; - if (!CanDoPRE) { - while (!NewInsts.empty()) { - // Erase instructions generated by the failed PHI translation before - // trying to number them. PHI translation might insert instructions - // in basic blocks other than the current one, and we delete them - // directly, as salvageAndRemoveInstruction only allows removing from the - // current basic block. - NewInsts.pop_back_val()->eraseFromParent(); - } - // HINT: Don't revert the edge-splitting as following transformation may - // also need to split these critical edges. - return !CriticalEdgePredSplit.empty(); - } + std::optional ReachingVal; + for (MemoryAccess *MA : ClobbersList) { + unsigned Scanned = 0; + for (User *U : MA->users()) { + if (++Scanned >= ScanUsersLimit) + return ReachingMemVal::getUnknown(BB, Loc.Ptr); - // Okay, we can eliminate this load by inserting a reload in the predecessor - // and using PHI construction to get the value in the other predecessors, do - // it. - LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n'); - LLVM_DEBUG(if (!NewInsts.empty()) dbgs() << "INSERTED " << NewInsts.size() - << " INSTS: " << *NewInsts.back() - << '\n'); + auto *UseOrDef = dyn_cast(U); + if (!UseOrDef || UseOrDef->getBlock() != BB) + continue; - // Assign value numbers to the new instructions. - for (Instruction *I : NewInsts) { - // Instructions that have been inserted in predecessor(s) to materialize - // the load address do not retain their original debug locations. Doing - // so could lead to confusing (but correct) source attributions. - I->updateLocationAfterHoist(); + Instruction *MemI = UseOrDef->getMemoryInst(); + if (MemI == L || + (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L)))) + continue; - // FIXME: We really _ought_ to insert these value numbers into their - // parent's availability map. However, in doing so, we risk getting into - // ordering issues. If a block hasn't been processed yet, we would be - // marking a value as AVAIL-IN, which isn't what we intend. - VN.lookupOrAdd(I); + if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) { + AliasResult AR = AA.alias(*MaybeLoc, Loc); + // If the locations do not certainly alias, we cannot possibly infer the + // following load loads the same value. + if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias) + continue; + + // Locations partially overlap, but neither is a subset of the other, or + // the second location is before the first. + if (AR == AliasResult::PartialAlias && + (!AR.hasOffset() || AR.getOffset() < 0)) + continue; + + // Found candidate, the new load memory location and the given location + // must alias: precise overlap, or subset with non-negative offset. + UpdateChoice(ReachingVal, AR, MemI); + } + } + if (ReachingVal) + break; } - eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads, - &CriticalEdgePredAndLoad); - ++NumPRELoad; - return true; + return ReachingVal; } -bool GVNPass::performLoopLoadPRE(LoadInst *Load, - AvailValInBlkVect &ValuesPerBlock, - UnavailBlkVect &UnavailableBlocks) { - const Loop *L = LI->getLoopFor(Load->getParent()); - // TODO: Generalize to other loop blocks that dominate the latch. - if (!L || L->getHeader() != Load->getParent()) - return false; - - BasicBlock *Preheader = L->getLoopPreheader(); - BasicBlock *Latch = L->getLoopLatch(); - if (!Preheader || !Latch) - return false; +/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a +/// given location. Returns a ReachingMemVal describing the dependency. +std::optional GVNPass::accessMayModifyLocation( + MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad, + BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) { + assert(ClobberMA->getBlock() == BB); - Value *LoadPtr = Load->getPointerOperand(); - // Must be available in preheader. - if (!L->isLoopInvariant(LoadPtr)) - return false; + // If the clobbering access is the entry memory state, we cannot say anything + // about the content of the memory, except when we are accessing a local + // object, which can be turned later into producing `undef`. + if (MSSA.isLiveOnEntryDef(ClobberMA)) { + if (auto *Alloc = dyn_cast(getUnderlyingObject(Loc.Ptr))) + if (Alloc->getParent() == BB) + return ReachingMemVal::getDef(Loc.Ptr, const_cast(Alloc)); + return ReachingMemVal::getUnknown(BB, Loc.Ptr); + } - // We plan to hoist the load to preheader without introducing a new fault. - // In order to do it, we need to prove that we cannot side-exit the loop - // once loop header is first entered before execution of the load. - if (ICF->isDominatedByICFIFromSameBlock(Load)) - return false; + // Loads from "constant" memory can't be clobbered. + if (IsInvariantLoad || AA.pointsToConstantMemory(Loc)) + return std::nullopt; - BasicBlock *LoopBlock = nullptr; - for (auto *Blocker : UnavailableBlocks) { - // Blockers from outside the loop are handled in preheader. - if (!L->contains(Blocker)) - continue; + auto GetOrdering = [](const Instruction *I) { + if (auto *L = dyn_cast(I)) + return L->getOrdering(); + return cast(I)->getOrdering(); + }; + Instruction *ClobberI = cast(ClobberMA)->getMemoryInst(); - // Only allow one loop block. Loop header is not less frequently executed - // than each loop block, and likely it is much more frequently executed. But - // in case of multiple loop blocks, we need extra information (such as block - // frequency info) to understand whether it is profitable to PRE into - // multiple loop blocks. - if (LoopBlock) - return false; + // Check if the clobbering access is a load or a store that we can reuse. + if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) { + AliasResult AR = AA.alias(*MaybeLoc, Loc); + if (AR == AliasResult::MustAlias) + return ReachingMemVal::getDef(Loc.Ptr, ClobberI); - // Do not sink into inner loops. This may be non-profitable. - if (L != LI->getLoopFor(Blocker)) - return false; + if (AR == AliasResult::NoAlias) { + // If the locations do not alias we may still be able to skip over the + // clobbering instruction, even if it is atomic. + // The original load is either non-atomic or unordered. We can reorder + // these across non-atomic, unordered or monotonic loads or across any + // store. + if (!ClobberI->isAtomic() || + !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) || + isa(ClobberI)) + return std::nullopt; + return ReachingMemVal::getClobber(Loc.Ptr, ClobberI); + } - // Blocks that dominate the latch execute on every single iteration, maybe - // except the last one. So PREing into these blocks doesn't make much sense - // in most cases. But the blocks that do not necessarily execute on each - // iteration are sometimes much colder than the header, and this is when - // PRE is potentially profitable. - if (DT->dominates(Blocker, Latch)) - return false; + // Skip over volatile loads (the original load is non-volatile, non-atomic). + if (!ClobberI->isAtomic() && isa(ClobberI)) + return std::nullopt; - // Make sure that the terminator itself doesn't clobber. - if (Blocker->getTerminator()->mayWriteToMemory()) - return false; + if (AR == AliasResult::MayAlias || + (AR == AliasResult::PartialAlias && + (!AR.hasOffset() || AR.getOffset() < 0))) + return ReachingMemVal::getClobber(Loc.Ptr, ClobberI); - LoopBlock = Blocker; + // The only option left is a store of the superset of the required bits. + assert(AR == AliasResult::PartialAlias && AR.hasOffset() && + AR.getOffset() > 0 && + "Must be the superset/partial overlap case with positive offset"); + return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset()); } - if (!LoopBlock) - return false; - - // Make sure the memory at this pointer cannot be freed, therefore we can - // safely reload from it after clobber. - if (LoadPtr->canBeFreed()) - return false; + if (auto *II = dyn_cast(ClobberI)) { + if (isa(II)) + return std::nullopt; + if (II->getIntrinsicID() == Intrinsic::lifetime_start) { + MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI); + if (AA.isMustAlias(IIObjLoc, Loc)) + return ReachingMemVal::getDef(Loc.Ptr, ClobberI); + return std::nullopt; + } + } - // TODO: Support critical edge splitting if blocker has more than 1 successor. - MapVector AvailableLoads; - AvailableLoads[LoopBlock] = LoadPtr; - AvailableLoads[Preheader] = LoadPtr; + // If we are at a malloc-like function call, we can turn the load into `undef` + // or zero. + if (isNoAliasCall(ClobberI)) { + const Value *Obj = getUnderlyingObject(Loc.Ptr); + if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr)) + return ReachingMemVal::getDef(Loc.Ptr, ClobberI); + } - LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n'); - eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads, - /*CriticalEdgePredAndLoad*/ nullptr); - ++NumPRELoopLoad; - return true; -} + // Can reorder loads across a release fence. + if (auto *FI = dyn_cast(ClobberI)) + if (FI->getOrdering() == AtomicOrdering::Release) + return std::nullopt; -static void reportLoadElim(LoadInst *Load, Value *AvailableValue, - OptimizationRemarkEmitter *ORE) { - using namespace ore; + // See if the clobber instruction (e.g., a generic call) may modify the + // location. + ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc); + // If may modify the location, analyze deeper, to exclude accesses to + // non-escaping local allocations. + if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref) + return std::nullopt; - ORE->emit([&]() { - return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load) - << "load of type " << NV("Type", Load->getType()) << " eliminated" - << setExtraArgs() << " in favor of " - << NV("InfavorOfValue", AvailableValue); - }); + // Conservatively assume the clobbering memory access may overwrite the + // location. + return ReachingMemVal::getClobber(Loc.Ptr, ClobberI); } -/// Attempt to eliminate a load whose dependencies are -/// non-local by performing PHI construction. -bool GVNPass::processNonLocalLoad(LoadInst *Load) { - // Non-local speculations are not allowed under asan. - if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) || - Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress)) +/// Collect the predecessors of block, while doing phi-translation of the memory +/// address and the memory clobber. Return false if the block should be marked +/// as clobbering the memory location in an unknown way. +bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr, + MemoryAccess *ClobberMA, + DependencyBlockSet &Blocks, + SmallVectorImpl &Worklist) { + if (Addr.needsPHITranslationFromBlock(BB) && + !Addr.isPotentiallyPHITranslatable()) return false; - // Find the non-local dependencies of the load. - LoadDepVect Deps; - MD->getNonLocalPointerDependency(Load, Deps); + auto *MPhi = + ClobberMA->getBlock() == BB ? dyn_cast(ClobberMA) : nullptr; + SmallVector, 8> Preds; + for (BasicBlock *Pred : predecessors(BB)) { + // Skip unreachable predecessors. + if (!DT->isReachableFromEntry(Pred)) + continue; - // If we had to process more than one hundred blocks to find the - // dependencies, this load isn't worth worrying about. Optimizing - // it will be too expensive. - unsigned NumDeps = Deps.size(); - if (NumDeps > MaxNumDeps) - return false; + // Skip already visited predecessors. + if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; })) + continue; - SmallVector MemVals; - MemVals.reserve(Deps.size()); + PHITransAddr TransAddr = Addr; + if (TransAddr.needsPHITranslationFromBlock(BB)) + TransAddr.translateValue(BB, Pred, DT, false); - for (const NonLocalDepResult &Dep : Deps) { - const auto &R = Dep.getResult(); - SelectAddr SelAddr = Dep.getAddress(); - BasicBlock *BB = Dep.getBB(); - Instruction *Inst = R.getInst(); - if (R.isSelect()) { - auto [Cond, Addrs] = SelAddr.getSelectCondAndAddrs(); - MemVals.emplace_back( - ReachingMemVal::getSelect(BB, Cond, Addrs.first, Addrs.second)); + auto It = Blocks.find(Pred); + if (It != Blocks.end()) { + // If we reach a visited block with a different address, set the + // current block as clobbering the memory location in an unknown way + // (by returning false). + if (It->second.Addr.getAddr() != TransAddr.getAddr()) + return false; + // Otherwise, just stop the traversal. continue; } - Value *Address = SelAddr.getAddr(); - if (R.isClobber()) - MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst)); - else if (R.isDef()) - MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst)); - else - MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst)); - } - return processNonLocalLoad(Load, MemVals); -} - -bool GVNPass::processNonLocalLoad(LoadInst *Load, - SmallVectorImpl &Deps) { - // If we had a phi translation failure, we'll have a single entry which is a - // clobber in the current block. Reject this early. - if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) { - LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs()); - dbgs() << " has unknown dependencies\n";); - return false; + Preds.emplace_back( + Pred, DependencyBlockInfo(TransAddr, + MPhi ? MPhi->getIncomingValueForBlock(Pred) + : ClobberMA)); } - bool Changed = false; - // This is a limited form of scalar PRE for load indices. If this load follows - // a GEP, see if we can PRE the indices before analyzing. - if (isScalarPREEnabled()) { - if (GetElementPtrInst *GEP = - dyn_cast(Load->getOperand(0))) { - for (Use &U : GEP->indices()) - if (Instruction *I = dyn_cast(U.get())) - Changed |= performScalarPRE(I); - } + // We collected the predecessors and stored them in Preds. Now, populate the + // worklist with the predecessors found, and cache the eventual translated + // address for each block. + for (auto &P : Preds) { + [[maybe_unused]] auto It = + Blocks.try_emplace(P.first, std::move(P.second)).first; + Worklist.push_back(P.first); } - // Step 1: Analyze the availability of the load. - AvailValInBlkVect ValuesPerBlock; - UnavailBlkVect UnavailableBlocks; - analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks); - - // If we have no predecessors that produce a known value for this load, exit - // early. - if (ValuesPerBlock.empty()) - return Changed; + return true; +} - // Step 2: Eliminate fully redundancy. - // - // If all of the instructions we depend on produce a known value for this - // load, then it is fully redundant and we can use PHI insertion to compute - // its value. Insert PHIs and remove the fully redundant value now. - if (UnavailableBlocks.empty()) { - LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n'); +/// Build a list of MemoryAccesses whose users could potentially alias the +/// memory location being queried. Starts from StartInfo's initial clobber, +/// walk the use-def chain to the final clobber. If the chain extends beyond +/// `BB`, continue into that block but only if it is in the previously collected +/// set. +void GVNPass::collectClobberList(SmallVectorImpl &Clobbers, + BasicBlock *BB, + const DependencyBlockInfo &StartInfo, + const DependencyBlockSet &Blocks, + MemorySSA &MSSA) { + MemoryAccess *MA = StartInfo.InitialClobberMA; + MemoryAccess *LastMA = StartInfo.ClobberMA; - // Perform PHI construction. - Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, *this); - // constructSSAForLoadSet is responsible for combining metadata. - ICF->removeUsersOf(Load); - Load->replaceAllUsesWith(V); + for (;;) { + while (MA != LastMA) { + Clobbers.push_back(MA); + MA = cast(MA)->getDefiningAccess(); + } + Clobbers.push_back(MA); - if (isa(V)) - V->takeName(Load); - if (Instruction *I = dyn_cast(V)) - // If instruction I has debug info, then we should not update it. - // Also, if I has a null DebugLoc, then it is still potentially incorrect - // to propagate Load's DebugLoc because Load may not post-dominate I. - if (Load->getDebugLoc() && Load->getParent() == I->getParent()) - I->setDebugLoc(Load->getDebugLoc()); - if (MD && V->getType()->isPtrOrPtrVectorTy()) - MD->invalidateCachedPointerInfo(V); - ++NumGVNLoad; - reportLoadElim(Load, V, ORE); - salvageAndRemoveInstruction(Load); - return true; - } + if (MSSA.isLiveOnEntryDef(MA) || + (MA->getBlock() == BB && !isa(MA))) + break; - // Step 3: Eliminate partial redundancy. - if (!isLoadPREEnabled()) - return Changed; - if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent())) - return Changed; + // If the final clobber in the current block is a MemoryPhi, go to the + // immediate dominator; otherwise, just get to the block containing the + // final clobber. + if (MA->getBlock() == BB) + BB = DT->getNode(BB)->getIDom()->getBlock(); + else + BB = MA->getBlock(); - if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) || - performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks)) - return true; + auto It = Blocks.find(BB); + if (It == Blocks.end()) + break; - return Changed; + MA = It->second.InitialClobberMA; + LastMA = It->second.ClobberMA; + if (MA == Clobbers.back()) + Clobbers.pop_back(); + } } -bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) { - Value *V = IntrinsicI->getArgOperand(0); - - if (ConstantInt *Cond = dyn_cast(V)) { - if (Cond->isZero()) { - Type *Int8Ty = Type::getInt8Ty(V->getContext()); - Type *PtrTy = PointerType::get(V->getContext(), 0); - // Insert a new store to null instruction before the load to indicate that - // this code is not reachable. FIXME: We could insert unreachable - // instruction directly because we can modify the CFG. - auto *NewS = - new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy), - IntrinsicI->getIterator()); - if (MSSAU) { - const MemoryUseOrDef *FirstNonDom = nullptr; - const auto *AL = - MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent()); - - // If there are accesses in the current basic block, find the first one - // that does not come before NewS. The new memory access is inserted - // after the found access or before the terminator if no such access is - // found. - if (AL) { - for (const auto &Acc : *AL) { - if (auto *Current = dyn_cast(&Acc)) - if (!Current->getMemoryInst()->comesBefore(NewS)) { - FirstNonDom = Current; - break; - } - } - } - - auto *NewDef = - FirstNonDom ? MSSAU->createMemoryAccessBefore( - NewS, nullptr, - const_cast(FirstNonDom)) - : MSSAU->createMemoryAccessInBB( - NewS, nullptr, - NewS->getParent(), MemorySSA::BeforeTerminator); +/// Entrypoint for the MemorySSA-based redundant load elimination algorithm. +/// Given as input a load instruction, the function computes the set of reaching +/// memory values, one per predecessor path, that analyzeLoadAvailability can +/// later use to establish whether the load may be eliminated. A reaching value +/// may be of the following descriptor kind: +/// * Def: a precise instruction that produces the exact bits the load would +/// read (e.g., an equivalent load or a MustAlias store); +/// * Clobber: a write that clobbers a superset of the bits the load would read +/// (e.g., a memset over a larger region); +/// * Other: we know which block defines the memory location in some way, but +/// could not identify a precise instruction (e.g., memory already live at +/// function entry). +bool GVNPass::findReachingValuesForLoad(LoadInst *L, + SmallVectorImpl &Values, + MemorySSA &MSSA, AAResults &AAR) { + EarliestEscapeAnalysis EA(*DT, LI); + BatchAAResults AA(AAR, &EA); + BasicBlock *StartBlock = L->getParent(); + bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load); + // TODO: Simplify later work by just getClobberingMemoryAccess(). + MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess(); + const MemoryLocation Loc = MemoryLocation::get(L); - MSSAU->insertDef(cast(NewDef), /*RenameUses=*/false); - } - } - if (isAssumeWithEmptyBundle(*IntrinsicI)) { - salvageAndRemoveInstruction(IntrinsicI); + // Fast path for load tagged with !invariant.group. + if (L->hasMetadata(LLVMContext::MD_invariant_group)) { + if (Instruction *G = findInvariantGroupValue(L, *DT)) { + Values.emplace_back( + ReachingMemVal::getDef(getLoadStorePointerOperand(G), G)); return true; } - return false; - } - - if (isa(V)) { - // If it's not false, and constant, it must evaluate to true. This means our - // assume is assume(true), and thus, pointless, and we don't want to do - // anything more here. - return false; } - Constant *True = ConstantInt::getTrue(V->getContext()); - return propagateEquality(V, True, IntrinsicI); -} + // Phase 1. First off, look for a local dependency to avoid having to + // disambiguate between before the load and after the load of the starting + // block (as the load may be visited from a backedge). + do { + // Scan users of the clobbering memory access. + if (auto RMV = scanMemoryAccessesUsers( + Loc, IsInvariantLoad, StartBlock, + SmallVector{ClobberMA}, MSSA, AA, L)) { + Values.emplace_back(*RMV); + return true; + } -static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) { - patchReplacementInstruction(I, Repl); - I->replaceAllUsesWith(Repl); -} + // Exit from here, and proceed visiting predecessors if the clobbering + // access is non-local or is a MemoryPhi. + if (ClobberMA->getBlock() != StartBlock || isa(ClobberMA)) + break; -/// If a load has !invariant.group, try to find the most-dominating instruction -/// with the same metadata and equivalent pointer (modulo bitcasts and zero -/// GEPs). If one is found that dominates the load, its value can be reused. -static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) { - Value *PointerOperand = L->getPointerOperand()->stripPointerCasts(); + // Check if the clobber actually aliases the load location. + if (auto RMV = accessMayModifyLocation(ClobberMA, Loc, IsInvariantLoad, + StartBlock, MSSA, AA)) { + Values.emplace_back(*RMV); + return true; + } - // It's not safe to walk the use list of a global value because function - // passes aren't allowed to look outside their functions. - // FIXME: this could be fixed by filtering instructions from outside of - // current function. - if (isa(PointerOperand)) - return nullptr; + // It may happen that the clobbering memory access does not actually + // clobber our load location, transition to its defining memory access. + ClobberMA = cast(ClobberMA)->getDefiningAccess(); + } while (ClobberMA->getBlock() == StartBlock); - // Queue to process all pointers that are equivalent to load operand. - SmallVector PointerUsesQueue; - PointerUsesQueue.push_back(PointerOperand); + // Non-local speculations are not allowed under ASan. + if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) || + L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress)) + return false; - Instruction *MostDominatingInstruction = L; + // Phase 2. Walk backwards through the CFG, collecting all the blocks that + // contain an instruction that modifies the load memory location, or that lie + // on a path between a clobbering block and our load. Start off by collecting + // the predecessors of `StartBlock`. All the visited blocks are stored in a + // the set `Blocks`. If possible, the memory address maintained for the block + // visited does get phi-translated. + DependencyBlockSet Blocks; + SmallVector InitialWorklist; + const DataLayout &DL = L->getModule()->getDataLayout(); + if (!collectPredecessors(StartBlock, + PHITransAddr(L->getPointerOperand(), DL, AC), + ClobberMA, Blocks, InitialWorklist)) + return false; - // FIXME: This loop is potentially O(n^2) due to repeated dominates checks. - while (!PointerUsesQueue.empty()) { - Value *Ptr = PointerUsesQueue.pop_back_val(); - assert(Ptr && !isa(Ptr) && - "Null or GlobalValue should not be inserted"); + // Do a bottom-up DFS. + auto Worklist = InitialWorklist; + while (!Worklist.empty()) { + auto *BB = Worklist.pop_back_val(); + DependencyBlockInfo &Info = Blocks.find(BB)->second; - for (User *U : Ptr->users()) { - auto *I = dyn_cast(U); - if (!I || I == L || !DT.dominates(I, MostDominatingInstruction)) - continue; + // Phi-translation may have failed. + if (!Info.Addr.getAddr()) + continue; - // Add bitcasts and zero GEPs to queue. - // TODO: Should drop bitcast? - if (isa(I) || - (isa(I) && - cast(I)->hasAllZeroIndices())) { - PointerUsesQueue.push_back(I); + // If the clobbering memory access is in the current block and it indeed + // clobbers our load location, record the dependency and do not visit the + // predecessors of this block further, continue with the blocks in the + // worklist. + if (Info.ClobberMA->getBlock() == BB && !isa(Info.ClobberMA)) { + if (auto RMV = accessMayModifyLocation( + Info.ClobberMA, Loc.getWithNewPtr(Info.Addr.getAddr()), + IsInvariantLoad, BB, MSSA, AA)) { + Info.MemVal = RMV; continue; } + assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) && + "LiveOnEntry aliases everything"); - // If we hit a load/store with an invariant.group metadata and the same - // pointer operand, we can assume that value pointed to by the pointer - // operand didn't change. - if (I->hasMetadata(LLVMContext::MD_invariant_group) && - Ptr == getLoadStorePointerOperand(I) && !I->isVolatile()) - MostDominatingInstruction = I; + // If, however, the clobbering memory access does not actually clobber + // our load location, transition to its defining memory access, but + // keep examining the same basic block. + Info.ClobberMA = + cast(Info.ClobberMA)->getDefiningAccess(); + Worklist.emplace_back(BB); + continue; } - } - - return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr; -} - -/// Return the memory location accessed by the (masked) load/store instruction -/// `I`, if the instruction could potentially provide a useful value for -/// eliminating the load. -static std::optional -maybeLoadStoreLocation(Instruction *I, bool AllowStores, - const TargetLibraryInfo *TLI) { - if (auto *LI = dyn_cast(I)) - return MemoryLocation::get(LI); - if (auto *II = dyn_cast(I)) { - switch (II->getIntrinsicID()) { - case Intrinsic::masked_load: - return MemoryLocation::getForArgument(II, 0, TLI); - case Intrinsic::masked_store: - if (AllowStores) - return MemoryLocation::getForArgument(II, 1, TLI); - return std::nullopt; - default: - break; + // At this point we know the current block is "transparent", i.e. the memory + // location is not modified when execution goes through this block. + // Continue to its predecessors, unless a predecessor has already been + // visited with a different address. We currently cannot represent such a + // dependency. + if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) { + Info.ForceUnknown = true; + continue; } + if (BB != StartBlock && + !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist)) + Info.ForceUnknown = true; } - if (!AllowStores) - return std::nullopt; - - if (auto *SI = dyn_cast(I)) - return MemoryLocation::get(SI); - return std::nullopt; -} + // Phase 3. We have collected all the blocks that either write a value to the + // memory location of the load, or there exists a path to the load, along + // which the memory location is not modified. Perform a second DFS to find + // load-to-load dependencies; namely, look at the dominating memory reads, + // that alias our load. These are the MemoryUses that are users of the + // MemoryDefs we previously identified. If no memory read is encountered, + // either confirm the clobbering write found before or set to unknown. + Worklist = InitialWorklist; + for (BasicBlock *BB : Worklist) { + DependencyBlockInfo &Info = Blocks.find(BB)->second; + Info.Visited = true; + } -/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`, -/// looking for memory reads whose location aliases `Loc` and dominates our -/// load. -std::optional GVNPass::scanMemoryAccessesUsers( - const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB, - const SmallVectorImpl &ClobbersList, MemorySSA &MSSA, - BatchAAResults &AA, LoadInst *L) { + SmallVector Clobbers; + while (!Worklist.empty()) { + auto *BB = Worklist.pop_back_val(); + DependencyBlockInfo &Info = Blocks.find(BB)->second; - // Prefer a candidate that is closer to the load within the same block. - auto UpdateChoice = [&](std::optional &Choice, - AliasResult &AR, Instruction *Candidate) { - if (!Choice) { - if (AR == AliasResult::PartialAlias) - Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset()); - else - Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate); - return; + // If phi-translation failed, assume the memory location is modified in + // unknown way. + if (!Info.Addr.getAddr()) { + Values.push_back(ReachingMemVal::getUnknown(BB, nullptr)); + continue; } - if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst), - MSSA.getMemoryAccess(Candidate))) - return; - if (AR == AliasResult::PartialAlias) { - Choice->Kind = DepKind::Clobber; - Choice->Offset = AR.getOffset(); - } else { - Choice->Kind = DepKind::Def; - Choice->Offset = -1; + Clobbers.clear(); + collectClobberList(Clobbers, BB, Info, Blocks, MSSA); + if (auto RMV = + scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()), + IsInvariantLoad, BB, Clobbers, MSSA, AA)) { + Values.push_back(*RMV); + continue; } - Choice->Inst = Candidate; - Choice->Block = Candidate->getParent(); - }; + // If no reusable memory use was found, and the current block is not + // transparent, use the already established memory def. + if (Info.MemVal) { + Values.push_back(*Info.MemVal); + continue; + } - std::optional ReachingVal; - for (MemoryAccess *MA : ClobbersList) { - unsigned Scanned = 0; - for (User *U : MA->users()) { - if (++Scanned >= ScanUsersLimit) - return ReachingMemVal::getUnknown(BB, Loc.Ptr); + if (Info.ForceUnknown) { + Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr())); + continue; + } - auto *UseOrDef = dyn_cast(U); - if (!UseOrDef || UseOrDef->getBlock() != BB) + // If the current block is transparent, continue to its predecessors. + for (BasicBlock *Pred : predecessors(BB)) { + auto It = Blocks.find(Pred); + if (It == Blocks.end()) continue; - - Instruction *MemI = UseOrDef->getMemoryInst(); - if (MemI == L || - (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L)))) + DependencyBlockInfo &PredInfo = It->second; + if (PredInfo.Visited) continue; + PredInfo.Visited = true; + Worklist.push_back(Pred); + } + } - if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) { - AliasResult AR = AA.alias(*MaybeLoc, Loc); - // If the locations do not certainly alias, we cannot possibly infer the - // following load loads the same value. - if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias) - continue; + return true; +} - // Locations partially overlap, but neither is a subset of the other, or - // the second location is before the first. - if (AR == AliasResult::PartialAlias && - (!AR.hasOffset() || AR.getOffset() < 0)) - continue; +bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock, + UnavailBlkVect &UnavailableBlocks) { + // Okay, we have *some* definitions of the value. This means that the value + // is available in some of our (transitive) predecessors. Lets think about + // doing PRE of this load. This will involve inserting a new load into the + // predecessor when it's not available. We could do this in general, but + // prefer to not increase code size. As such, we only do this when we know + // that we only have to insert *one* load (which means we're basically moving + // the load, not inserting a new one). - // Found candidate, the new load memory location and the given location - // must alias: precise overlap, or subset with non-negative offset. - UpdateChoice(ReachingVal, AR, MemI); - } - } - if (ReachingVal) - break; - } + SmallPtrSet Blockers(llvm::from_range, UnavailableBlocks); - return ReachingVal; -} + // Let's find the first basic block with more than one predecessor. Walk + // backwards through predecessors if needed. + BasicBlock *LoadBB = Load->getParent(); + BasicBlock *TmpBB = LoadBB; -/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a -/// given location. Returns a ReachingMemVal describing the dependency. -std::optional GVNPass::accessMayModifyLocation( - MemoryAccess *ClobberMA, const MemoryLocation &Loc, bool IsInvariantLoad, - BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) { - assert(ClobberMA->getBlock() == BB); + // Check that there is no implicit control flow instructions above our load in + // its block. If there is an instruction that doesn't always pass the + // execution to the following instruction, then moving through it may become + // invalid. For example: + // + // int arr[LEN]; + // int index = ???; + // ... + // guard(0 <= index && index < LEN); + // use(arr[index]); + // + // It is illegal to move the array access to any point above the guard, + // because if the index is out of bounds we should deoptimize rather than + // access the array. + // Check that there is no guard in this block above our instruction. + bool MustEnsureSafetyOfSpeculativeExecution = + ICF->isDominatedByICFIFromSameBlock(Load); - // If the clobbering access is the entry memory state, we cannot say anything - // about the content of the memory, except when we are accessing a local - // object, which can be turned later into producing `undef`. - if (MSSA.isLiveOnEntryDef(ClobberMA)) { - if (auto *Alloc = dyn_cast(getUnderlyingObject(Loc.Ptr))) - if (Alloc->getParent() == BB) - return ReachingMemVal::getDef(Loc.Ptr, const_cast(Alloc)); - return ReachingMemVal::getUnknown(BB, Loc.Ptr); + while (TmpBB->getSinglePredecessor()) { + TmpBB = TmpBB->getSinglePredecessor(); + if (TmpBB == LoadBB) // Infinite (unreachable) loop. + return false; + if (Blockers.count(TmpBB)) + return false; + + // If any of these blocks has more than one successor (i.e. if the edge we + // just traversed was critical), then there are other paths through this + // block along which the load may not be anticipated. Hoisting the load + // above this block would be adding the load to execution paths along + // which it was not previously executed. + if (TmpBB->getTerminator()->getNumSuccessors() != 1) + return false; + + // Check that there is no implicit control flow in a block above. + MustEnsureSafetyOfSpeculativeExecution = + MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB); } - // Loads from "constant" memory can't be clobbered. - if (IsInvariantLoad || AA.pointsToConstantMemory(Loc)) - return std::nullopt; + assert(TmpBB); + LoadBB = TmpBB; - auto GetOrdering = [](const Instruction *I) { - if (auto *L = dyn_cast(I)) - return L->getOrdering(); - return cast(I)->getOrdering(); - }; - Instruction *ClobberI = cast(ClobberMA)->getMemoryInst(); + // Check to see how many predecessors have the loaded value fully + // available. + MapVector PredLoads; + DenseMap FullyAvailableBlocks; + for (const AvailableValueInBlock &AV : ValuesPerBlock) + FullyAvailableBlocks[AV.BB] = AvailabilityState::Available; + for (BasicBlock *UnavailableBB : UnavailableBlocks) + FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable; - // Check if the clobbering access is a load or a store that we can reuse. - if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) { - AliasResult AR = AA.alias(*MaybeLoc, Loc); - if (AR == AliasResult::MustAlias) - return ReachingMemVal::getDef(Loc.Ptr, ClobberI); + // The edge from Pred to LoadBB is a critical edge will be splitted. + SmallVector CriticalEdgePredSplit; + // The edge from Pred to LoadBB is a critical edge, another successor of Pred + // contains a load can be moved to Pred. This data structure maps the Pred to + // the movable load. + MapVector CriticalEdgePredAndLoad; + for (BasicBlock *Pred : predecessors(LoadBB)) { + // If any predecessor block is an EH pad that does not allow non-PHI + // instructions before the terminator, we can't PRE the load. + if (Pred->getTerminator()->isEHPad()) { + LLVM_DEBUG( + dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '" + << Pred->getName() << "': " << *Load << '\n'); + return false; + } - if (AR == AliasResult::NoAlias) { - // If the locations do not alias we may still be able to skip over the - // clobbering instruction, even if it is atomic. - // The original load is either non-atomic or unordered. We can reorder - // these across non-atomic, unordered or monotonic loads or across any - // store. - if (!ClobberI->isAtomic() || - !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) || - isa(ClobberI)) - return std::nullopt; - return ReachingMemVal::getClobber(Loc.Ptr, ClobberI); + if (isValueFullyAvailableInBlock(Pred, FullyAvailableBlocks)) { + continue; } - // Skip over volatile loads (the original load is non-volatile, non-atomic). - if (!ClobberI->isAtomic() && isa(ClobberI)) - return std::nullopt; + if (Pred->getTerminator()->getNumSuccessors() != 1) { + if (isa(Pred->getTerminator())) { + LLVM_DEBUG( + dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '" + << Pred->getName() << "': " << *Load << '\n'); + return false; + } - if (AR == AliasResult::MayAlias || - (AR == AliasResult::PartialAlias && - (!AR.hasOffset() || AR.getOffset() < 0))) - return ReachingMemVal::getClobber(Loc.Ptr, ClobberI); + if (LoadBB->isEHPad()) { + LLVM_DEBUG( + dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '" + << Pred->getName() << "': " << *Load << '\n'); + return false; + } - // The only option left is a store of the superset of the required bits. - assert(AR == AliasResult::PartialAlias && AR.hasOffset() && - AR.getOffset() > 0 && - "Must be the superset/partial overlap case with positive offset"); - return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset()); - } + // Do not split backedge as it will break the canonical loop form. + if (!isLoadPRESplitBackedgeEnabled()) + if (DT->dominates(LoadBB, Pred)) { + LLVM_DEBUG( + dbgs() + << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '" + << Pred->getName() << "': " << *Load << '\n'); + return false; + } - if (auto *II = dyn_cast(ClobberI)) { - if (isa(II)) - return std::nullopt; - if (II->getIntrinsicID() == Intrinsic::lifetime_start) { - MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI); - if (AA.isMustAlias(IIObjLoc, Loc)) - return ReachingMemVal::getDef(Loc.Ptr, ClobberI); - return std::nullopt; + if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load)) + CriticalEdgePredAndLoad[Pred] = LI; + else + CriticalEdgePredSplit.push_back(Pred); + } else { + // Only add the predecessors that will not be split for now. + PredLoads[Pred] = nullptr; } } - // If we are at a malloc-like function call, we can turn the load into `undef` - // or zero. - if (isNoAliasCall(ClobberI)) { - const Value *Obj = getUnderlyingObject(Loc.Ptr); - if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr)) - return ReachingMemVal::getDef(Loc.Ptr, ClobberI); + // Decide whether PRE is profitable for this load. + unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size(); + unsigned NumUnavailablePreds = + NumInsertPreds + CriticalEdgePredAndLoad.size(); + assert(NumUnavailablePreds != 0 && + "Fully available value should already be eliminated!"); + (void)NumUnavailablePreds; + + // If we need to insert new load in multiple predecessors, reject it. + // FIXME: If we could restructure the CFG, we could make a common pred with + // all the preds that don't have an available Load and insert a new load into + // that one block. + if (NumInsertPreds > 1) + return false; + + // Now we know where we will insert load. We must ensure that it is safe + // to speculatively execute the load at that points. + if (MustEnsureSafetyOfSpeculativeExecution) { + if (CriticalEdgePredSplit.size()) + if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC, + DT)) + return false; + for (auto &PL : PredLoads) + if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC, + DT)) + return false; + for (auto &CEP : CriticalEdgePredAndLoad) + if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC, + DT)) + return false; } - // Can reorder loads across a release fence. - if (auto *FI = dyn_cast(ClobberI)) - if (FI->getOrdering() == AtomicOrdering::Release) - return std::nullopt; + // Split critical edges, and update the unavailable predecessors accordingly. + for (BasicBlock *OrigPred : CriticalEdgePredSplit) { + BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB); + assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!"); + PredLoads[NewPred] = nullptr; + LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->" + << LoadBB->getName() << '\n'); + } - // See if the clobber instruction (e.g., a generic call) may modify the - // location. - ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc); - // If may modify the location, analyze deeper, to exclude accesses to - // non-escaping local allocations. - if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref) - return std::nullopt; + for (auto &CEP : CriticalEdgePredAndLoad) + PredLoads[CEP.first] = nullptr; - // Conservatively assume the clobbering memory access may overwrite the - // location. - return ReachingMemVal::getClobber(Loc.Ptr, ClobberI); -} + // Check if the load can safely be moved to all the unavailable predecessors. + bool CanDoPRE = true; + const DataLayout &DL = Load->getDataLayout(); + SmallVector NewInsts; + for (auto &PredLoad : PredLoads) { + BasicBlock *UnavailablePred = PredLoad.first; -/// Collect the predecessors of block, while doing phi-translation of the memory -/// address and the memory clobber. Return false if the block should be marked -/// as clobbering the memory location in an unknown way. -bool GVNPass::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr, - MemoryAccess *ClobberMA, - DependencyBlockSet &Blocks, - SmallVectorImpl &Worklist) { - if (Addr.needsPHITranslationFromBlock(BB) && - !Addr.isPotentiallyPHITranslatable()) - return false; + // Do PHI translation to get its value in the predecessor if necessary. The + // returned pointer (if non-null) is guaranteed to dominate UnavailablePred. + // We do the translation for each edge we skipped by going from Load's block + // to LoadBB, otherwise we might miss pieces needing translation. - auto *MPhi = - ClobberMA->getBlock() == BB ? dyn_cast(ClobberMA) : nullptr; - SmallVector, 8> Preds; - for (BasicBlock *Pred : predecessors(BB)) { - // Skip unreachable predecessors. - if (!DT->isReachableFromEntry(Pred)) - continue; + // If all preds have a single successor, then we know it is safe to insert + // the load on the pred (?!?), so we can insert code to materialize the + // pointer if it is not available. + Value *LoadPtr = Load->getPointerOperand(); + BasicBlock *Cur = Load->getParent(); + while (Cur != LoadBB) { + PHITransAddr Address(LoadPtr, DL, AC); + LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(), + *DT, NewInsts); + if (!LoadPtr) { + CanDoPRE = false; + break; + } + Cur = Cur->getSinglePredecessor(); + } - // Skip already visited predecessors. - if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; })) - continue; + if (LoadPtr) { + PHITransAddr Address(LoadPtr, DL, AC); + LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT, + NewInsts); + } + // If we couldn't find or insert a computation of this phi translated value, + // we fail PRE. + if (!LoadPtr) { + LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: " + << *Load->getPointerOperand() << "\n"); + CanDoPRE = false; + break; + } - PHITransAddr TransAddr = Addr; - if (TransAddr.needsPHITranslationFromBlock(BB)) - TransAddr.translateValue(BB, Pred, DT, false); + PredLoad.second = LoadPtr; + } - auto It = Blocks.find(Pred); - if (It != Blocks.end()) { - // If we reach a visited block with a different address, set the - // current block as clobbering the memory location in an unknown way - // (by returning false). - if (It->second.Addr.getAddr() != TransAddr.getAddr()) - return false; - // Otherwise, just stop the traversal. - continue; + if (!CanDoPRE) { + while (!NewInsts.empty()) { + // Erase instructions generated by the failed PHI translation before + // trying to number them. PHI translation might insert instructions + // in basic blocks other than the current one, and we delete them + // directly, as salvageAndRemoveInstruction only allows removing from the + // current basic block. + NewInsts.pop_back_val()->eraseFromParent(); } - - Preds.emplace_back( - Pred, DependencyBlockInfo(TransAddr, - MPhi ? MPhi->getIncomingValueForBlock(Pred) - : ClobberMA)); + // HINT: Don't revert the edge-splitting as following transformation may + // also need to split these critical edges. + return !CriticalEdgePredSplit.empty(); } - // We collected the predecessors and stored them in Preds. Now, populate the - // worklist with the predecessors found, and cache the eventual translated - // address for each block. - for (auto &P : Preds) { - [[maybe_unused]] auto It = - Blocks.try_emplace(P.first, std::move(P.second)).first; - Worklist.push_back(P.first); + // Okay, we can eliminate this load by inserting a reload in the predecessor + // and using PHI construction to get the value in the other predecessors, do + // it. + LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n'); + LLVM_DEBUG(if (!NewInsts.empty()) dbgs() + << "INSERTED " << NewInsts.size() << " INSTS: " << *NewInsts.back() + << '\n'); + + // Assign value numbers to the new instructions. + for (Instruction *I : NewInsts) { + // Instructions that have been inserted in predecessor(s) to materialize + // the load address do not retain their original debug locations. Doing + // so could lead to confusing (but correct) source attributions. + I->updateLocationAfterHoist(); + + // FIXME: We really _ought_ to insert these value numbers into their + // parent's availability map. However, in doing so, we risk getting into + // ordering issues. If a block hasn't been processed yet, we would be + // marking a value as AVAIL-IN, which isn't what we intend. + VN.lookupOrAdd(I); } + eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads, + &CriticalEdgePredAndLoad); + ++NumPRELoad; return true; } -/// Build a list of MemoryAccesses whose users could potentially alias the -/// memory location being queried. Starts from StartInfo's initial clobber, -/// walk the use-def chain to the final clobber. If the chain extends beyond -/// `BB`, continue into that block but only if it is in the previously collected -/// set. -void GVNPass::collectClobberList(SmallVectorImpl &Clobbers, - BasicBlock *BB, - const DependencyBlockInfo &StartInfo, - const DependencyBlockSet &Blocks, - MemorySSA &MSSA) { - MemoryAccess *MA = StartInfo.InitialClobberMA; - MemoryAccess *LastMA = StartInfo.ClobberMA; - - for (;;) { - while (MA != LastMA) { - Clobbers.push_back(MA); - MA = cast(MA)->getDefiningAccess(); - } - Clobbers.push_back(MA); - - if (MSSA.isLiveOnEntryDef(MA) || - (MA->getBlock() == BB && !isa(MA))) - break; +bool GVNPass::performLoopLoadPRE(LoadInst *Load, + AvailValInBlkVect &ValuesPerBlock, + UnavailBlkVect &UnavailableBlocks) { + const Loop *L = LI->getLoopFor(Load->getParent()); + // TODO: Generalize to other loop blocks that dominate the latch. + if (!L || L->getHeader() != Load->getParent()) + return false; - // If the final clobber in the current block is a MemoryPhi, go to the - // immediate dominator; otherwise, just get to the block containing the - // final clobber. - if (MA->getBlock() == BB) - BB = DT->getNode(BB)->getIDom()->getBlock(); - else - BB = MA->getBlock(); + BasicBlock *Preheader = L->getLoopPreheader(); + BasicBlock *Latch = L->getLoopLatch(); + if (!Preheader || !Latch) + return false; - auto It = Blocks.find(BB); - if (It == Blocks.end()) - break; + Value *LoadPtr = Load->getPointerOperand(); + // Must be available in preheader. + if (!L->isLoopInvariant(LoadPtr)) + return false; - MA = It->second.InitialClobberMA; - LastMA = It->second.ClobberMA; - if (MA == Clobbers.back()) - Clobbers.pop_back(); - } -} + // We plan to hoist the load to preheader without introducing a new fault. + // In order to do it, we need to prove that we cannot side-exit the loop + // once loop header is first entered before execution of the load. + if (ICF->isDominatedByICFIFromSameBlock(Load)) + return false; -/// Entrypoint for the MemorySSA-based redundant load elimination algorithm. -/// Given as input a load instruction, the function computes the set of reaching -/// memory values, one per predecessor path, that analyzeLoadAvailability can -/// later use to establish whether the load may be eliminated. A reaching value -/// may be of the following descriptor kind: -/// * Def: a precise instruction that produces the exact bits the load would -/// read (e.g., an equivalent load or a MustAlias store); -/// * Clobber: a write that clobbers a superset of the bits the load would read -/// (e.g., a memset over a larger region); -/// * Other: we know which block defines the memory location in some way, but -/// could not identify a precise instruction (e.g., memory already live at -/// function entry). -bool GVNPass::findReachingValuesForLoad(LoadInst *L, - SmallVectorImpl &Values, - MemorySSA &MSSA, AAResults &AAR) { - EarliestEscapeAnalysis EA(*DT, LI); - BatchAAResults AA(AAR, &EA); - BasicBlock *StartBlock = L->getParent(); - bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load); - // TODO: Simplify later work by just getClobberingMemoryAccess(). - MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess(); - const MemoryLocation Loc = MemoryLocation::get(L); + BasicBlock *LoopBlock = nullptr; + for (auto *Blocker : UnavailableBlocks) { + // Blockers from outside the loop are handled in preheader. + if (!L->contains(Blocker)) + continue; - // Fast path for load tagged with !invariant.group. - if (L->hasMetadata(LLVMContext::MD_invariant_group)) { - if (Instruction *G = findInvariantGroupValue(L, *DT)) { - Values.emplace_back( - ReachingMemVal::getDef(getLoadStorePointerOperand(G), G)); - return true; - } - } + // Only allow one loop block. Loop header is not less frequently executed + // than each loop block, and likely it is much more frequently executed. But + // in case of multiple loop blocks, we need extra information (such as block + // frequency info) to understand whether it is profitable to PRE into + // multiple loop blocks. + if (LoopBlock) + return false; - // Phase 1. First off, look for a local dependency to avoid having to - // disambiguate between before the load and after the load of the starting - // block (as the load may be visited from a backedge). - do { - // Scan users of the clobbering memory access. - if (auto RMV = scanMemoryAccessesUsers( - Loc, IsInvariantLoad, StartBlock, - SmallVector{ClobberMA}, MSSA, AA, L)) { - Values.emplace_back(*RMV); - return true; - } + // Do not sink into inner loops. This may be non-profitable. + if (L != LI->getLoopFor(Blocker)) + return false; - // Exit from here, and proceed visiting predecessors if the clobbering - // access is non-local or is a MemoryPhi. - if (ClobberMA->getBlock() != StartBlock || isa(ClobberMA)) - break; + // Blocks that dominate the latch execute on every single iteration, maybe + // except the last one. So PREing into these blocks doesn't make much sense + // in most cases. But the blocks that do not necessarily execute on each + // iteration are sometimes much colder than the header, and this is when + // PRE is potentially profitable. + if (DT->dominates(Blocker, Latch)) + return false; - // Check if the clobber actually aliases the load location. - if (auto RMV = accessMayModifyLocation(ClobberMA, Loc, IsInvariantLoad, - StartBlock, MSSA, AA)) { - Values.emplace_back(*RMV); - return true; - } + // Make sure that the terminator itself doesn't clobber. + if (Blocker->getTerminator()->mayWriteToMemory()) + return false; - // It may happen that the clobbering memory access does not actually - // clobber our load location, transition to its defining memory access. - ClobberMA = cast(ClobberMA)->getDefiningAccess(); - } while (ClobberMA->getBlock() == StartBlock); + LoopBlock = Blocker; + } - // Non-local speculations are not allowed under ASan. - if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) || - L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress)) + if (!LoopBlock) return false; - // Phase 2. Walk backwards through the CFG, collecting all the blocks that - // contain an instruction that modifies the load memory location, or that lie - // on a path between a clobbering block and our load. Start off by collecting - // the predecessors of `StartBlock`. All the visited blocks are stored in a - // the set `Blocks`. If possible, the memory address maintained for the block - // visited does get phi-translated. - DependencyBlockSet Blocks; - SmallVector InitialWorklist; - const DataLayout &DL = L->getModule()->getDataLayout(); - if (!collectPredecessors(StartBlock, - PHITransAddr(L->getPointerOperand(), DL, AC), - ClobberMA, Blocks, InitialWorklist)) + // Make sure the memory at this pointer cannot be freed, therefore we can + // safely reload from it after clobber. + if (LoadPtr->canBeFreed()) return false; - // Do a bottom-up DFS. - auto Worklist = InitialWorklist; - while (!Worklist.empty()) { - auto *BB = Worklist.pop_back_val(); - DependencyBlockInfo &Info = Blocks.find(BB)->second; + // TODO: Support critical edge splitting if blocker has more than 1 successor. + MapVector AvailableLoads; + AvailableLoads[LoopBlock] = LoadPtr; + AvailableLoads[Preheader] = LoadPtr; + + LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n'); + eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads, + /*CriticalEdgePredAndLoad*/ nullptr); + ++NumPRELoopLoad; + return true; +} + +/// Attempt to eliminate a load whose dependencies are +/// non-local by performing PHI construction. +bool GVNPass::processNonLocalLoad(LoadInst *Load) { + // Non-local speculations are not allowed under asan. + if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) || + Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress)) + return false; - // Phi-translation may have failed. - if (!Info.Addr.getAddr()) - continue; + // Find the non-local dependencies of the load. + LoadDepVect Deps; + MD->getNonLocalPointerDependency(Load, Deps); - // If the clobbering memory access is in the current block and it indeed - // clobbers our load location, record the dependency and do not visit the - // predecessors of this block further, continue with the blocks in the - // worklist. - if (Info.ClobberMA->getBlock() == BB && !isa(Info.ClobberMA)) { - if (auto RMV = accessMayModifyLocation( - Info.ClobberMA, Loc.getWithNewPtr(Info.Addr.getAddr()), - IsInvariantLoad, BB, MSSA, AA)) { - Info.MemVal = RMV; - continue; - } - assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) && - "LiveOnEntry aliases everything"); + // If we had to process more than one hundred blocks to find the + // dependencies, this load isn't worth worrying about. Optimizing + // it will be too expensive. + unsigned NumDeps = Deps.size(); + if (NumDeps > MaxNumDeps) + return false; - // If, however, the clobbering memory access does not actually clobber - // our load location, transition to its defining memory access, but - // keep examining the same basic block. - Info.ClobberMA = - cast(Info.ClobberMA)->getDefiningAccess(); - Worklist.emplace_back(BB); - continue; - } + SmallVector MemVals; + MemVals.reserve(Deps.size()); - // At this point we know the current block is "transparent", i.e. the memory - // location is not modified when execution goes through this block. - // Continue to its predecessors, unless a predecessor has already been - // visited with a different address. We currently cannot represent such a - // dependency. - if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) { - Info.ForceUnknown = true; + for (const NonLocalDepResult &Dep : Deps) { + const auto &R = Dep.getResult(); + SelectAddr SelAddr = Dep.getAddress(); + BasicBlock *BB = Dep.getBB(); + Instruction *Inst = R.getInst(); + if (R.isSelect()) { + auto [Cond, Addrs] = SelAddr.getSelectCondAndAddrs(); + MemVals.emplace_back( + ReachingMemVal::getSelect(BB, Cond, Addrs.first, Addrs.second)); continue; } - if (BB != StartBlock && - !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist)) - Info.ForceUnknown = true; + Value *Address = SelAddr.getAddr(); + if (R.isClobber()) + MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst)); + else if (R.isDef()) + MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst)); + else + MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst)); } - // Phase 3. We have collected all the blocks that either write a value to the - // memory location of the load, or there exists a path to the load, along - // which the memory location is not modified. Perform a second DFS to find - // load-to-load dependencies; namely, look at the dominating memory reads, - // that alias our load. These are the MemoryUses that are users of the - // MemoryDefs we previously identified. If no memory read is encountered, - // either confirm the clobbering write found before or set to unknown. - Worklist = InitialWorklist; - for (BasicBlock *BB : Worklist) { - DependencyBlockInfo &Info = Blocks.find(BB)->second; - Info.Visited = true; - } + return processNonLocalLoad(Load, MemVals); +} - SmallVector Clobbers; - while (!Worklist.empty()) { - auto *BB = Worklist.pop_back_val(); - DependencyBlockInfo &Info = Blocks.find(BB)->second; +bool GVNPass::processNonLocalLoad(LoadInst *Load, + SmallVectorImpl &Deps) { + // If we had a phi translation failure, we'll have a single entry which is a + // clobber in the current block. Reject this early. + if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) { + LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs()); + dbgs() << " has unknown dependencies\n";); + return false; + } - // If phi-translation failed, assume the memory location is modified in - // unknown way. - if (!Info.Addr.getAddr()) { - Values.push_back(ReachingMemVal::getUnknown(BB, nullptr)); - continue; + bool Changed = false; + // This is a limited form of scalar PRE for load indices. If this load follows + // a GEP, see if we can PRE the indices before analyzing. + if (isScalarPREEnabled()) { + if (GetElementPtrInst *GEP = + dyn_cast(Load->getOperand(0))) { + for (Use &U : GEP->indices()) + if (Instruction *I = dyn_cast(U.get())) + Changed |= performScalarPRE(I); } + } - Clobbers.clear(); - collectClobberList(Clobbers, BB, Info, Blocks, MSSA); - if (auto RMV = - scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()), - IsInvariantLoad, BB, Clobbers, MSSA, AA)) { - Values.push_back(*RMV); - continue; - } + // Step 1: Analyze the availability of the load. + AvailValInBlkVect ValuesPerBlock; + UnavailBlkVect UnavailableBlocks; + analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks); - // If no reusable memory use was found, and the current block is not - // transparent, use the already established memory def. - if (Info.MemVal) { - Values.push_back(*Info.MemVal); - continue; - } + // If we have no predecessors that produce a known value for this load, exit + // early. + if (ValuesPerBlock.empty()) + return Changed; - if (Info.ForceUnknown) { - Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr())); - continue; - } + // Step 2: Eliminate fully redundancy. + // + // If all of the instructions we depend on produce a known value for this + // load, then it is fully redundant and we can use PHI insertion to compute + // its value. Insert PHIs and remove the fully redundant value now. + if (UnavailableBlocks.empty()) { + LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n'); - // If the current block is transparent, continue to its predecessors. - for (BasicBlock *Pred : predecessors(BB)) { - auto It = Blocks.find(Pred); - if (It == Blocks.end()) - continue; - DependencyBlockInfo &PredInfo = It->second; - if (PredInfo.Visited) - continue; - PredInfo.Visited = true; - Worklist.push_back(Pred); - } + // Perform PHI construction. + Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree()); + // constructSSAForLoadSet is responsible for combining metadata. + ICF->removeUsersOf(Load); + Load->replaceAllUsesWith(V); + + if (isa(V)) + V->takeName(Load); + if (Instruction *I = dyn_cast(V)) + // If instruction I has debug info, then we should not update it. + // Also, if I has a null DebugLoc, then it is still potentially incorrect + // to propagate Load's DebugLoc because Load may not post-dominate I. + if (Load->getDebugLoc() && Load->getParent() == I->getParent()) + I->setDebugLoc(Load->getDebugLoc()); + if (MD && V->getType()->isPtrOrPtrVectorTy()) + MD->invalidateCachedPointerInfo(V); + ++NumGVNLoad; + reportLoadElim(Load, V, ORE); + salvageAndRemoveInstruction(Load); + return true; } - return true; + // Step 3: Eliminate partial redundancy. + if (!isLoadPREEnabled()) + return Changed; + if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent())) + return Changed; + + if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) || + performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks)) + return true; + + return Changed; } /// Attempt to eliminate a load, first by eliminating it @@ -2874,188 +2963,99 @@ bool GVNPass::processMaskedLoad(IntrinsicInst *I) { return true; } -/// Return a pair the first field showing the value number of \p Exp and the -/// second field showing whether it is a value number newly created. -std::pair -GVNPass::ValueTable::assignExpNewValueNum(Expression &Exp) { - uint32_t &E = ExpressionNumbering[Exp]; - bool CreateNewValNum = !E; - if (CreateNewValNum) { - Expressions.push_back(Exp); - if (ExprIdx.size() < NextValueNumber + 1) - ExprIdx.resize(NextValueNumber * 2); - E = NextValueNumber; - ExprIdx[NextValueNumber++] = NextExprNumber++; - } - return {E, CreateNewValNum}; -} - -/// Return whether all the values related with the same \p num are -/// defined in \p BB. -bool GVNPass::ValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB, - GVNPass &GVN) { - return all_of( - GVN.LeaderTable.getLeaders(Num), - [=](const LeaderMap::LeaderTableEntry &L) { return L.BB == BB; }); -} - -/// Wrap phiTranslateImpl to provide caching functionality. -uint32_t GVNPass::ValueTable::phiTranslate(const BasicBlock *Pred, - const BasicBlock *PhiBlock, - uint32_t Num, GVNPass &GVN) { - auto FindRes = PhiTranslateTable.find({Num, Pred}); - if (FindRes != PhiTranslateTable.end()) - return FindRes->second; - uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, GVN); - PhiTranslateTable.insert({{Num, Pred}, NewNum}); - return NewNum; -} - -// Return true if the value number \p Num and NewNum have equal value. -// Return false if the result is unknown. -bool GVNPass::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum, - const BasicBlock *Pred, - const BasicBlock *PhiBlock, - GVNPass &GVN) { - CallInst *Call = nullptr; - auto Leaders = GVN.LeaderTable.getLeaders(Num); - for (const auto &Entry : Leaders) { - Call = dyn_cast(&*Entry.Val); - if (Call && Call->getParent() == PhiBlock) - break; - } - - if (AA->doesNotAccessMemory(Call)) - return true; +// If the given branch is recognized as a foldable branch (i.e. conditional +// branch with constant condition), it will perform following analyses and +// transformation. +// 1) If the dead out-coming edge is a critical-edge, split it. Let +// R be the target of the dead out-coming edge. +// 1) Identify the set of dead blocks implied by the branch's dead outcoming +// edge. The result of this step will be {X| X is dominated by R} +// 2) Identify those blocks which haves at least one dead predecessor. The +// result of this step will be dominance-frontier(R). +// 3) Update the PHIs in DF(R) by replacing the operands corresponding to +// dead blocks with "UndefVal" in an hope these PHIs will optimized away. +// +// Return true iff *NEW* dead code are found. +bool GVNPass::processFoldableCondBr(CondBrInst *BI) { + // If a branch has two identical successors, we cannot declare either dead. + if (BI->getSuccessor(0) == BI->getSuccessor(1)) + return false; - if (!MD || !AA->onlyReadsMemory(Call)) + ConstantInt *Cond = dyn_cast(BI->getCondition()); + if (!Cond) return false; - MemDepResult LocalDep = MD->getDependency(Call); - if (!LocalDep.isNonLocal()) + BasicBlock *DeadRoot = + Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0); + if (DeadBlocks.count(DeadRoot)) return false; - const MemoryDependenceResults::NonLocalDepInfo &Deps = - MD->getNonLocalCallDependency(Call); + if (!DeadRoot->getSinglePredecessor()) + DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot); - // Check to see if the Call has no function local clobber. - for (const NonLocalDepEntry &D : Deps) { - if (D.getResult().isNonFuncLocal()) - return true; - } - return false; + addDeadBlock(DeadRoot); + return true; } -/// Translate value number \p Num using phis, so that it has the values of -/// the phis in BB. -uint32_t GVNPass::ValueTable::phiTranslateImpl(const BasicBlock *Pred, - const BasicBlock *PhiBlock, - uint32_t Num, GVNPass &GVN) { - // See if we can refine the value number by looking at the PN incoming value - // for the given predecessor. - if (PHINode *PN = NumberingPhi[Num]) { - if (PN->getParent() != PhiBlock) - return Num; - for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) { - if (PN->getIncomingBlock(I) != Pred) - continue; - if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false)) - return TransVal; - } - return Num; - } - - if (BasicBlock *BB = NumberingBB[Num]) { - assert(MSSA && "NumberingBB is non-empty only when using MemorySSA"); - // Value numbers of basic blocks are used to represent memory state in - // load/store instructions and read-only function calls when said state is - // set by a MemoryPhi. - if (BB != PhiBlock) - return Num; - MemoryPhi *MPhi = MSSA->getMemoryAccess(BB); - for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) { - if (MPhi->getIncomingBlock(i) != Pred) - continue; - MemoryAccess *MA = MPhi->getIncomingValue(i); - if (auto *PredPhi = dyn_cast(MA)) - return lookupOrAdd(PredPhi->getBlock()); - if (MSSA->isLiveOnEntryDef(MA)) - return lookupOrAdd(&BB->getParent()->getEntryBlock()); - return lookupOrAdd(cast(MA)->getMemoryInst()); - } - llvm_unreachable( - "CFG/MemorySSA mismatch: predecessor not found among incoming blocks"); - } +bool GVNPass::processAssumeIntrinsic(AssumeInst *IntrinsicI) { + Value *V = IntrinsicI->getArgOperand(0); - // If there is any value related with Num is defined in a BB other than - // PhiBlock, it cannot depend on a phi in PhiBlock without going through - // a backedge. We can do an early exit in that case to save compile time. - if (!areAllValsInBB(Num, PhiBlock, GVN)) - return Num; + if (ConstantInt *Cond = dyn_cast(V)) { + if (Cond->isZero()) { + Type *Int8Ty = Type::getInt8Ty(V->getContext()); + Type *PtrTy = PointerType::get(V->getContext(), 0); + // Insert a new store to null instruction before the load to indicate that + // this code is not reachable. FIXME: We could insert unreachable + // instruction directly because we can modify the CFG. + auto *NewS = + new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy), + IntrinsicI->getIterator()); + if (MSSAU) { + const MemoryUseOrDef *FirstNonDom = nullptr; + const auto *AL = + MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent()); - if (Num >= ExprIdx.size() || ExprIdx[Num] == 0) - return Num; - Expression Exp = Expressions[ExprIdx[Num]]; + // If there are accesses in the current basic block, find the first one + // that does not come before NewS. The new memory access is inserted + // after the found access or before the terminator if no such access is + // found. + if (AL) { + for (const auto &Acc : *AL) { + if (auto *Current = dyn_cast(&Acc)) + if (!Current->getMemoryInst()->comesBefore(NewS)) { + FirstNonDom = Current; + break; + } + } + } - for (unsigned I = 0; I < Exp.VarArgs.size(); I++) { - // For InsertValue and ExtractValue, some varargs are index numbers - // instead of value numbers. Those index numbers should not be - // translated. - if ((I > 1 && Exp.Opcode == Instruction::InsertValue) || - (I > 0 && Exp.Opcode == Instruction::ExtractValue) || - (I > 1 && Exp.Opcode == Instruction::ShuffleVector)) - continue; - Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], GVN); - } + auto *NewDef = + FirstNonDom + ? MSSAU->createMemoryAccessBefore( + NewS, nullptr, const_cast(FirstNonDom)) + : MSSAU->createMemoryAccessInBB(NewS, nullptr, + NewS->getParent(), + MemorySSA::BeforeTerminator); - if (Exp.Commutative) { - assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!"); - if (Exp.VarArgs[0] > Exp.VarArgs[1]) { - std::swap(Exp.VarArgs[0], Exp.VarArgs[1]); - uint32_t Opcode = Exp.Opcode >> 8; - if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) - Exp.Opcode = (Opcode << 8) | - CmpInst::getSwappedPredicate( - static_cast(Exp.Opcode & 255)); + MSSAU->insertDef(cast(NewDef), /*RenameUses=*/false); + } } + if (isAssumeWithEmptyBundle(*IntrinsicI)) { + salvageAndRemoveInstruction(IntrinsicI); + return true; + } + return false; } - if (uint32_t NewNum = ExpressionNumbering[Exp]) { - if (Exp.Opcode == Instruction::Call && NewNum != Num) - return areCallValsEqual(Num, NewNum, Pred, PhiBlock, GVN) ? NewNum : Num; - return NewNum; - } - return Num; -} - -/// Erase stale entry from phiTranslate cache so phiTranslate can be computed -/// again. -void GVNPass::ValueTable::eraseTranslateCacheEntry( - uint32_t Num, const BasicBlock &CurrBlock) { - for (const BasicBlock *Pred : predecessors(&CurrBlock)) - PhiTranslateTable.erase({Num, Pred}); -} - -// In order to find a leader for a given value number at a -// specific basic block, we first obtain the list of all Values for that number, -// and then scan the list to find one whose block dominates the block in -// question. This is fast because dominator tree queries consist of only -// a few comparisons of DFS numbers. -Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) { - auto Leaders = LeaderTable.getLeaders(Num); - if (Leaders.empty()) - return nullptr; - - Value *Val = nullptr; - for (const auto &Entry : Leaders) { - if (DT->dominates(Entry.BB, BB)) { - Val = Entry.Val; - if (isa(Val)) - return Val; - } + if (isa(V)) { + // If it's not false, and constant, it must evaluate to true. This means our + // assume is assume(true), and thus, pointless, and we don't want to do + // anything more here. + return false; } - return Val; + Constant *True = ConstantInt::getTrue(V->getContext()); + return propagateEquality(V, True, IntrinsicI); } /// There is an edge from 'Src' to 'Dst'. Return @@ -3074,15 +3074,6 @@ static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E, return Pred != nullptr; } -void GVNPass::assignBlockRPONumber(Function &F) { - BlockRPONumber.clear(); - uint32_t NextBlockNumber = 1; - ReversePostOrderTraversal RPOT(&F); - for (BasicBlock *BB : RPOT) - BlockRPONumber[BB] = NextBlockNumber++; - InvalidBlockRPONumbers = false; -} - /// The given values are known to be equal in every use /// dominated by 'Root'. Exploit this, for example by replacing 'LHS' with /// 'RHS' everywhere in the scope. Returns whether a change was made. @@ -3290,6 +3281,11 @@ bool GVNPass::propagateEquality( return Changed; } +static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) { + patchReplacementInstruction(I, Repl); + I->replaceAllUsesWith(Repl); +} + /// When calculating availability, handle an instruction /// by inserting it into the appropriate sets. bool GVNPass::processInstruction(Instruction *I) { @@ -3433,94 +3429,18 @@ bool GVNPass::processInstruction(Instruction *I) { return false; } - if (Repl == I) { - // If I was the result of a shortcut PRE, it might already be in the table - // and the best replacement for itself. Nothing to do. - return false; - } - - // Remove it! - patchAndReplaceAllUsesWith(I, Repl); - if (MD && Repl->getType()->isPtrOrPtrVectorTy()) - MD->invalidateCachedPointerInfo(Repl); - salvageAndRemoveInstruction(I); - return true; -} - -/// runOnFunction - This is the main transformation entry point for a function. -bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, - const TargetLibraryInfo &RunTLI, AAResults &RunAA, - MemoryDependenceResults *RunMD, LoopInfo &LI, - OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) { - AC = &RunAC; - DT = &RunDT; - VN.setDomTree(DT); - TLI = &RunTLI; - AA = &RunAA; - VN.setAliasAnalysis(&RunAA); - MD = RunMD; - ImplicitControlFlowTracking ImplicitCFT; - ICF = &ImplicitCFT; - this->LI = &LI; - VN.setMemDep(MD); - // Propagate the MSSA-enabled flag so the value-numbering paths in - // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether - // IsMSSAEnabled is turned on. - VN.setMemorySSA(MSSA, isMemorySSAEnabled()); - ORE = RunORE; - InvalidBlockRPONumbers = true; - MemorySSAUpdater Updater(MSSA); - MSSAU = MSSA ? &Updater : nullptr; - - bool Changed = false; - bool ShouldContinue = true; - - DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); - // Merge unconditional branches, allowing PRE to catch more - // optimization opportunities. - for (BasicBlock &BB : make_early_inc_range(F)) { - bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD); - if (RemovedBlock) - ++NumGVNBlocks; - - Changed |= RemovedBlock; - } - DTU.flush(); - - unsigned Iteration = 0; - while (ShouldContinue) { - LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n"); - (void) Iteration; - ShouldContinue = iterateOnFunction(F); - Changed |= ShouldContinue; - ++Iteration; - } - - if (isScalarPREEnabled()) { - // Fabricate val-num for dead-code in order to suppress assertion in - // performPRE(). - assignValNumForDeadCode(); - bool PREChanged = true; - while (PREChanged) { - PREChanged = performPRE(F); - Changed |= PREChanged; - } - } - - // FIXME: Should perform GVN again after PRE does something. PRE can move - // computations into blocks where they become fully redundant. Note that - // we can't do this until PRE's critical edge splitting updates memdep. - // Actually, when this happens, we should just fully integrate PRE into GVN. - - cleanupGlobalSets(); - // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each - // iteration. - DeadBlocks.clear(); - - if (MSSA && VerifyMemorySSA) - MSSA->verifyMemorySSA(); + if (Repl == I) { + // If I was the result of a shortcut PRE, it might already be in the table + // and the best replacement for itself. Nothing to do. + return false; + } - return Changed; + // Remove it! + patchAndReplaceAllUsesWith(I, Repl); + if (MD && Repl->getType()->isPtrOrPtrVectorTy()) + MD->invalidateCachedPointerInfo(Repl); + salvageAndRemoveInstruction(I); + return true; } bool GVNPass::processBlock(BasicBlock *BB) { @@ -3543,6 +3463,23 @@ bool GVNPass::processBlock(BasicBlock *BB) { return ChangedFunction; } +/// Executes one iteration of GVN. +bool GVNPass::iterateOnFunction(Function &F) { + cleanupGlobalSets(); + + // Top-down walk of the dominator tree. + bool Changed = false; + // Needed for value numbering with phi construction to work. + // RPOT walks the graph in its constructor and will not be invalidated during + // processBlock. + ReversePostOrderTraversal RPOT(&F); + + for (BasicBlock *BB : RPOT) + Changed |= processBlock(BB); + + return Changed; +} + // Instantiate an expression in a predecessor that lacked it. bool GVNPass::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred, BasicBlock *Curr, unsigned int ValNo) { @@ -3780,60 +3717,104 @@ bool GVNPass::performPRE(Function &F) { return Changed; } -/// Split the critical edge connecting the given two blocks, and return -/// the block inserted to the critical edge. -BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) { - // GVN does not require loop-simplify, do not try to preserve it if it is not - // possible. - BasicBlock *BB = SplitCriticalEdge( - Pred, Succ, - CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify()); - if (BB) { - if (MD) - MD->invalidateCachedPredecessors(); - InvalidBlockRPONumbers = true; +/// runOnFunction - This is the main transformation entry point for a function. +bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT, + const TargetLibraryInfo &RunTLI, AAResults &RunAA, + MemoryDependenceResults *RunMD, LoopInfo &LI, + OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) { + AC = &RunAC; + DT = &RunDT; + VN.setDomTree(DT); + TLI = &RunTLI; + AA = &RunAA; + VN.setAliasAnalysis(&RunAA); + MD = RunMD; + ImplicitControlFlowTracking ImplicitCFT; + ICF = &ImplicitCFT; + this->LI = &LI; + VN.setMemDep(MD); + // Propagate the MSSA-enabled flag so the value-numbering paths in + // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether + // IsMSSAEnabled is turned on. + VN.setMemorySSA(MSSA, isMemorySSAEnabled()); + ORE = RunORE; + InvalidBlockRPONumbers = true; + MemorySSAUpdater Updater(MSSA); + MSSAU = MSSA ? &Updater : nullptr; + + bool Changed = false; + bool ShouldContinue = true; + + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); + // Merge unconditional branches, allowing PRE to catch more + // optimization opportunities. + for (BasicBlock &BB : make_early_inc_range(F)) { + bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD); + if (RemovedBlock) + ++NumGVNBlocks; + + Changed |= RemovedBlock; } - return BB; -} + DTU.flush(); -/// Split critical edges found during the previous -/// iteration that may enable further optimization. -bool GVNPass::splitCriticalEdges() { - if (ToSplit.empty()) - return false; + unsigned Iteration = 0; + while (ShouldContinue) { + LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n"); + (void)Iteration; + ShouldContinue = iterateOnFunction(F); + Changed |= ShouldContinue; + ++Iteration; + } - bool Changed = false; - do { - std::pair Edge = ToSplit.pop_back_val(); - Changed |= SplitCriticalEdge(Edge.first, Edge.second, - CriticalEdgeSplittingOptions(DT, LI, MSSAU)) != - nullptr; - } while (!ToSplit.empty()); - if (Changed) { - if (MD) - MD->invalidateCachedPredecessors(); - InvalidBlockRPONumbers = true; + if (isScalarPREEnabled()) { + // Fabricate val-num for dead-code in order to suppress assertion in + // performPRE(). + assignValNumForDeadCode(); + bool PREChanged = true; + while (PREChanged) { + PREChanged = performPRE(F); + Changed |= PREChanged; + } } - return Changed; -} -/// Executes one iteration of GVN. -bool GVNPass::iterateOnFunction(Function &F) { - cleanupGlobalSets(); + // FIXME: Should perform GVN again after PRE does something. PRE can move + // computations into blocks where they become fully redundant. Note that + // we can't do this until PRE's critical edge splitting updates memdep. + // Actually, when this happens, we should just fully integrate PRE into GVN. - // Top-down walk of the dominator tree. - bool Changed = false; - // Needed for value numbering with phi construction to work. - // RPOT walks the graph in its constructor and will not be invalidated during - // processBlock. - ReversePostOrderTraversal RPOT(&F); + cleanupGlobalSets(); + // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each + // iteration. + DeadBlocks.clear(); - for (BasicBlock *BB : RPOT) - Changed |= processBlock(BB); + if (MSSA && VerifyMemorySSA) + MSSA->verifyMemorySSA(); return Changed; } +// In order to find a leader for a given value number at a +// specific basic block, we first obtain the list of all Values for that number, +// and then scan the list to find one whose block dominates the block in +// question. This is fast because dominator tree queries consist of only +// a few comparisons of DFS numbers. +Value *GVNPass::findLeader(const BasicBlock *BB, uint32_t Num) { + auto Leaders = LeaderTable.getLeaders(Num); + if (Leaders.empty()) + return nullptr; + + Value *Val = nullptr; + for (const auto &Entry : Leaders) { + if (DT->dominates(Entry.BB, BB)) { + Val = Entry.Val; + if (isa(Val)) + return Val; + } + } + + return Val; +} + void GVNPass::cleanupGlobalSets() { VN.clear(); LeaderTable.clear(); @@ -3855,12 +3836,55 @@ void GVNPass::removeInstruction(Instruction *I) { ++NumGVNInstr; } +void GVNPass::salvageAndRemoveInstruction(Instruction *I) { + salvageKnowledge(I, AC); + salvageDebugInfo(*I); + removeInstruction(I); +} + /// Verify that the specified instruction does not occur in our /// internal data structures. void GVNPass::verifyRemoved(const Instruction *Inst) const { VN.verifyRemoved(Inst); } +/// Split critical edges found during the previous +/// iteration that may enable further optimization. +bool GVNPass::splitCriticalEdges() { + if (ToSplit.empty()) + return false; + + bool Changed = false; + do { + std::pair Edge = ToSplit.pop_back_val(); + Changed |= SplitCriticalEdge(Edge.first, Edge.second, + CriticalEdgeSplittingOptions(DT, LI, MSSAU)) != + nullptr; + } while (!ToSplit.empty()); + if (Changed) { + if (MD) + MD->invalidateCachedPredecessors(); + InvalidBlockRPONumbers = true; + } + return Changed; +} + +/// Split the critical edge connecting the given two blocks, and return +/// the block inserted to the critical edge. +BasicBlock *GVNPass::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) { + // GVN does not require loop-simplify, do not try to preserve it if it is not + // possible. + BasicBlock *BB = SplitCriticalEdge( + Pred, Succ, + CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify()); + if (BB) { + if (MD) + MD->invalidateCachedPredecessors(); + InvalidBlockRPONumbers = true; + } + return BB; +} + /// BB is declared dead, which implied other blocks become dead as well. This /// function is to add all these blocks to "DeadBlocks". For the dead blocks' /// live successors, update their phi nodes by replacing the operands @@ -3940,40 +3964,6 @@ void GVNPass::addDeadBlock(BasicBlock *BB) { } } -// If the given branch is recognized as a foldable branch (i.e. conditional -// branch with constant condition), it will perform following analyses and -// transformation. -// 1) If the dead out-coming edge is a critical-edge, split it. Let -// R be the target of the dead out-coming edge. -// 1) Identify the set of dead blocks implied by the branch's dead outcoming -// edge. The result of this step will be {X| X is dominated by R} -// 2) Identify those blocks which haves at least one dead predecessor. The -// result of this step will be dominance-frontier(R). -// 3) Update the PHIs in DF(R) by replacing the operands corresponding to -// dead blocks with "UndefVal" in an hope these PHIs will optimized away. -// -// Return true iff *NEW* dead code are found. -bool GVNPass::processFoldableCondBr(CondBrInst *BI) { - // If a branch has two identical successors, we cannot declare either dead. - if (BI->getSuccessor(0) == BI->getSuccessor(1)) - return false; - - ConstantInt *Cond = dyn_cast(BI->getCondition()); - if (!Cond) - return false; - - BasicBlock *DeadRoot = - Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0); - if (DeadBlocks.count(DeadRoot)) - return false; - - if (!DeadRoot->getSinglePredecessor()) - DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot); - - addDeadBlock(DeadRoot); - return true; -} - // performPRE() will trigger assert if it comes across an instruction without // associated val-num. As it normally has far more live instructions than dead // instructions, it makes more sense just to "fabricate" a val-number for the @@ -3987,6 +3977,15 @@ void GVNPass::assignValNumForDeadCode() { } } +void GVNPass::assignBlockRPONumber(Function &F) { + BlockRPONumber.clear(); + uint32_t NextBlockNumber = 1; + ReversePostOrderTraversal RPOT(&F); + for (BasicBlock *BB : RPOT) + BlockRPONumber[BB] = NextBlockNumber++; + InvalidBlockRPONumbers = false; +} + class llvm::GVNLegacyPass : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid.