diff --git a/llvm/include/llvm/Analysis/IndirectCallVisitor.h b/llvm/include/llvm/Analysis/IndirectCallVisitor.h index 1d1f3f4cc5c0bc..eb72f2c5d14ddb 100644 --- a/llvm/include/llvm/Analysis/IndirectCallVisitor.h +++ b/llvm/include/llvm/Analysis/IndirectCallVisitor.h @@ -18,7 +18,7 @@ namespace llvm { // Visitor class that finds all indirect call. struct PGOIndirectCallVisitor : public InstVisitor { - std::vector IndirectCalls; + std::vector IndirectCalls; PGOIndirectCallVisitor() {} void visitCallBase(CallBase &Call) { @@ -28,7 +28,7 @@ struct PGOIndirectCallVisitor : public InstVisitor { }; // Helper function that finds all indirect call sites. -inline std::vector findIndirectCalls(Function &F) { +inline std::vector findIndirectCalls(Function &F) { PGOIndirectCallVisitor ICV; ICV.visit(F); return ICV.IndirectCalls; diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h index 99cafba4f0a8eb..584c65403b1ba6 100644 --- a/llvm/include/llvm/Transforms/Instrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation.h @@ -28,6 +28,7 @@ class FunctionPass; class ModulePass; class OptimizationRemarkEmitter; class Comdat; +class CallBase; /// Instrumentation passes often insert conditional checks into entry blocks. /// Call this function before splitting the entry block to move instructions @@ -106,7 +107,7 @@ FunctionPass *createPGOMemOPSizeOptLegacyPass(); // generic utilities. namespace pgo { -// Helper function that transforms Inst (either an indirect-call instruction, or +// Helper function that transforms CB (either an indirect-call instruction, or // an invoke instruction , to a conditional call to F. This is like: // if (Inst.CalledValue == F) // F(...); @@ -119,10 +120,9 @@ namespace pgo { // If \p AttachProfToDirectCall is true, a prof metadata is attached to the // new direct call to contain \p Count. // Returns the promoted direct call instruction. -Instruction *promoteIndirectCall(Instruction *Inst, Function *F, uint64_t Count, - uint64_t TotalCount, - bool AttachProfToDirectCall, - OptimizationRemarkEmitter *ORE); +CallBase &promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, + uint64_t TotalCount, bool AttachProfToDirectCall, + OptimizationRemarkEmitter *ORE); } // namespace pgo /// Options for the frontend instrumentation based profiling pass. diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 489976a0cb8e78..230982d321f883 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1044,13 +1044,13 @@ bool SampleProfileLoader::inlineHotFunctions( R->getValue()->getSubprogram() && isLegalToPromote(*I, R->getValue(), &Reason)) { uint64_t C = FS->getEntrySamples(); - Instruction *DI = - pgo::promoteIndirectCall(I, R->getValue(), C, Sum, false, ORE); + auto &DI = + pgo::promoteIndirectCall(*I, R->getValue(), C, Sum, false, ORE); Sum -= C; PromotedInsns.insert(I); // If profile mismatches, we should not attempt to inline DI. if ((isa(DI) || isa(DI)) && - inlineCallInstruction(*cast(DI))) { + inlineCallInstruction(cast(DI))) { localNotInlinedCallSites.erase(I); LocalChanged = true; ++NumCSInlined; diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index 9857769e880fa9..941a51cc3b3648 100644 --- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -198,7 +198,7 @@ class ICallPromotionFunc { // Promote a list of targets for one indirect-call callsite. Return // the number of promotions. - uint32_t tryToPromote(Instruction *Inst, + uint32_t tryToPromote(CallBase &CB, const std::vector &Candidates, uint64_t &TotalCount); @@ -294,23 +294,20 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite( return Ret; } -// FIXME(callsite): the Instruction* parameter and return can be changed to -// CallBase -Instruction *llvm::pgo::promoteIndirectCall(Instruction *Inst, - Function *DirectCallee, - uint64_t Count, uint64_t TotalCount, - bool AttachProfToDirectCall, - OptimizationRemarkEmitter *ORE) { +CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee, + uint64_t Count, uint64_t TotalCount, + bool AttachProfToDirectCall, + OptimizationRemarkEmitter *ORE) { uint64_t ElseCount = TotalCount - Count; uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount); uint64_t Scale = calculateCountScale(MaxCount); - MDBuilder MDB(Inst->getContext()); + MDBuilder MDB(CB.getContext()); MDNode *BranchWeights = MDB.createBranchWeights( scaleBranchCount(Count, Scale), scaleBranchCount(ElseCount, Scale)); - CallBase &NewInst = promoteCallWithIfThenElse(*cast(Inst), - DirectCallee, BranchWeights); + CallBase &NewInst = + promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights); if (AttachProfToDirectCall) { MDBuilder MDB(NewInst.getContext()); @@ -323,24 +320,24 @@ Instruction *llvm::pgo::promoteIndirectCall(Instruction *Inst, if (ORE) ORE->emit([&]() { - return OptimizationRemark(DEBUG_TYPE, "Promoted", Inst) + return OptimizationRemark(DEBUG_TYPE, "Promoted", &CB) << "Promote indirect call to " << NV("DirectCallee", DirectCallee) << " with count " << NV("Count", Count) << " out of " << NV("TotalCount", TotalCount); }); - return &NewInst; + return NewInst; } // Promote indirect-call to conditional direct-call for one callsite. uint32_t ICallPromotionFunc::tryToPromote( - Instruction *Inst, const std::vector &Candidates, + CallBase &CB, const std::vector &Candidates, uint64_t &TotalCount) { uint32_t NumPromoted = 0; for (auto &C : Candidates) { uint64_t Count = C.Count; - pgo::promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount, - SamplePGO, &ORE); + pgo::promoteIndirectCall(CB, C.TargetFunction, Count, TotalCount, SamplePGO, + &ORE); assert(TotalCount >= Count); TotalCount -= Count; NumOfPGOICallPromotion++; @@ -354,28 +351,28 @@ uint32_t ICallPromotionFunc::tryToPromote( bool ICallPromotionFunc::processFunction(ProfileSummaryInfo *PSI) { bool Changed = false; ICallPromotionAnalysis ICallAnalysis; - for (auto &I : findIndirectCalls(F)) { + for (auto *CB : findIndirectCalls(F)) { uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction( - I, NumVals, TotalCount, NumCandidates); + CB, NumVals, TotalCount, NumCandidates); if (!NumCandidates || (PSI && PSI->hasProfileSummary() && !PSI->isHotCount(TotalCount))) continue; auto PromotionCandidates = getPromotionCandidatesForCallSite( - I, ICallProfDataRef, TotalCount, NumCandidates); - uint32_t NumPromoted = tryToPromote(I, PromotionCandidates, TotalCount); + CB, ICallProfDataRef, TotalCount, NumCandidates); + uint32_t NumPromoted = tryToPromote(*CB, PromotionCandidates, TotalCount); if (NumPromoted == 0) continue; Changed = true; // Adjust the MD.prof metadata. First delete the old one. - I->setMetadata(LLVMContext::MD_prof, nullptr); + CB->setMetadata(LLVMContext::MD_prof, nullptr); // If all promoted, we don't need the MD.prof metadata. if (TotalCount == 0 || NumPromoted == NumVals) continue; // Otherwise we need update with the un-promoted records back. - annotateValueSite(*M, *I, ICallProfDataRef.slice(NumPromoted), TotalCount, + annotateValueSite(*M, *CB, ICallProfDataRef.slice(NumPromoted), TotalCount, IPVK_IndirectCallTarget, NumCandidates); } return Changed; diff --git a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc index d9542dcf028ef0..361035b178c85e 100644 --- a/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc +++ b/llvm/lib/Transforms/Instrumentation/ValueProfilePlugins.inc @@ -57,7 +57,7 @@ public: IndirectCallPromotionPlugin(Function &Fn) : F(Fn) {} void run(std::vector &Candidates) { - std::vector Result = findIndirectCalls(F); + std::vector Result = findIndirectCalls(F); for (Instruction *I : Result) { Value *Callee = cast(I)->getCalledOperand(); Instruction *InsertPt = I;