-
Notifications
You must be signed in to change notification settings - Fork 10.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move "auto-init" instructions to the dominator of their users
As a result of -ftrivial-auto-var-init, clang generates instructions to set alloca'd memory to a given pattern, right after the allocation site. In some cases, this (somehow costly) operation could be delayed, leading to conditional execution in some cases. This is not an uncommon situation: it happens ~500 times on the cPython code base, and much more on the LLVM codebase. The benefit greatly varies on the execution path, but it should not regress on performance. This is a recommit of cca0100 with MemorySSA update fixes. Differential Revision: https://reviews.llvm.org/D137707
- Loading branch information
1 parent
a198ade
commit 50b2a11
Showing
19 changed files
with
613 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
//===- MoveAutoInit.h - Move insts marked as auto-init Pass --*- C++ -*-======// | ||
// | ||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
// See https://llvm.org/LICENSE.txt for license information. | ||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
// | ||
//===----------------------------------------------------------------------===// | ||
// | ||
// This pass moves instructions marked as auto-init closer to their use if | ||
// profitable, generally because it moves them under a guard, potentially | ||
// skipping the overhead of the auto-init under some execution paths. | ||
// | ||
// | ||
//===----------------------------------------------------------------------===// | ||
|
||
#ifndef LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H | ||
#define LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H | ||
|
||
#include "llvm/IR/PassManager.h" | ||
|
||
namespace llvm { | ||
|
||
class MoveAutoInitPass : public PassInfoMixin<MoveAutoInitPass> { | ||
public: | ||
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); | ||
}; | ||
} // end namespace llvm | ||
|
||
#endif // LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,220 @@ | ||
//===-- MoveAutoInit.cpp - move auto-init inst closer to their use site----===// | ||
// | ||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
// See https://llvm.org/LICENSE.txt for license information. | ||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
// | ||
//===----------------------------------------------------------------------===// | ||
// | ||
// This pass moves instruction maked as auto-init closer to the basic block that | ||
// use it, eventually removing it from some control path of the function. | ||
// | ||
//===----------------------------------------------------------------------===// | ||
|
||
#include "llvm/Transforms/Utils/MoveAutoInit.h" | ||
#include "llvm/ADT/STLExtras.h" | ||
#include "llvm/ADT/Statistic.h" | ||
#include "llvm/ADT/StringSet.h" | ||
#include "llvm/Analysis/MemorySSA.h" | ||
#include "llvm/Analysis/MemorySSAUpdater.h" | ||
#include "llvm/IR/DebugInfo.h" | ||
#include "llvm/IR/Dominators.h" | ||
#include "llvm/IR/IRBuilder.h" | ||
#include "llvm/IR/Instructions.h" | ||
#include "llvm/IR/IntrinsicInst.h" | ||
#include "llvm/Support/CommandLine.h" | ||
#include "llvm/Transforms/Utils.h" | ||
#include "llvm/Transforms/Utils/LoopUtils.h" | ||
|
||
using namespace llvm; | ||
|
||
#define DEBUG_TYPE "move-auto-init" | ||
|
||
STATISTIC(NumMoved, "Number of instructions moved"); | ||
|
||
static cl::opt<unsigned> MoveAutoInitThreshold( | ||
"move-auto-init-threshold", cl::Hidden, cl::init(128), | ||
cl::desc("Maximum instructions to analyze per moved initialization")); | ||
|
||
static bool hasAutoInitMetadata(const Instruction &I) { | ||
return I.hasMetadata(LLVMContext::MD_annotation) && | ||
any_of(I.getMetadata(LLVMContext::MD_annotation)->operands(), | ||
[](const MDOperand &Op) { | ||
return cast<MDString>(Op.get())->getString() == "auto-init"; | ||
}); | ||
} | ||
|
||
/// Finds a BasicBlock in the CFG where instruction `I` can be moved to while | ||
/// not changing the Memory SSA ordering and being guarded by at least one | ||
/// condition. | ||
static BasicBlock *usersDominator(Instruction *I, DominatorTree &DT, | ||
MemorySSA &MSSA) { | ||
BasicBlock *CurrentDominator = nullptr; | ||
MemoryLocation ML; | ||
if (auto *MI = dyn_cast<MemIntrinsic>(I)) | ||
ML = MemoryLocation::getForDest(MI); | ||
else if (auto *SI = dyn_cast<StoreInst>(I)) | ||
ML = MemoryLocation::get(SI); | ||
else | ||
assert(false && "memory location set"); | ||
|
||
MemoryUseOrDef &IMA = *MSSA.getMemoryAccess(I); | ||
BatchAAResults AA(MSSA.getAA()); | ||
|
||
SmallPtrSet<MemoryAccess *, 8> Visited; | ||
|
||
auto AsMemoryAccess = [](User *U) { return cast<MemoryAccess>(U); }; | ||
SmallVector<MemoryAccess *> WorkList(map_range(IMA.users(), AsMemoryAccess)); | ||
|
||
while (!WorkList.empty()) { | ||
MemoryAccess *MA = WorkList.pop_back_val(); | ||
if (!Visited.insert(MA).second) | ||
continue; | ||
|
||
if (Visited.size() > MoveAutoInitThreshold) | ||
return nullptr; | ||
|
||
bool FoundClobberingUser = false; | ||
if (auto *M = dyn_cast<MemoryUseOrDef>(MA)) { | ||
Instruction *MI = M->getMemoryInst(); | ||
|
||
// If this memory instruction may not clobber `I`, we can skip it. | ||
// LifetimeEnd is a valid user, but we do not want it in the user | ||
// dominator. | ||
if (AA.getModRefInfo(MI, ML) != ModRefInfo::NoModRef && | ||
!MI->isLifetimeStartOrEnd() && MI != I) { | ||
FoundClobberingUser = true; | ||
CurrentDominator = CurrentDominator | ||
? DT.findNearestCommonDominator(CurrentDominator, | ||
MI->getParent()) | ||
: MI->getParent(); | ||
} | ||
} | ||
if (!FoundClobberingUser) { | ||
auto UsersAsMemoryAccesses = map_range(MA->users(), AsMemoryAccess); | ||
append_range(WorkList, UsersAsMemoryAccesses); | ||
} | ||
} | ||
return CurrentDominator; | ||
} | ||
|
||
static bool runMoveAutoInit(Function &F, DominatorTree &DT, MemorySSA &MSSA) { | ||
BasicBlock &EntryBB = F.getEntryBlock(); | ||
SmallVector<std::pair<Instruction *, BasicBlock *>> JobList; | ||
|
||
// | ||
// Compute movable instructions. | ||
// | ||
for (Instruction &I : EntryBB) { | ||
if (!hasAutoInitMetadata(I)) | ||
continue; | ||
|
||
assert(!I.isVolatile() && "auto init instructions cannot be volatile."); | ||
|
||
BasicBlock *UsersDominator = usersDominator(&I, DT, MSSA); | ||
if (!UsersDominator) | ||
continue; | ||
|
||
if (UsersDominator == &EntryBB) | ||
continue; | ||
|
||
// Traverse the CFG to detect cycles `UsersDominator` would be part of. | ||
SmallPtrSet<BasicBlock *, 8> TransitiveSuccessors; | ||
SmallVector<BasicBlock *> WorkList(successors(UsersDominator)); | ||
bool HasCycle = false; | ||
while (!WorkList.empty()) { | ||
BasicBlock *CurrBB = WorkList.pop_back_val(); | ||
if (CurrBB == UsersDominator) | ||
// No early exit because we want to compute the full set of transitive | ||
// successors. | ||
HasCycle = true; | ||
for (BasicBlock *Successor : successors(CurrBB)) { | ||
if (!TransitiveSuccessors.insert(Successor).second) | ||
continue; | ||
WorkList.push_back(Successor); | ||
} | ||
} | ||
|
||
// Don't insert if that could create multiple execution of I, | ||
// but we can insert it in the non back-edge predecessors, if it exists. | ||
if (HasCycle) { | ||
BasicBlock *UsersDominatorHead = UsersDominator; | ||
while (BasicBlock *UniquePredecessor = | ||
UsersDominatorHead->getUniquePredecessor()) | ||
UsersDominatorHead = UniquePredecessor; | ||
|
||
if (UsersDominatorHead == &EntryBB) | ||
continue; | ||
|
||
BasicBlock *DominatingPredecessor = nullptr; | ||
for (BasicBlock *Pred : predecessors(UsersDominatorHead)) { | ||
// If one of the predecessor of the dominator also transitively is a | ||
// successor, moving to the dominator would do the inverse of loop | ||
// hoisting, and we don't want that. | ||
if (TransitiveSuccessors.count(Pred)) | ||
continue; | ||
|
||
DominatingPredecessor = | ||
DominatingPredecessor | ||
? DT.findNearestCommonDominator(DominatingPredecessor, Pred) | ||
: Pred; | ||
} | ||
|
||
if (!DominatingPredecessor || DominatingPredecessor == &EntryBB) | ||
continue; | ||
|
||
UsersDominator = DominatingPredecessor; | ||
} | ||
|
||
// CatchSwitchInst blocks can only have one instruction, so they are not | ||
// good candidates for insertion. | ||
while (isa<CatchSwitchInst>(UsersDominator->getFirstInsertionPt())) { | ||
for (BasicBlock *Pred : predecessors(UsersDominator)) | ||
UsersDominator = DT.findNearestCommonDominator(UsersDominator, Pred); | ||
} | ||
|
||
// We finally found a place where I can be moved while not introducing extra | ||
// execution, and guarded by at least one condition. | ||
if (UsersDominator != &EntryBB) | ||
JobList.emplace_back(&I, UsersDominator); | ||
} | ||
|
||
// | ||
// Perform the actual substitution. | ||
// | ||
if (JobList.empty()) | ||
return false; | ||
|
||
MemorySSAUpdater MSSAU(&MSSA); | ||
|
||
// Reverse insertion to respect relative order between instructions: | ||
// if two instructions are moved from the same BB to the same BB, we insert | ||
// the second one in the front, then the first on top of it. | ||
for (auto &Job : reverse(JobList)) { | ||
Job.first->moveBefore(&*Job.second->getFirstInsertionPt()); | ||
MSSAU.moveToPlace(MSSA.getMemoryAccess(Job.first), Job.first->getParent(), | ||
MemorySSA::InsertionPlace::Beginning); | ||
} | ||
|
||
if (VerifyMemorySSA) | ||
MSSA.verifyMemorySSA(); | ||
|
||
NumMoved += JobList.size(); | ||
|
||
return true; | ||
} | ||
|
||
PreservedAnalyses MoveAutoInitPass::run(Function &F, | ||
FunctionAnalysisManager &AM) { | ||
|
||
auto &DT = AM.getResult<DominatorTreeAnalysis>(F); | ||
auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA(); | ||
if (!runMoveAutoInit(F, DT, MSSA)) | ||
return PreservedAnalyses::all(); | ||
|
||
PreservedAnalyses PA; | ||
PA.preserve<DominatorTreeAnalysis>(); | ||
PA.preserve<MemorySSAAnalysis>(); | ||
PA.preserveSet<CFGAnalyses>(); | ||
return PA; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.