diff --git a/llvm/lib/Transforms/Scalar/LoopPredication.cpp b/llvm/lib/Transforms/Scalar/LoopPredication.cpp index 4f97641e2027c8..49c5b9632c003e 100644 --- a/llvm/lib/Transforms/Scalar/LoopPredication.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPredication.cpp @@ -183,6 +183,8 @@ #include "llvm/Analysis/GuardUtils.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopPass.h" +#include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/MemorySSAUpdater.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/IR/Function.h" @@ -255,6 +257,7 @@ class LoopPredication { ScalarEvolution *SE; LoopInfo *LI; BranchProbabilityInfo *BPI; + MemorySSAUpdater *MSSAU; Loop *L; const DataLayout *DL; @@ -308,10 +311,10 @@ class LoopPredication { bool predicateLoopExits(Loop *L, SCEVExpander &Rewriter); public: - LoopPredication(AliasAnalysis *AA, DominatorTree *DT, - ScalarEvolution *SE, LoopInfo *LI, - BranchProbabilityInfo *BPI) - : AA(AA), DT(DT), SE(SE), LI(LI), BPI(BPI) {}; + LoopPredication(AliasAnalysis *AA, DominatorTree *DT, ScalarEvolution *SE, + LoopInfo *LI, BranchProbabilityInfo *BPI, + MemorySSAUpdater *MSSAU) + : AA(AA), DT(DT), SE(SE), LI(LI), BPI(BPI), MSSAU(MSSAU){}; bool runOnLoop(Loop *L); }; @@ -325,6 +328,7 @@ class LoopPredicationLegacyPass : public LoopPass { void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); getLoopAnalysisUsage(AU); + AU.addPreserved(); } bool runOnLoop(Loop *L, LPPassManager &LPM) override { @@ -333,10 +337,14 @@ class LoopPredicationLegacyPass : public LoopPass { auto *SE = &getAnalysis().getSE(); auto *LI = &getAnalysis().getLoopInfo(); auto *DT = &getAnalysis().getDomTree(); + auto *MSSAWP = getAnalysisIfAvailable(); + std::unique_ptr MSSAU; + if (MSSAWP) + MSSAU = std::make_unique(&MSSAWP->getMSSA()); BranchProbabilityInfo &BPI = getAnalysis().getBPI(); auto *AA = &getAnalysis().getAAResults(); - LoopPredication LP(AA, DT, SE, LI, &BPI); + LoopPredication LP(AA, DT, SE, LI, &BPI, MSSAU ? MSSAU.get() : nullptr); return LP.runOnLoop(L); } }; @@ -363,11 +371,18 @@ PreservedAnalyses LoopPredicationPass::run(Loop &L, LoopAnalysisManager &AM, // pass. Function analyses need to be preserved across loop transformations // but BPI is not preserved, hence a newly built one is needed. BranchProbabilityInfo BPI(*F, AR.LI, &AR.TLI, &AR.DT, nullptr); - LoopPredication LP(&AR.AA, &AR.DT, &AR.SE, &AR.LI, &BPI); + std::unique_ptr MSSAU; + if (AR.MSSA) + MSSAU = std::make_unique(AR.MSSA); + LoopPredication LP(&AR.AA, &AR.DT, &AR.SE, &AR.LI, &BPI, + MSSAU ? MSSAU.get() : nullptr); if (!LP.runOnLoop(&L)) return PreservedAnalyses::all(); - return getLoopPassPreservedAnalyses(); + auto PA = getLoopPassPreservedAnalyses(); + if (AR.MSSA) + PA.preserve(); + return PA; } Optional @@ -809,7 +824,7 @@ bool LoopPredication::widenGuardConditions(IntrinsicInst *Guard, Value *AllChecks = Builder.CreateAnd(Checks); auto *OldCond = Guard->getOperand(0); Guard->setOperand(0, AllChecks); - RecursivelyDeleteTriviallyDeadInstructions(OldCond); + RecursivelyDeleteTriviallyDeadInstructions(OldCond, nullptr /* TLI */, MSSAU); LLVM_DEBUG(dbgs() << "Widened checks = " << NumWidened << "\n"); return true; @@ -835,7 +850,7 @@ bool LoopPredication::widenWidenableBranchGuardConditions( Value *AllChecks = Builder.CreateAnd(Checks); auto *OldCond = BI->getCondition(); BI->setCondition(AllChecks); - RecursivelyDeleteTriviallyDeadInstructions(OldCond); + RecursivelyDeleteTriviallyDeadInstructions(OldCond, nullptr /* TLI */, MSSAU); assert(isGuardAsWidenableBranch(BI) && "Stopped being a guard after transform?"); @@ -1242,5 +1257,8 @@ bool LoopPredication::runOnLoop(Loop *Loop) { for (auto *Guard : GuardsAsWidenableBranches) Changed |= widenWidenableBranchGuardConditions(Guard, Expander); Changed |= predicateLoopExits(L, Expander); + + if (MSSAU && VerifyMemorySSA) + MSSAU->getMemorySSA()->verifyMemorySSA(); return Changed; } diff --git a/llvm/test/Transforms/LoopPredication/basic.ll b/llvm/test/Transforms/LoopPredication/basic.ll index 4a9cf671319333..e9cbdb4f9600e3 100644 --- a/llvm/test/Transforms/LoopPredication/basic.ll +++ b/llvm/test/Transforms/LoopPredication/basic.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication < %s 2>&1 | FileCheck %s -; RUN: opt -S -passes='require,loop(loop-predication)' < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) diff --git a/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll b/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll index 70cf1e9dd8d69b..70c5f8232091dc 100644 --- a/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll +++ b/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s -; RUN: opt -S -passes='require,loop(loop-predication)' -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s -; RUN: opt -S -passes='require,require,loop(loop-predication)' -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,require,loop-mssa(loop-predication)' -verify-memoryssa -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) diff --git a/llvm/test/Transforms/LoopPredication/invariant_load.ll b/llvm/test/Transforms/LoopPredication/invariant_load.ll index 1002d970818518..4be516d8e57b06 100644 --- a/llvm/test/Transforms/LoopPredication/invariant_load.ll +++ b/llvm/test/Transforms/LoopPredication/invariant_load.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -basic-aa -loop-predication < %s 2>&1 | FileCheck %s -; RUN: opt -S -aa-pipeline=basic-aa -passes='require,require,loop(loop-predication)' < %s 2>&1 | FileCheck %s +; RUN: opt -S -aa-pipeline=basic-aa -passes='require,require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) diff --git a/llvm/test/Transforms/LoopPredication/nested.ll b/llvm/test/Transforms/LoopPredication/nested.ll index 9c25738667451e..62be706ae16820 100644 --- a/llvm/test/Transforms/LoopPredication/nested.ll +++ b/llvm/test/Transforms/LoopPredication/nested.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication < %s 2>&1 | FileCheck %s -; RUN: opt -S -passes='require,loop(loop-predication)' < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) diff --git a/llvm/test/Transforms/LoopPredication/predicate-exits.ll b/llvm/test/Transforms/LoopPredication/predicate-exits.ll index 01c78580ca9e63..326c967d80f39f 100644 --- a/llvm/test/Transforms/LoopPredication/predicate-exits.ll +++ b/llvm/test/Transforms/LoopPredication/predicate-exits.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -loop-predication -S | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s declare void @prevent_merging() diff --git a/llvm/test/Transforms/LoopPredication/profitability.ll b/llvm/test/Transforms/LoopPredication/profitability.ll index 5f17df9e0a13fa..7451aa1fcc9b35 100644 --- a/llvm/test/Transforms/LoopPredication/profitability.ll +++ b/llvm/test/Transforms/LoopPredication/profitability.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication -loop-predication-skip-profitability-checks=false < %s 2>&1 | FileCheck %s -; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require,require,loop(loop-predication)' < %s 2>&1 | FileCheck %s +; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require,require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s ; latch block exits to a speculation block. BPI already knows (without prof ; data) that deopt is very rarely diff --git a/llvm/test/Transforms/LoopPredication/reverse.ll b/llvm/test/Transforms/LoopPredication/reverse.ll index 95bfe20c04e371..2a6d9fac6fe365 100644 --- a/llvm/test/Transforms/LoopPredication/reverse.ll +++ b/llvm/test/Transforms/LoopPredication/reverse.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication -loop-predication-enable-count-down-loop=true < %s 2>&1 | FileCheck %s -; RUN: opt -S -passes='require,loop(loop-predication)' -loop-predication-enable-count-down-loop=true < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -loop-predication-enable-count-down-loop=true < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) diff --git a/llvm/test/Transforms/LoopPredication/unswitch-exit-loop.ll b/llvm/test/Transforms/LoopPredication/unswitch-exit-loop.ll index 40ced7d13c65da..0e03fa7c6d6543 100644 --- a/llvm/test/Transforms/LoopPredication/unswitch-exit-loop.ll +++ b/llvm/test/Transforms/LoopPredication/unswitch-exit-loop.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -loop-predication -S | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s ;; This is a simplified copy of @unswitch_exit_form test that should trigger loop-predication ;; activity and properly bail out when discovering that widenable check does not lead to deopt. diff --git a/llvm/test/Transforms/LoopPredication/visited.ll b/llvm/test/Transforms/LoopPredication/visited.ll index 8d40386e025d28..6120ecb2e8338b 100644 --- a/llvm/test/Transforms/LoopPredication/visited.ll +++ b/llvm/test/Transforms/LoopPredication/visited.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication < %s 2>&1 | FileCheck %s -; RUN: opt -S -passes='require,loop(loop-predication)' < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) diff --git a/llvm/test/Transforms/LoopPredication/widened.ll b/llvm/test/Transforms/LoopPredication/widened.ll index 38325cbec50c60..0cb592108136f4 100644 --- a/llvm/test/Transforms/LoopPredication/widened.ll +++ b/llvm/test/Transforms/LoopPredication/widened.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -S -loop-predication -loop-predication-enable-iv-truncation=true < %s 2>&1 | FileCheck %s +; RUN: opt -S -passes='require,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s declare void @llvm.experimental.guard(i1, ...) declare i32 @length(i8*)