diff --git a/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h b/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h index 980b55f46f114..e8565e6942bc2 100644 --- a/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h +++ b/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h @@ -24,6 +24,12 @@ class MoveAutoInitPass : public PassInfoMixin { public: PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + +/// Move entry block initializations of allocas closer to their guarded users. +class MoveEntryAllocaInitPass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index bd03ac090721c..470bc0afa22d7 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -765,6 +765,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, FPM.addPass(DSEPass()); FPM.addPass(MoveAutoInitPass()); + FPM.addPass(MoveEntryAllocaInitPass()); FPM.addPass(createFunctionToLoopPassAdaptor( LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap, @@ -2140,6 +2141,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, // Nuke dead stores. MainFPM.addPass(DSEPass()); MainFPM.addPass(MoveAutoInitPass()); + MainFPM.addPass(MoveEntryAllocaInitPass()); MainFPM.addPass(MergedLoadStoreMotionPass()); invokeVectorizerStartEPCallbacks(MainFPM, Level); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 1853cdd45d0ee..7950040cd3623 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -485,6 +485,7 @@ FUNCTION_PASS("memprof", MemProfilerPass()) FUNCTION_PASS("mergeicmps", MergeICmpsPass()) FUNCTION_PASS("mergereturn", UnifyFunctionExitNodesPass()) FUNCTION_PASS("move-auto-init", MoveAutoInitPass()) +FUNCTION_PASS("move-entry-alloca-init", MoveEntryAllocaInitPass()) FUNCTION_PASS("nary-reassociate", NaryReassociatePass()) FUNCTION_PASS("newgvn", NewGVNPass()) FUNCTION_PASS("no-op-function", NoOpFunctionPass()) diff --git a/llvm/lib/Transforms/Utils/MoveAutoInit.cpp b/llvm/lib/Transforms/Utils/MoveAutoInit.cpp index ad105f5a57b49..0881ea12c2478 100644 --- a/llvm/lib/Transforms/Utils/MoveAutoInit.cpp +++ b/llvm/lib/Transforms/Utils/MoveAutoInit.cpp @@ -101,7 +101,9 @@ static BasicBlock *usersDominator(const MemoryLocation &ML, Instruction *I, return CurrentDominator; } -static bool runMoveAutoInit(Function &F, DominatorTree &DT, MemorySSA &MSSA) { +static bool +runMoveAutoInit(Function &F, DominatorTree &DT, MemorySSA &MSSA, + function_ref ShouldProcess) { BasicBlock &EntryBB = F.getEntryBlock(); SmallVector> JobList; @@ -109,7 +111,7 @@ static bool runMoveAutoInit(Function &F, DominatorTree &DT, MemorySSA &MSSA) { // Compute movable instructions. // for (Instruction &I : EntryBB) { - if (!hasAutoInitMetadata(I)) + if (!ShouldProcess(I)) continue; std::optional ML = writeToAlloca(I); @@ -221,7 +223,26 @@ PreservedAnalyses MoveAutoInitPass::run(Function &F, auto &DT = AM.getResult(F); auto &MSSA = AM.getResult(F).getMSSA(); - if (!runMoveAutoInit(F, DT, MSSA)) + auto ShouldProcess = [](const Instruction &I) -> bool { + return hasAutoInitMetadata(I); + }; + if (!runMoveAutoInit(F, DT, MSSA, ShouldProcess)) + return PreservedAnalyses::all(); + + PreservedAnalyses PA; + PA.preserve(); + PA.preserve(); + PA.preserveSet(); + return PA; +} + +PreservedAnalyses MoveEntryAllocaInitPass::run(Function &F, + FunctionAnalysisManager &AM) { + + auto &DT = AM.getResult(F); + auto &MSSA = AM.getResult(F).getMSSA(); + auto ShouldProcess = [](const Instruction &) { return true; }; + if (!runMoveAutoInit(F, DT, MSSA, ShouldProcess)) return PreservedAnalyses::all(); PreservedAnalyses PA; diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index 65b96c8b8ef5d..0bc59cbffa594 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -218,6 +218,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll index f595dfe1d6845..4ce7a968fe3b2 100644 --- a/llvm/test/Other/new-pm-lto-defaults.ll +++ b/llvm/test/Other/new-pm-lto-defaults.ll @@ -118,6 +118,7 @@ ; CHECK-O23SZ-NEXT: Running analysis: PostDominatorTreeAnalysis on foo ; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo ; CHECK-EP-VECTORIZER-START-NEXT: Running pass: NoOpFunctionPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo diff --git a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll index 3a0fffe426da1..6f94522401396 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll @@ -143,6 +143,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass on loop diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index 4623edcaf6656..05321565e2c2d 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -127,6 +127,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index 590afd925e841..c0a6b3851b376 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -136,6 +136,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll index dd6acd2c51ee7..280f36749850c 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll @@ -175,6 +175,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass on loop diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll index ee054527e20bd..b719045f890a4 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -176,6 +176,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll index fd95e94f3c8b9..3d73e3ca2c2b7 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -140,6 +140,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass ; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo +; CHECK-O23SZ-NEXT: Running pass: MoveEntryAllocaInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Transforms/MoveAutoInit/entry-alloca.ll b/llvm/test/Transforms/MoveAutoInit/entry-alloca.ll new file mode 100644 index 0000000000000..31906551e60fc --- /dev/null +++ b/llvm/test/Transforms/MoveAutoInit/entry-alloca.ll @@ -0,0 +1,50 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -passes='move-entry-alloca-init' -verify-memoryssa -S < %s | FileCheck %s + +declare void @bar(ptr) + +define void @src(i32 %a, i32 %b) { +; CHECK-LABEL: define void @src( +; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) { +; CHECK-NEXT: [[A_ADDR:%.*]] = alloca i32 +; CHECK-NEXT: [[B_ADDR:%.*]] = alloca i32 +; CHECK-NEXT: [[S_ADDR:%.*]] = alloca { ptr, ptr } +; CHECK: [[S_X:%.*]] = getelementptr inbounds { ptr, ptr }, ptr [[S_ADDR]], i32 0, i32 0 +; CHECK: [[S_Y:%.*]] = getelementptr inbounds { ptr, ptr }, ptr [[S_ADDR]], i32 0, i32 1 +; CHECK: [[CMP:%.*]] = icmp slt i32 [[A]], [[B]] +; CHECK: br i1 [[CMP]], label [[COLD:%.*]], label [[RET:%.*]] +; +; CHECK: cold: +; CHECK-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4 +; CHECK-NEXT: store i32 [[B]], ptr [[B_ADDR]], align 4 +; CHECK-NEXT: store ptr [[A_ADDR]], ptr [[S_X]], align 8 +; CHECK-NEXT: store ptr [[B_ADDR]], ptr [[S_Y]], align 8 +; CHECK-NEXT: call void @bar(ptr [[S_ADDR]]) +; CHECK-NEXT: br label [[RET]] +; +; CHECK: ret: +; CHECK-NEXT: ret void +; + %a.addr = alloca i32 + %b.addr = alloca i32 + %s.addr = alloca { ptr, ptr } + + store i32 %a, ptr %a.addr + store i32 %b, ptr %b.addr + + %s.x = getelementptr inbounds { ptr, ptr }, ptr %s.addr, i32 0, i32 0 + store ptr %a.addr, ptr %s.x + + %s.y = getelementptr inbounds { ptr, ptr }, ptr %s.addr, i32 0, i32 1 + store ptr %b.addr, ptr %s.y + + %cmp = icmp slt i32 %a, %b + br i1 %cmp, label %cold, label %ret + +cold: + call void @bar(ptr %s.addr) + br label %ret + +ret: + ret void +}