diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll index 1e9d5d4d2629c..398b65116bbd7 100644 --- a/clang/test/CodeGen/thinlto-distributed-newpm.ll +++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll @@ -31,6 +31,7 @@ ; CHECK-O: Running analysis: OptimizationRemarkEmitterAnalysis on main ; CHECK-O: Running pass: InferFunctionAttrsPass ; CHECK-O: Starting {{.*}}Function pass manager run. +; CHECK-O: Running pass: LowerExpectIntrinsicPass on main ; CHECK-O: Running pass: SimplifyCFGPass on main ; CHECK-O: Running analysis: TargetIRAnalysis on main ; CHECK-O: Running analysis: AssumptionAnalysis on main @@ -38,7 +39,6 @@ ; CHECK-O: Running analysis: DominatorTreeAnalysis on main ; CHECK-O: Running pass: EarlyCSEPass on main ; CHECK-O: Running analysis: TargetLibraryAnalysis on main -; CHECK-O: Running pass: LowerExpectIntrinsicPass on main ; CHECK-O3: Running pass: CallSiteSplittingPass on main ; CHECK-O: Finished {{.*}}Function pass manager run. ; CHECK-O: Running pass: LowerTypeTestsPass diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 6307e468e7017..75ba9da4214d7 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -1066,10 +1066,12 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // Create an early function pass manager to cleanup the output of the // frontend. FunctionPassManager EarlyFPM(DebugLogging); + // Lower llvm.expect to metadata before attempting transforms. + // Compare/branch metadata may alter the behavior of passes like SimplifyCFG. + EarlyFPM.addPass(LowerExpectIntrinsicPass()); EarlyFPM.addPass(SimplifyCFGPass()); EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); - EarlyFPM.addPass(LowerExpectIntrinsicPass()); if (PTO.Coroutines) EarlyFPM.addPass(CoroEarlyPass()); if (Level == OptimizationLevel::O3) diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp index 2c80a16febeff..19e212f738ade 100644 --- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -316,10 +316,12 @@ void PassManagerBuilder::populateFunctionPassManager( addInitialAliasAnalysisPasses(FPM); + // Lower llvm.expect to metadata before attempting transforms. + // Compare/branch metadata may alter the behavior of passes like SimplifyCFG. + FPM.add(createLowerExpectIntrinsicPass()); FPM.add(createCFGSimplificationPass()); FPM.add(createSROAPass()); FPM.add(createEarlyCSEPass()); - FPM.add(createLowerExpectIntrinsicPass()); } // Do PGO instrumentation generation or use pass as the option specified. diff --git a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll index 5a05310177bb6..3ba9099176d8f 100644 --- a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll @@ -41,11 +41,11 @@ ; GCN-O1-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O1-NEXT: Function Alias Analysis Results ; GCN-O1-NEXT: Simplify well-known AMD library calls +; GCN-O1-NEXT: Lower 'expect' Intrinsics ; GCN-O1-NEXT: Simplify the CFG ; GCN-O1-NEXT: Dominator Tree Construction ; GCN-O1-NEXT: SROA ; GCN-O1-NEXT: Early CSE -; GCN-O1-NEXT: Lower 'expect' Intrinsics ; GCN-O1-NEXT: Pass Arguments: ; GCN-O1-NEXT: Target Library Information @@ -350,11 +350,11 @@ ; GCN-O2-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O2-NEXT: Function Alias Analysis Results ; GCN-O2-NEXT: Simplify well-known AMD library calls +; GCN-O2-NEXT: Lower 'expect' Intrinsics ; GCN-O2-NEXT: Simplify the CFG ; GCN-O2-NEXT: Dominator Tree Construction ; GCN-O2-NEXT: SROA ; GCN-O2-NEXT: Early CSE -; GCN-O2-NEXT: Lower 'expect' Intrinsics ; GCN-O2-NEXT: Pass Arguments: ; GCN-O2-NEXT: Target Library Information @@ -704,11 +704,11 @@ ; GCN-O3-NEXT: Basic Alias Analysis (stateless AA impl) ; GCN-O3-NEXT: Function Alias Analysis Results ; GCN-O3-NEXT: Simplify well-known AMD library calls +; GCN-O3-NEXT: Lower 'expect' Intrinsics ; GCN-O3-NEXT: Simplify the CFG ; GCN-O3-NEXT: Dominator Tree Construction ; GCN-O3-NEXT: SROA ; GCN-O3-NEXT: Early CSE -; GCN-O3-NEXT: Lower 'expect' Intrinsics ; GCN-O3-NEXT: Pass Arguments: ; GCN-O3-NEXT: Target Library Information diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index 8dc54e4ccb84e..ae268fd0c3a36 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -79,6 +79,7 @@ ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Running analysis: PreservedCFGCheckerAnalysis on foo ; CHECK-O-NEXT: Starting llvm::Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -86,7 +87,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished llvm::Function pass manager run. ; CHECK-EP-PIPELINE-EARLY-SIMPLIFICATION-NEXT: Running pass: NoOpModulePass diff --git a/llvm/test/Other/new-pm-pgo.ll b/llvm/test/Other/new-pm-pgo.ll index 09fd103e172b5..b2f6b666150a2 100644 --- a/llvm/test/Other/new-pm-pgo.ll +++ b/llvm/test/Other/new-pm-pgo.ll @@ -20,10 +20,10 @@ ; USE_POST_LINK: Running pass: PGOMemOPSizeOpt ; SAMPLE_USE_O: Running pass: AddDiscriminatorsPass ; SAMPLE_USE_PRE_LINK: Running pass: AddDiscriminatorsPass +; SAMPLE_USE: Running pass: LowerExpectIntrinsicPass ; SAMPLE_USE: Running pass: SimplifyCFGPass ; SAMPLE_USE: Running pass: SROA ; SAMPLE_USE: Running pass: EarlyCSEPass -; SAMPLE_USE: Running pass: LowerExpectIntrinsicPass ; SAMPLE_USE_POST_LINK: Running pass: InstCombinePass ; SAMPLE_USE: Running pass: SampleProfileLoaderPass ; SAMPLE_USE_O: Running pass: PGOIndirectCallPromotion diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll index 46620aea31db1..ceb024074796e 100644 --- a/llvm/test/Other/new-pm-thinlto-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-defaults.ll @@ -64,6 +64,7 @@ ; CHECK-PRELINK-O-NODIS-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting llvm::Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -71,7 +72,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished llvm::Function pass manager run. ; CHECK-POSTLINK-O-NEXT: Running pass: LowerTypeTestsPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index ed0e17175fd0c..77a556c278b48 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -34,6 +34,7 @@ ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -41,7 +42,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run. ; CHECK-O-NEXT: Running pass: LowerTypeTestsPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index 05e7a887271e1..b66b28f266786 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -36,6 +36,7 @@ ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -43,7 +44,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: InstCombinePass on foo ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll index e1bab9c6fc7cb..0d7a1c44e0c2a 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -35,6 +35,7 @@ ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -42,7 +43,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run. ; CHECK-O-NEXT: Running pass: IPSCCPPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll index c2412ff4073d6..e577e837e8e74 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -34,6 +34,7 @@ ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis ; CHECK-O-NEXT: Starting {{.*}}Function pass manager run. +; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O-NEXT: Running pass: SimplifyCFGPass ; CHECK-O-NEXT: Running analysis: TargetIRAnalysis ; CHECK-O-NEXT: Running analysis: AssumptionAnalysis @@ -41,7 +42,6 @@ ; CHECK-O-NEXT: Running analysis: DominatorTreeAnalysis ; CHECK-O-NEXT: Running pass: EarlyCSEPass ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis -; CHECK-O-NEXT: Running pass: LowerExpectIntrinsicPass ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass ; CHECK-O-NEXT: Running pass: InstCombinePass on foo ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll index af7093b0bd53f..36370d6cd6d39 100644 --- a/llvm/test/Other/opt-O2-pipeline.ll +++ b/llvm/test/Other/opt-O2-pipeline.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll b/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll index 12b5e188e508a..e5d9e51402354 100644 --- a/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll +++ b/llvm/test/Other/opt-O3-pipeline-enable-matrix.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll index bee06f816a532..af77e09c24968 100644 --- a/llvm/test/Other/opt-O3-pipeline.ll +++ b/llvm/test/Other/opt-O3-pipeline.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll index 5d7ce952a78e8..e24d3d3f57bfc 100644 --- a/llvm/test/Other/opt-Os-pipeline.ll +++ b/llvm/test/Other/opt-Os-pipeline.ll @@ -12,11 +12,11 @@ ; CHECK-NEXT: Module Verifier ; CHECK-EXT: Good Bye World Pass ; CHECK-NOEXT-NOT: Good Bye World Pass +; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Simplify the CFG ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: SROA ; CHECK-NEXT: Early CSE -; CHECK-NEXT: Lower 'expect' Intrinsics ; CHECK-NEXT: Pass Arguments: ; CHECK-NEXT: Target Library Information ; CHECK-NEXT: Target Transform Information diff --git a/llvm/test/Transforms/PhaseOrdering/expect.ll b/llvm/test/Transforms/PhaseOrdering/expect.ll index 14a9284baa5eb..3a23785ddff78 100644 --- a/llvm/test/Transforms/PhaseOrdering/expect.ll +++ b/llvm/test/Transforms/PhaseOrdering/expect.ll @@ -1,7 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -O2 -S < %s | FileCheck %s -; FIXME: ; Given an "expect" on a compare, we should not combine ; that compare with other instructions in a way that the ; backend can't undo. Expect lowering becomes metadata, @@ -10,10 +9,11 @@ define void @PR49336(i32 %delta, i32 %tag_type, i8* %ip) { ; CHECK-LABEL: @PR49336( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TOBOOL:%.*]] = icmp slt i32 [[DELTA:%.*]], 0 -; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i32 [[TAG_TYPE:%.*]], 0 -; CHECK-NEXT: [[OR_COND:%.*]] = and i1 [[TOBOOL]], [[CMP1]] -; CHECK-NEXT: br i1 [[OR_COND]], label [[IF_THEN2:%.*]], label [[IF_END3:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[DELTA:%.*]], 0 +; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END3:%.*]], !prof !0 +; CHECK: if.then: +; CHECK-NEXT: [[CMP1_NOT:%.*]] = icmp eq i32 [[TAG_TYPE:%.*]], 0 +; CHECK-NEXT: br i1 [[CMP1_NOT]], label [[IF_END3]], label [[IF_THEN2:%.*]] ; CHECK: if.then2: ; CHECK-NEXT: store i8 42, i8* [[IP:%.*]], align 1 ; CHECK-NEXT: br label [[IF_END3]]