[SLU][profcheck] create likely branch weights for guard->branch

[mtrofin]

The llvm.experimental.guard intrinsic is a call, so its metadata - if present - would be one value (as per Verifier::visitProfMetadata). That wouldn't be a correct branch_weights metadata. Likely, GI->getMetadata(LLVMContext::MD_prof) was always nullptr.

We can bias away from deopt instead.

Issue #147390

[mtrofin]

Warning

This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack on Graphite.
Learn more

• [LSCFG][profcheck] Add dummy branch weights for the dummy switch to dead exits #164714
• [LIR][profcheck] Reuse the loop's exit condition profile #164523
• [LVer][profcheck] explicitly set unknown branch weights for the versioned/unversioned selector #164507
• [SLU][profcheck] Propagate profile for branches on injected conditions. #164476
• [SLU][profcheck] create likely branch weights for guard->branch #164271 👈 (View in Graphite)
• [NFC][SLU] Update SimpleLoopUnswitch/guards.ll #166285
• main

This stack of pull requests is managed by Graphite. Learn more about stacking.

[llvmbot]

@llvm/pr-subscribers-llvm-transforms

Author: Mircea Trofin (mtrofin)

Changes

---

Full diff: https://github.com/llvm/llvm-project/pull/164271.diff

2 Files Affected:

• (modified) llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp (+9-3)
• (modified) llvm/test/Transforms/SimpleLoopUnswitch/guards.ll (+130-51)

diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 8fb9b7edbadec..28f6af7769183 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -40,6 +40,7 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/ProfDataUtils.h"
@@ -2823,9 +2824,14 @@ static BranchInst *turnGuardIntoBranch(IntrinsicInst *GI, Loop &L,
      MSSAU->getMemorySSA()->verifyMemorySSA();
 
   DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
-  Instruction *DeoptBlockTerm =
-      SplitBlockAndInsertIfThen(GI->getArgOperand(0), GI, true,
-                                GI->getMetadata(LLVMContext::MD_prof), &DTU, &LI);
+  // llvm.experimental.guard doesn't have branch weights. We can assume,
+  // however, that the deopt path is unlikely.
+  Instruction *DeoptBlockTerm = SplitBlockAndInsertIfThen(
+      GI->getArgOperand(0), GI, true,
+      !ProfcheckDisableMetadataFixes && EstimateProfile
+          ? MDBuilder(GI->getContext()).createUnlikelyBranchWeights()
+          : nullptr,
+      &DTU, &LI);
   BranchInst *CheckBI = cast<BranchInst>(CheckBB->getTerminator());
   // SplitBlockAndInsertIfThen inserts control flow that branches to
   // DeoptBlockTerm if the condition is true.  We want the opposite.
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
index 533b1f691f5ad..e83047e397d3d 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
@@ -1,26 +1,34 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: -p --check-globals all --version 5
 ; RUN: opt -passes='loop(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
 ; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
 ; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards  -verify-memoryssa -verify-loop-info -S < %s | FileCheck %s
 
 declare void @llvm.experimental.guard(i1, ...)
 
-define void @test_simple_case(i1 %cond, i32 %N) {
-; CHECK-LABEL: @test_simple_case(
+define void @test_simple_case(i1 %cond, i32 %N) !prof !0 {
+; CHECK-LABEL: define void @test_simple_case(i1 %cond, i32 %N) !prof !0 {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split, !prof !1
 ; CHECK:       entry.split.us:
-; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK-NEXT:    br label %loop.us
 ; CHECK:       loop.us:
-; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
-; CHECK-NEXT:    br label [[GUARDED_US]]
+; CHECK-NEXT:    %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %guarded.us ]
+; CHECK-NEXT:    br label %guarded.us
 ; CHECK:       guarded.us:
-; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]]
+; CHECK-NEXT:    %iv.next.us = add i32 %iv.us, 1
+; CHECK-NEXT:    %loop.cond.us = icmp slt i32 %iv.next.us, %N
+; CHECK-NEXT:    br i1 %loop.cond.us, label %loop.us, label %exit.split.us
+; CHECK:       exit.split.us:
+; CHECK-NEXT:    br label %exit
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    br label %deopt
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
 ;
 
 entry:
@@ -38,25 +46,39 @@ exit:
 }
 
 define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
-; CHECK-LABEL: @test_two_guards(
+; CHECK-LABEL: define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split, !prof !1
 ; CHECK:       entry.split.us:
-; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK-NEXT:    br i1 %cond2, label %entry.split.us.split.us, label %entry.split.us.split, !prof !1
 ; CHECK:       entry.split.us.split.us:
-; CHECK-NEXT:    br label [[LOOP_US_US:%.*]]
+; CHECK-NEXT:    br label %loop.us.us
 ; CHECK:       loop.us.us:
-; CHECK-NEXT:    [[IV_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[IV_NEXT_US_US:%.*]], [[GUARDED_US2:%.*]] ]
-; CHECK-NEXT:    br label [[GUARDED_US_US:%.*]]
+; CHECK-NEXT:    %iv.us.us = phi i32 [ 0, %entry.split.us.split.us ], [ %iv.next.us.us, %guarded.us2 ]
+; CHECK-NEXT:    br label %guarded.us.us
 ; CHECK:       guarded.us.us:
-; CHECK-NEXT:    br label [[GUARDED_US2]]
+; CHECK-NEXT:    br label %guarded.us2
 ; CHECK:       guarded.us2:
-; CHECK-NEXT:    [[IV_NEXT_US_US]] = add i32 [[IV_US_US]], 1
-; CHECK-NEXT:    [[LOOP_COND_US_US:%.*]] = icmp slt i32 [[IV_NEXT_US_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US_US]], label [[LOOP_US_US]], label [[EXIT_SPLIT_US_SPLIT_US:%.*]]
+; CHECK-NEXT:    %iv.next.us.us = add i32 %iv.us.us, 1
+; CHECK-NEXT:    %loop.cond.us.us = icmp slt i32 %iv.next.us.us, %N
+; CHECK-NEXT:    br i1 %loop.cond.us.us, label %loop.us.us, label %exit.split.us.split.us
+; CHECK:       exit.split.us.split.us:
+; CHECK-NEXT:    br label %exit.split.us
+; CHECK:       entry.split.us.split:
+; CHECK-NEXT:    br label %loop.us
+; CHECK:       loop.us:
+; CHECK-NEXT:    br label %guarded.us
+; CHECK:       guarded.us:
+; CHECK-NEXT:    br label %deopt1
 ; CHECK:       deopt1:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
+; CHECK:       exit.split.us:
+; CHECK-NEXT:    br label %exit
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    br label %deopt
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
@@ -80,35 +102,45 @@ exit:
 }
 
 define void @test_conditional_guards(i1 %cond, i32 %N) {
-; CHECK-LABEL: @test_conditional_guards(
+; CHECK-LABEL: define void @test_conditional_guards(i1 %cond, i32 %N) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 [[COND:%.*]]
-; CHECK-NEXT:    br i1 [[FROZEN]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    %cond.fr = freeze i1 %cond
+; CHECK-NEXT:    br i1 %cond.fr, label %entry.split.us, label %entry.split, !prof !1
 ; CHECK:       entry.split.us:
-; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK-NEXT:    br label %loop.us
 ; CHECK:       loop.us:
-; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[BACKEDGE_US:%.*]] ]
-; CHECK-NEXT:    [[CONDITION_US:%.*]] = icmp eq i32 [[IV_US]], 123
-; CHECK-NEXT:    br i1 [[CONDITION_US]], label [[GUARD_US:%.*]], label [[BACKEDGE_US]]
+; CHECK-NEXT:    %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %backedge.us ]
+; CHECK-NEXT:    %condition.us = icmp eq i32 %iv.us, 123
+; CHECK-NEXT:    br i1 %condition.us, label %guard.us, label %backedge.us
 ; CHECK:       guard.us:
-; CHECK-NEXT:    br label [[GUARDED_US:%.*]]
+; CHECK-NEXT:    br label %guarded.us
 ; CHECK:       backedge.us:
-; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]]
+; CHECK-NEXT:    %iv.next.us = add i32 %iv.us, 1
+; CHECK-NEXT:    %loop.cond.us = icmp slt i32 %iv.next.us, %N
+; CHECK-NEXT:    br i1 %loop.cond.us, label %loop.us, label %exit.split.us
+; CHECK:       guarded.us:
+; CHECK-NEXT:    br label %backedge.us
+; CHECK:       exit.split.us:
+; CHECK-NEXT:    br label %exit
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    [[CONDITION:%.*]] = icmp eq i32 [[IV]], 123
-; CHECK-NEXT:    br i1 [[CONDITION]], label [[GUARD:%.*]], label [[BACKEDGE]]
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry.split ], [ %iv.next, %backedge ]
+; CHECK-NEXT:    %condition = icmp eq i32 %iv, 123
+; CHECK-NEXT:    br i1 %condition, label %guard, label %backedge
 ; CHECK:       guard:
-; CHECK-NEXT:    br label [[DEOPT:%.*]]
+; CHECK-NEXT:    br label %deopt
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       backedge:
-; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label %loop, label [[EXIT_SPLIT:%.*]]
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    %loop.cond = icmp slt i32 %iv.next, %N
+; CHECK-NEXT:    br i1 %loop.cond, label %loop, label %exit.split
+; CHECK:       exit.split:
+; CHECK-NEXT:    br label %exit
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
 ;
 
 entry:
@@ -135,7 +167,7 @@ exit:
 define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) {
 ; CHECK-LABEL: define void @test_nested_loop(i1 %cond, i32 %N, i1 %arg) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split, label %outer_loop.split
+; CHECK-NEXT:    br i1 %cond, label %entry.split, label %outer_loop.split, !prof !1
 ; CHECK:       entry.split:
 ; CHECK-NEXT:    br i1 %arg, label %entry.split.split.us, label %entry.split.split
 ; CHECK:       entry.split.split.us:
@@ -204,17 +236,49 @@ exit:
 }
 
 define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) {
-; CHECK-LABEL: @test_sibling_loops(
+; CHECK-LABEL: define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
-; CHECK:         [[IV1_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV1_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
-; CHECK-NEXT:    br label [[GUARDED_US]]
-; CHECK:         call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split, !prof !1
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop1.us
+; CHECK:       loop1.us:
+; CHECK-NEXT:    %iv1.us = phi i32 [ 0, %entry.split.us ], [ %iv1.next.us, %guarded.us ]
+; CHECK-NEXT:    br label %guarded.us
+; CHECK:       guarded.us:
+; CHECK-NEXT:    %iv1.next.us = add i32 %iv1.us, 1
+; CHECK-NEXT:    %loop1.cond.us = icmp slt i32 %iv1.next.us, %N
+; CHECK-NEXT:    br i1 %loop1.cond.us, label %loop1.us, label %between.split.us
+; CHECK:       between.split.us:
+; CHECK-NEXT:    br label %between
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop1
+; CHECK:       loop1:
+; CHECK-NEXT:    br label %deopt
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
-; CHECK:         [[IV2_US:%.*]] = phi i32 [ 0, [[BETWEEN:%.*]] ], [ [[IV1_NEXT_US2:%.*]], [[GUARDED_US2:%.*]] ]
-; CHECK-NEXT:    br label [[GUARDED_US2]]
-; CHECK:         call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK:       between:
+; CHECK-NEXT:    br i1 %cond2, label %between.split.us2, label %between.split, !prof !1
+; CHECK:       between.split.us2:
+; CHECK-NEXT:    br label %loop2.us
+; CHECK:       loop2.us:
+; CHECK-NEXT:    %iv2.us = phi i32 [ 0, %between.split.us2 ], [ %iv2.next.us, %guarded.us3 ]
+; CHECK-NEXT:    br label %guarded.us3
+; CHECK:       guarded.us3:
+; CHECK-NEXT:    %iv2.next.us = add i32 %iv2.us, 1
+; CHECK-NEXT:    %loop2.cond.us = icmp slt i32 %iv2.next.us, %N
+; CHECK-NEXT:    br i1 %loop2.cond.us, label %loop2.us, label %exit.split.us
+; CHECK:       exit.split.us:
+; CHECK-NEXT:    br label %exit
+; CHECK:       between.split:
+; CHECK-NEXT:    br label %loop2
+; CHECK:       loop2:
+; CHECK-NEXT:    br label %deopt1
+; CHECK:       deopt1:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
 ;
 
 entry:
@@ -242,11 +306,20 @@ exit:
 }
 
 ; Check that we don't do anything because of cleanuppad.
-; CHECK-LABEL: @test_cleanuppad(
-; CHECK:       call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
-; CHECK-NOT:   call void (i1, ...) @llvm.experimental.guard(
 define void @test_cleanuppad(i1 %cond, i32 %N) personality ptr @__CxxFrameHandler3 {
-
+; CHECK-LABEL: define void @test_cleanuppad(i1 %cond, i32 %N) personality ptr @__CxxFrameHandler3 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+; CHECK-NEXT:    %iv.next = add i32 %iv, 1
+; CHECK-NEXT:    invoke void @may_throw(i32 %iv)
+; CHECK-NEXT:            to label %loop unwind label %exit
+; CHECK:       exit:
+; CHECK-NEXT:    %cp = cleanuppad within none []
+; CHECK-NEXT:    cleanupret from %cp unwind to caller
+;
 entry:
   br label %loop
 
@@ -264,3 +337,9 @@ exit:
 
 declare void @may_throw(i32 %i)
 declare i32 @__CxxFrameHandler3(...)
+
+!0 = !{!"function_entry_count", i32 10}
+;.
+; CHECK: !0 = !{!"function_entry_count", i32 10}
+; CHECK: !1 = !{!"branch_weights", i32 1048575, i32 1}
+;.