59 changes: 46 additions & 13 deletions llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,36 @@ exit:
ret void
}


define void @or_icmps_probably_not_harmful(i32 %x, i32 %y, i8* %p) {
; CHECK-LABEL: @or_icmps_probably_not_harmful(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF21:![0-9]+]], !unpredictable !22
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
entry:
%expected_true = icmp sgt i32 %x, -1
br i1 %expected_true, label %exit, label %rare, !prof !15, !unpredictable !20

rare:
%expensive = icmp eq i32 %y, 0
br i1 %expensive, label %exit, label %false

false:
store i8 42, i8* %p, align 1
br label %exit

exit:
ret void
}

; The probability threshold is determined by a TTI setting.
; In this example, we are just short of strongly expected, so speculate.

Expand All @@ -737,7 +767,7 @@ define void @or_icmps_not_that_harmful(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF21:![0-9]+]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF23:![0-9]+]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -769,7 +799,7 @@ define void @or_icmps_not_that_harmful_inverted(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF22:![0-9]+]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF24:![0-9]+]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -800,7 +830,7 @@ define void @or_icmps_useful(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_TRUE:%.*]] = icmp sle i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 true, i1 [[EXPENSIVE]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF23:![0-9]+]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF25:![0-9]+]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -831,7 +861,7 @@ define void @or_icmps_useful_inverted(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_FALSE:%.*]] = icmp sgt i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_FALSE]], i1 true, i1 [[EXPENSIVE]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF23]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[EXIT:%.*]], label [[FALSE:%.*]], !prof [[PROF25]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -960,7 +990,7 @@ define void @and_icmps_not_that_harmful(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_FALSE:%.*]] = icmp sgt i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_FALSE]], i1 [[EXPENSIVE]], i1 false
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF24:![0-9]+]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF26:![0-9]+]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -992,7 +1022,7 @@ define void @and_icmps_not_that_harmful_inverted(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_TRUE:%.*]] = icmp sle i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 [[EXPENSIVE]], i1 false
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF24]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF26]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -1023,7 +1053,7 @@ define void @and_icmps_useful(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_TRUE:%.*]] = icmp sgt i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_TRUE]], i1 [[EXPENSIVE]], i1 false
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF25:![0-9]+]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF27:![0-9]+]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -1054,7 +1084,7 @@ define void @and_icmps_useful_inverted(i32 %x, i32 %y, i8* %p) {
; CHECK-NEXT: [[EXPECTED_FALSE:%.*]] = icmp sle i32 [[X:%.*]], -1
; CHECK-NEXT: [[EXPENSIVE:%.*]] = icmp eq i32 [[Y:%.*]], 0
; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[EXPECTED_FALSE]], i1 [[EXPENSIVE]], i1 false
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF25]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[FALSE:%.*]], label [[EXIT:%.*]], !prof [[PROF27]]
; CHECK: false:
; CHECK-NEXT: store i8 42, i8* [[P:%.*]], align 1
; CHECK-NEXT: br label [[EXIT]]
Expand Down Expand Up @@ -1098,6 +1128,7 @@ exit:
!17 = !{!"branch_weights", i32 98, i32 1}
!18 = !{!"branch_weights", i32 1, i32 98}
!19 = !{!"branch_weights", i32 0, i32 0}
!20 = !{}

;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind uwtable }
Expand Down Expand Up @@ -1125,9 +1156,11 @@ exit:
; CHECK: [[PROF18]] = !{!"branch_weights", i32 8, i32 2}
; CHECK: [[PROF19]] = !{!"branch_weights", i32 99, i32 1}
; CHECK: [[PROF20]] = !{!"branch_weights", i32 1, i32 99}
; CHECK: [[PROF21]] = !{!"branch_weights", i32 197, i32 1}
; CHECK: [[PROF22]] = !{!"branch_weights", i32 100, i32 98}
; CHECK: [[PROF23]] = !{!"branch_weights", i32 101, i32 99}
; CHECK: [[PROF24]] = !{!"branch_weights", i32 1, i32 197}
; CHECK: [[PROF25]] = !{!"branch_weights", i32 99, i32 101}
; CHECK: [[PROF21]] = !{!"branch_weights", i32 199, i32 1}
; CHECK: [[META22:![0-9]+]] = !{}
; CHECK: [[PROF23]] = !{!"branch_weights", i32 197, i32 1}
; CHECK: [[PROF24]] = !{!"branch_weights", i32 100, i32 98}
; CHECK: [[PROF25]] = !{!"branch_weights", i32 101, i32 99}
; CHECK: [[PROF26]] = !{!"branch_weights", i32 1, i32 197}
; CHECK: [[PROF27]] = !{!"branch_weights", i32 99, i32 101}
;.
58 changes: 48 additions & 10 deletions llvm/test/Transforms/SimplifyCFG/speculatively-execute-block-profmd.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,12 @@ define i32 @predictably_nontaken(i1 %c, i32 %a, i32 %b) {
; CHECK: dispatch:
; CHECK-NEXT: call void @sideeffect1()
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: br i1 [[CMP]], label [[END]], label [[COND_TRUE:%.*]], !prof [[PROF0]]
; CHECK: cond.true:
; CHECK-NEXT: [[VAL:%.*]] = add i32 [[A]], [[B]]
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 0, i32 [[VAL]], !prof [[PROF0]]
; CHECK-NEXT: br label [[END]]
; CHECK: end:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[DISPATCH]] ]
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ 0, [[DISPATCH]] ], [ [[VAL]], [[COND_TRUE]] ]
; CHECK-NEXT: call void @sideeffect2()
; CHECK-NEXT: ret i32 [[RES]]
;
Expand All @@ -110,6 +111,41 @@ end:
ret i32 %res
}

define i32 @almost_predictably_nontaken(i1 %c, i32 %a, i32 %b) {
; CHECK-LABEL: @almost_predictably_nontaken(
; CHECK-NEXT: entry:
; CHECK-NEXT: call void @sideeffect0()
; CHECK-NEXT: br i1 [[C:%.*]], label [[DISPATCH:%.*]], label [[END:%.*]]
; CHECK: dispatch:
; CHECK-NEXT: call void @sideeffect1()
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[VAL:%.*]] = add i32 [[A]], [[B]]
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 0, i32 [[VAL]], !prof [[PROF1:![0-9]+]]
; CHECK-NEXT: br label [[END]]
; CHECK: end:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[DISPATCH]] ]
; CHECK-NEXT: call void @sideeffect2()
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
call void @sideeffect0()
br i1 %c, label %dispatch, label %end

dispatch:
call void @sideeffect1()
%cmp = icmp eq i32 %a, %b
br i1 %cmp, label %end, label %cond.true, !prof !1 ; almost likely branches to %end

cond.true:
%val = add i32 %a, %b
br label %end

end:
%res = phi i32 [ -1, %entry ], [ 0, %dispatch ], [ %val, %cond.true ]
call void @sideeffect2()
ret i32 %res
}

define i32 @unpredictable(i1 %c, i32 %a, i32 %b) {
; CHECK-LABEL: @unpredictable(
; CHECK-NEXT: entry:
Expand All @@ -119,7 +155,7 @@ define i32 @unpredictable(i1 %c, i32 %a, i32 %b) {
; CHECK-NEXT: call void @sideeffect1()
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[VAL:%.*]] = add i32 [[A]], [[B]]
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 [[VAL]], i32 0, !unpredictable !1
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 [[VAL]], i32 0, !unpredictable !2
; CHECK-NEXT: br label [[END]]
; CHECK: end:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[DISPATCH]] ]
Expand All @@ -133,7 +169,7 @@ entry:
dispatch:
call void @sideeffect1()
%cmp = icmp eq i32 %a, %b
br i1 %cmp, label %cond.true, label %end, !unpredictable !1 ; unpredictable
br i1 %cmp, label %cond.true, label %end, !unpredictable !2 ; unpredictable

cond.true:
%val = add i32 %a, %b
Expand All @@ -154,7 +190,7 @@ define i32 @unpredictable_yet_taken(i1 %c, i32 %a, i32 %b) {
; CHECK-NEXT: call void @sideeffect1()
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[VAL:%.*]] = add i32 [[A]], [[B]]
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 [[VAL]], i32 0, !prof [[PROF0]], !unpredictable !1
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 [[VAL]], i32 0, !prof [[PROF0]], !unpredictable !2
; CHECK-NEXT: br label [[END]]
; CHECK: end:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[DISPATCH]] ]
Expand All @@ -168,7 +204,7 @@ entry:
dispatch:
call void @sideeffect1()
%cmp = icmp eq i32 %a, %b
br i1 %cmp, label %cond.true, label %end, !prof !0, !unpredictable !1 ; likely branches to %cond.true, yet unpredictable
br i1 %cmp, label %cond.true, label %end, !prof !0, !unpredictable !2 ; likely branches to %cond.true, yet unpredictable

cond.true:
%val = add i32 %a, %b
Expand All @@ -189,7 +225,7 @@ define i32 @unpredictable_yet_nontaken(i1 %c, i32 %a, i32 %b) {
; CHECK-NEXT: call void @sideeffect1()
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[VAL:%.*]] = add i32 [[A]], [[B]]
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 0, i32 [[VAL]], !prof [[PROF0]], !unpredictable !1
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[CMP]], i32 0, i32 [[VAL]], !prof [[PROF0]], !unpredictable !2
; CHECK-NEXT: br label [[END]]
; CHECK: end:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[DISPATCH]] ]
Expand All @@ -203,7 +239,7 @@ entry:
dispatch:
call void @sideeffect1()
%cmp = icmp eq i32 %a, %b
br i1 %cmp, label %end, label %cond.true, !prof !0, !unpredictable !1 ; likely branches to %end, yet unpredictable
br i1 %cmp, label %end, label %cond.true, !prof !0, !unpredictable !2 ; likely branches to %end, yet unpredictable

cond.true:
%val = add i32 %a, %b
Expand All @@ -216,7 +252,9 @@ end:
}

!0 = !{!"branch_weights", i32 99, i32 1}
!1 = !{}
!1 = !{!"branch_weights", i32 70, i32 1}
!2 = !{}

; CHECK: !0 = !{!"branch_weights", i32 99, i32 1}
; CHECK: !1 = !{}
; CHECK: !1 = !{!"branch_weights", i32 70, i32 1}
; CHECK: !2 = !{}