Skip to content
This repository was archived by the owner on Apr 23, 2020. It is now read-only.

Commit e0e019f

Browse files
bognerbogner
committed
CodeGen: Initial instrumentation based PGO implementation
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@198640 91177308-0d34-0410-b5e6-96231b3b80d8
1 parent f4c386b commit e0e019f

20 files changed

+1710
-59
lines changed

‎lib/CodeGen/CGCUDARuntime.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ RValue CGCUDARuntime::EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
3131
llvm::BasicBlock *ContBlock = CGF.createBasicBlock("kcall.end");
3232

3333
CodeGenFunction::ConditionalEvaluation eval(CGF);
34-
CGF.EmitBranchOnBoolExpr(E->getConfig(), ContBlock, ConfigOKBlock);
34+
CGF.EmitBranchOnBoolExpr(E->getConfig(), ContBlock, ConfigOKBlock,
35+
/*TrueCount=*/0);
3536

3637
eval.begin(CGF);
3738
CGF.EmitBlock(ConfigOKBlock);

‎lib/CodeGen/CGCall.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2184,6 +2184,7 @@ void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee,
21842184
call->setCallingConv(getRuntimeCC());
21852185
Builder.CreateUnreachable();
21862186
}
2187+
PGO.setCurrentRegionCount(0);
21872188
}
21882189

21892190
/// Emits a call or invoke instruction to the given nullary runtime

‎lib/CodeGen/CGException.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1294,6 +1294,10 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
12941294
// Initialize the catch variable and set up the cleanups.
12951295
BeginCatch(*this, C);
12961296

1297+
// Emit the PGO counter increment
1298+
RegionCounter CatchCnt = getPGORegionCounter(C);
1299+
CatchCnt.beginRegion(Builder);
1300+
12971301
// Perform the body of the catch.
12981302
EmitStmt(C->getHandlerBlock());
12991303

@@ -1320,7 +1324,9 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
13201324
Builder.CreateBr(ContBB);
13211325
}
13221326

1327+
RegionCounter ContCnt = getPGORegionCounter(&S);
13231328
EmitBlock(ContBB);
1329+
ContCnt.beginRegion(Builder);
13241330
}
13251331

13261332
namespace {

‎lib/CodeGen/CGExpr.cpp

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2651,29 +2651,36 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
26512651
}
26522652

26532653
OpaqueValueMapping binding(*this, expr);
2654+
RegionCounter Cnt = getPGORegionCounter(expr);
26542655

26552656
const Expr *condExpr = expr->getCond();
26562657
bool CondExprBool;
26572658
if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
26582659
const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
26592660
if (!CondExprBool) std::swap(live, dead);
26602661

2661-
if (!ContainsLabel(dead))
2662+
if (!ContainsLabel(dead)) {
2663+
// If the true case is live, we need to track its region
2664+
if (CondExprBool)
2665+
Cnt.beginRegion(Builder);
26622666
return EmitLValue(live);
2667+
}
26632668
}
26642669

26652670
llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
26662671
llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
26672672
llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
26682673

26692674
ConditionalEvaluation eval(*this);
2670-
EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
2675+
EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, Cnt.getCount());
26712676

26722677
// Any temporaries created here are conditional.
26732678
EmitBlock(lhsBlock);
2679+
Cnt.beginRegion(Builder);
26742680
eval.begin(*this);
26752681
LValue lhs = EmitLValue(expr->getTrueExpr());
26762682
eval.end(*this);
2683+
Cnt.adjustFallThroughCount();
26772684

26782685
if (!lhs.isSimple())
26792686
return EmitUnsupportedLValue(expr, "conditional operator");
@@ -2683,14 +2690,17 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
26832690

26842691
// Any temporaries created here are conditional.
26852692
EmitBlock(rhsBlock);
2693+
Cnt.beginElseRegion();
26862694
eval.begin(*this);
26872695
LValue rhs = EmitLValue(expr->getFalseExpr());
26882696
eval.end(*this);
2697+
Cnt.adjustFallThroughCount();
26892698
if (!rhs.isSimple())
26902699
return EmitUnsupportedLValue(expr, "conditional operator");
26912700
rhsBlock = Builder.GetInsertBlock();
26922701

26932702
EmitBlock(contBlock);
2703+
Cnt.applyAdjustmentsToRegion();
26942704

26952705
llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
26962706
"cond-lvalue");

‎lib/CodeGen/CGExprAgg.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -892,15 +892,18 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
892892
// Bind the common expression if necessary.
893893
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
894894

895+
RegionCounter Cnt = CGF.getPGORegionCounter(E);
895896
CodeGenFunction::ConditionalEvaluation eval(CGF);
896-
CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
897+
CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
897898

898899
// Save whether the destination's lifetime is externally managed.
899900
bool isExternallyDestructed = Dest.isExternallyDestructed();
900901

901902
eval.begin(CGF);
902903
CGF.EmitBlock(LHSBlock);
904+
Cnt.beginRegion(Builder);
903905
Visit(E->getTrueExpr());
906+
Cnt.adjustFallThroughCount();
904907
eval.end(CGF);
905908

906909
assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
@@ -914,10 +917,13 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
914917

915918
eval.begin(CGF);
916919
CGF.EmitBlock(RHSBlock);
920+
Cnt.beginElseRegion();
917921
Visit(E->getFalseExpr());
922+
Cnt.adjustFallThroughCount();
918923
eval.end(CGF);
919924

920925
CGF.EmitBlock(ContBlock);
926+
Cnt.applyAdjustmentsToRegion();
921927
}
922928

923929
void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {

‎lib/CodeGen/CGExprComplex.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -752,22 +752,28 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
752752
// Bind the common expression if necessary.
753753
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
754754

755+
RegionCounter Cnt = CGF.getPGORegionCounter(E);
755756
CodeGenFunction::ConditionalEvaluation eval(CGF);
756-
CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
757+
CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
757758

758759
eval.begin(CGF);
759760
CGF.EmitBlock(LHSBlock);
761+
Cnt.beginRegion(Builder);
760762
ComplexPairTy LHS = Visit(E->getTrueExpr());
763+
Cnt.adjustFallThroughCount();
761764
LHSBlock = Builder.GetInsertBlock();
762765
CGF.EmitBranch(ContBlock);
763766
eval.end(CGF);
764767

765768
eval.begin(CGF);
766769
CGF.EmitBlock(RHSBlock);
770+
Cnt.beginElseRegion();
767771
ComplexPairTy RHS = Visit(E->getFalseExpr());
772+
Cnt.adjustFallThroughCount();
768773
RHSBlock = Builder.GetInsertBlock();
769774
CGF.EmitBlock(ContBlock);
770775
eval.end(CGF);
776+
Cnt.applyAdjustmentsToRegion();
771777

772778
// Create a PHI node for the real part.
773779
llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.r");

‎lib/CodeGen/CGExprScalar.cpp

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2874,8 +2874,12 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
28742874
}
28752875

28762876
Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
2877+
RegionCounter Cnt = CGF.getPGORegionCounter(E);
2878+
28772879
// Perform vector logical and on comparisons with zero vectors.
28782880
if (E->getType()->isVectorType()) {
2881+
Cnt.beginRegion(Builder);
2882+
28792883
Value *LHS = Visit(E->getLHS());
28802884
Value *RHS = Visit(E->getRHS());
28812885
Value *Zero = llvm::ConstantAggregateZero::get(LHS->getType());
@@ -2897,6 +2901,8 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
28972901
bool LHSCondVal;
28982902
if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
28992903
if (LHSCondVal) { // If we have 1 && X, just emit X.
2904+
Cnt.beginRegion(Builder);
2905+
29002906
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
29012907
// ZExt result to int or bool.
29022908
return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "land.ext");
@@ -2913,7 +2919,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
29132919
CodeGenFunction::ConditionalEvaluation eval(CGF);
29142920

29152921
// Branch on the LHS first. If it is false, go to the failure (cont) block.
2916-
CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock);
2922+
CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, Cnt.getCount());
29172923

29182924
// Any edges into the ContBlock are now from an (indeterminate number of)
29192925
// edges from this first condition. All of these values will be false. Start
@@ -2926,7 +2932,9 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
29262932

29272933
eval.begin(CGF);
29282934
CGF.EmitBlock(RHSBlock);
2935+
Cnt.beginRegion(Builder);
29292936
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
2937+
Cnt.adjustFallThroughCount();
29302938
eval.end(CGF);
29312939

29322940
// Reaquire the RHS block, as there may be subblocks inserted.
@@ -2939,14 +2947,19 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
29392947
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
29402948
CGF.EmitBlock(ContBlock);
29412949
PN->addIncoming(RHSCond, RHSBlock);
2950+
Cnt.applyAdjustmentsToRegion();
29422951

29432952
// ZExt result to int.
29442953
return Builder.CreateZExtOrBitCast(PN, ResTy, "land.ext");
29452954
}
29462955

29472956
Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
2957+
RegionCounter Cnt = CGF.getPGORegionCounter(E);
2958+
29482959
// Perform vector logical or on comparisons with zero vectors.
29492960
if (E->getType()->isVectorType()) {
2961+
Cnt.beginRegion(Builder);
2962+
29502963
Value *LHS = Visit(E->getLHS());
29512964
Value *RHS = Visit(E->getRHS());
29522965
Value *Zero = llvm::ConstantAggregateZero::get(LHS->getType());
@@ -2968,6 +2981,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
29682981
bool LHSCondVal;
29692982
if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
29702983
if (!LHSCondVal) { // If we have 0 || X, just emit X.
2984+
Cnt.beginRegion(Builder);
2985+
29712986
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
29722987
// ZExt result to int or bool.
29732988
return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "lor.ext");
@@ -2984,7 +2999,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
29842999
CodeGenFunction::ConditionalEvaluation eval(CGF);
29853000

29863001
// Branch on the LHS first. If it is true, go to the success (cont) block.
2987-
CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock);
3002+
CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock,
3003+
Cnt.getParentCount() - Cnt.getCount());
29883004

29893005
// Any edges into the ContBlock are now from an (indeterminate number of)
29903006
// edges from this first condition. All of these values will be true. Start
@@ -2999,7 +3015,9 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
29993015

30003016
// Emit the RHS condition as a bool value.
30013017
CGF.EmitBlock(RHSBlock);
3018+
Cnt.beginRegion(Builder);
30023019
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
3020+
Cnt.adjustFallThroughCount();
30033021

30043022
eval.end(CGF);
30053023

@@ -3010,6 +3028,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
30103028
// into the phi node for the edge with the value of RHSCond.
30113029
CGF.EmitBlock(ContBlock);
30123030
PN->addIncoming(RHSCond, RHSBlock);
3031+
Cnt.applyAdjustmentsToRegion();
30133032

30143033
// ZExt result to int.
30153034
return Builder.CreateZExtOrBitCast(PN, ResTy, "lor.ext");
@@ -3049,6 +3068,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
30493068

30503069
// Bind the common expression if necessary.
30513070
CodeGenFunction::OpaqueValueMapping binding(CGF, E);
3071+
RegionCounter Cnt = CGF.getPGORegionCounter(E);
30523072

30533073
Expr *condExpr = E->getCond();
30543074
Expr *lhsExpr = E->getTrueExpr();
@@ -3063,6 +3083,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
30633083

30643084
// If the dead side doesn't have labels we need, just emit the Live part.
30653085
if (!CGF.ContainsLabel(dead)) {
3086+
if (CondExprBool)
3087+
Cnt.beginRegion(Builder);
30663088
Value *Result = Visit(live);
30673089

30683090
// If the live part is a throw expression, it acts like it has a void
@@ -3079,6 +3101,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
30793101
// the select function.
30803102
if (CGF.getLangOpts().OpenCL
30813103
&& condExpr->getType()->isVectorType()) {
3104+
Cnt.beginRegion(Builder);
3105+
30823106
llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
30833107
llvm::Value *LHS = Visit(lhsExpr);
30843108
llvm::Value *RHS = Visit(rhsExpr);
@@ -3122,6 +3146,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
31223146
// safe to evaluate the LHS and RHS unconditionally.
31233147
if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) &&
31243148
isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) {
3149+
Cnt.beginRegion(Builder);
3150+
31253151
llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr);
31263152
llvm::Value *LHS = Visit(lhsExpr);
31273153
llvm::Value *RHS = Visit(rhsExpr);
@@ -3138,23 +3164,28 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
31383164
llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
31393165

31403166
CodeGenFunction::ConditionalEvaluation eval(CGF);
3141-
CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock);
3167+
CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, Cnt.getCount());
31423168

31433169
CGF.EmitBlock(LHSBlock);
3170+
Cnt.beginRegion(Builder);
31443171
eval.begin(CGF);
31453172
Value *LHS = Visit(lhsExpr);
31463173
eval.end(CGF);
3174+
Cnt.adjustFallThroughCount();
31473175

31483176
LHSBlock = Builder.GetInsertBlock();
31493177
Builder.CreateBr(ContBlock);
31503178

31513179
CGF.EmitBlock(RHSBlock);
3180+
Cnt.beginElseRegion();
31523181
eval.begin(CGF);
31533182
Value *RHS = Visit(rhsExpr);
31543183
eval.end(CGF);
3184+
Cnt.adjustFallThroughCount();
31553185

31563186
RHSBlock = Builder.GetInsertBlock();
31573187
CGF.EmitBlock(ContBlock);
3188+
Cnt.applyAdjustmentsToRegion();
31583189

31593190
// If the LHS or RHS is a throw expression, it will be legitimately null.
31603191
if (!LHS)

‎lib/CodeGen/CGObjC.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1523,10 +1523,13 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
15231523
llvm::Value *initialMutations =
15241524
Builder.CreateLoad(StateMutationsPtr, "forcoll.initial-mutations");
15251525

1526+
RegionCounter Cnt = getPGORegionCounter(&S);
1527+
15261528
// Start looping. This is the point we return to whenever we have a
15271529
// fresh, non-empty batch of objects.
15281530
llvm::BasicBlock *LoopBodyBB = createBasicBlock("forcoll.loopbody");
15291531
EmitBlock(LoopBodyBB);
1532+
Cnt.beginRegion(Builder);
15301533

15311534
// The current index into the buffer.
15321535
llvm::PHINode *index = Builder.CreatePHI(UnsignedLongLTy, 3, "forcoll.index");
@@ -1623,7 +1626,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
16231626
EmitAutoVarCleanups(variable);
16241627

16251628
// Perform the loop body, setting up break and continue labels.
1626-
BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody));
1629+
BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody, &Cnt));
16271630
{
16281631
RunCleanupsScope Scope(*this);
16291632
EmitStmt(S.getBody());
@@ -1642,6 +1645,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
16421645
llvm::Value *indexPlusOne
16431646
= Builder.CreateAdd(index, llvm::ConstantInt::get(UnsignedLongLTy, 1));
16441647

1648+
// TODO: We should probably model this as a "continue" for PGO
16451649
// If we haven't overrun the buffer yet, we can continue.
16461650
Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count),
16471651
LoopBodyBB, FetchMoreBB);
@@ -1665,6 +1669,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
16651669
index->addIncoming(zero, Builder.GetInsertBlock());
16661670
count->addIncoming(refetchCount, Builder.GetInsertBlock());
16671671

1672+
// TODO: We should be applying PGO weights here, but this needs to handle the
1673+
// branch before FetchMoreBB or we risk getting the numbers wrong.
16681674
Builder.CreateCondBr(Builder.CreateICmpEQ(refetchCount, zero),
16691675
EmptyBB, LoopBodyBB);
16701676

@@ -1687,6 +1693,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
16871693
PopCleanupBlock();
16881694

16891695
EmitBlock(LoopEnd.getBlock());
1696+
// TODO: Once we calculate PGO weights above, set the region count here
16901697
}
16911698

16921699
void CodeGenFunction::EmitObjCAtTryStmt(const ObjCAtTryStmt &S) {

0 commit comments

Comments
 (0)