CodeGen: Initial instrumentation based PGO implementation

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@198640 91177308-0d34-0410-b5e6-96231b3b80d8
llvm-mirror · Jan 6, 2014 · e0e019f6a2d76e140221d9c10159a444839e2563 · e0e019f
1 parent f4c386b
commit e0e019f6a2d76e140221d9c10159a444839e2563
diff --git a/lib/CodeGen/CGCUDARuntime.cpp b/lib/CodeGen/CGCUDARuntime.cpp
@@ -31,7 +31,8 @@ RValue CGCUDARuntime::EmitCUDAKernelCallExpr(CodeGenFunction &CGF,
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("kcall.end");

  CodeGenFunction::ConditionalEvaluation eval(CGF);
-  CGF.EmitBranchOnBoolExpr(E->getConfig(), ContBlock, ConfigOKBlock);
+  CGF.EmitBranchOnBoolExpr(E->getConfig(), ContBlock, ConfigOKBlock,
+                           /*TrueCount=*/0);

  eval.begin(CGF);
  CGF.EmitBlock(ConfigOKBlock);
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
@@ -2184,6 +2184,7 @@ void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee,
    call->setCallingConv(getRuntimeCC());
    Builder.CreateUnreachable();
  }
+  PGO.setCurrentRegionCount(0);
 }

 /// Emits a call or invoke instruction to the given nullary runtime
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
@@ -1294,6 +1294,10 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
    // Initialize the catch variable and set up the cleanups.
    BeginCatch(*this, C);

+    // Emit the PGO counter increment
+    RegionCounter CatchCnt = getPGORegionCounter(C);
+    CatchCnt.beginRegion(Builder);
+
    // Perform the body of the catch.
    EmitStmt(C->getHandlerBlock());

@@ -1320,7 +1324,9 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
      Builder.CreateBr(ContBB);
  }

+  RegionCounter ContCnt = getPGORegionCounter(&S);
  EmitBlock(ContBB);
+  ContCnt.beginRegion(Builder);
 }

 namespace {
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
@@ -2651,29 +2651,36 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
  }

  OpaqueValueMapping binding(*this, expr);
+  RegionCounter Cnt = getPGORegionCounter(expr);

  const Expr *condExpr = expr->getCond();
  bool CondExprBool;
  if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
    const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
    if (!CondExprBool) std::swap(live, dead);

-    if (!ContainsLabel(dead))
+    if (!ContainsLabel(dead)) {
+      // If the true case is live, we need to track its region
+      if (CondExprBool)
+        Cnt.beginRegion(Builder);
      return EmitLValue(live);
+    }
  }

  llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
  llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
  llvm::BasicBlock *contBlock = createBasicBlock("cond.end");

  ConditionalEvaluation eval(*this);
-  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
+  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, Cnt.getCount());

  // Any temporaries created here are conditional.
  EmitBlock(lhsBlock);
+  Cnt.beginRegion(Builder);
  eval.begin(*this);
  LValue lhs = EmitLValue(expr->getTrueExpr());
  eval.end(*this);
+  Cnt.adjustFallThroughCount();

  if (!lhs.isSimple())
    return EmitUnsupportedLValue(expr, "conditional operator");
@@ -2683,14 +2690,17 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {

  // Any temporaries created here are conditional.
  EmitBlock(rhsBlock);
+  Cnt.beginElseRegion();
  eval.begin(*this);
  LValue rhs = EmitLValue(expr->getFalseExpr());
  eval.end(*this);
+  Cnt.adjustFallThroughCount();
  if (!rhs.isSimple())
    return EmitUnsupportedLValue(expr, "conditional operator");
  rhsBlock = Builder.GetInsertBlock();

  EmitBlock(contBlock);
+  Cnt.applyAdjustmentsToRegion();

  llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
                                         "cond-lvalue");
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
@@ -892,15 +892,18 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
  // Bind the common expression if necessary.
  CodeGenFunction::OpaqueValueMapping binding(CGF, E);

+  RegionCounter Cnt = CGF.getPGORegionCounter(E);
  CodeGenFunction::ConditionalEvaluation eval(CGF);
-  CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
+  CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());

  // Save whether the destination's lifetime is externally managed.
  bool isExternallyDestructed = Dest.isExternallyDestructed();

  eval.begin(CGF);
  CGF.EmitBlock(LHSBlock);
+  Cnt.beginRegion(Builder);
  Visit(E->getTrueExpr());
+  Cnt.adjustFallThroughCount();
  eval.end(CGF);

  assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
@@ -914,10 +917,13 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {

  eval.begin(CGF);
  CGF.EmitBlock(RHSBlock);
+  Cnt.beginElseRegion();
  Visit(E->getFalseExpr());
+  Cnt.adjustFallThroughCount();
  eval.end(CGF);

  CGF.EmitBlock(ContBlock);
+  Cnt.applyAdjustmentsToRegion();
 }

 void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
@@ -752,22 +752,28 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
  // Bind the common expression if necessary.
  CodeGenFunction::OpaqueValueMapping binding(CGF, E);

+  RegionCounter Cnt = CGF.getPGORegionCounter(E);
  CodeGenFunction::ConditionalEvaluation eval(CGF);
-  CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock);
+  CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount());

  eval.begin(CGF);
  CGF.EmitBlock(LHSBlock);
+  Cnt.beginRegion(Builder);
  ComplexPairTy LHS = Visit(E->getTrueExpr());
+  Cnt.adjustFallThroughCount();
  LHSBlock = Builder.GetInsertBlock();
  CGF.EmitBranch(ContBlock);
  eval.end(CGF);

  eval.begin(CGF);
  CGF.EmitBlock(RHSBlock);
+  Cnt.beginElseRegion();
  ComplexPairTy RHS = Visit(E->getFalseExpr());
+  Cnt.adjustFallThroughCount();
  RHSBlock = Builder.GetInsertBlock();
  CGF.EmitBlock(ContBlock);
  eval.end(CGF);
+  Cnt.applyAdjustmentsToRegion();

  // Create a PHI node for the real part.
  llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.r");
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
@@ -2874,8 +2874,12 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
 }

 Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
+  RegionCounter Cnt = CGF.getPGORegionCounter(E);
+
  // Perform vector logical and on comparisons with zero vectors.
  if (E->getType()->isVectorType()) {
+    Cnt.beginRegion(Builder);
+
    Value *LHS = Visit(E->getLHS());
    Value *RHS = Visit(E->getRHS());
    Value *Zero = llvm::ConstantAggregateZero::get(LHS->getType());
@@ -2897,6 +2901,8 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
  bool LHSCondVal;
  if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
    if (LHSCondVal) { // If we have 1 && X, just emit X.
+      Cnt.beginRegion(Builder);
+
      Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
      // ZExt result to int or bool.
      return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "land.ext");
@@ -2913,7 +2919,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
  CodeGenFunction::ConditionalEvaluation eval(CGF);

  // Branch on the LHS first.  If it is false, go to the failure (cont) block.
-  CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock);
+  CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, Cnt.getCount());

  // Any edges into the ContBlock are now from an (indeterminate number of)
  // edges from this first condition.  All of these values will be false.  Start
@@ -2926,7 +2932,9 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {

  eval.begin(CGF);
  CGF.EmitBlock(RHSBlock);
+  Cnt.beginRegion(Builder);
  Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
+  Cnt.adjustFallThroughCount();
  eval.end(CGF);

  // Reaquire the RHS block, as there may be subblocks inserted.
@@ -2939,14 +2947,19 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
    Builder.SetCurrentDebugLocation(llvm::DebugLoc());
  CGF.EmitBlock(ContBlock);
  PN->addIncoming(RHSCond, RHSBlock);
+  Cnt.applyAdjustmentsToRegion();

  // ZExt result to int.
  return Builder.CreateZExtOrBitCast(PN, ResTy, "land.ext");
 }

 Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
+  RegionCounter Cnt = CGF.getPGORegionCounter(E);
+
  // Perform vector logical or on comparisons with zero vectors.
  if (E->getType()->isVectorType()) {
+    Cnt.beginRegion(Builder);
+
    Value *LHS = Visit(E->getLHS());
    Value *RHS = Visit(E->getRHS());
    Value *Zero = llvm::ConstantAggregateZero::get(LHS->getType());
@@ -2968,6 +2981,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
  bool LHSCondVal;
  if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) {
    if (!LHSCondVal) { // If we have 0 || X, just emit X.
+      Cnt.beginRegion(Builder);
+
      Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
      // ZExt result to int or bool.
      return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "lor.ext");
@@ -2984,7 +2999,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
  CodeGenFunction::ConditionalEvaluation eval(CGF);

  // Branch on the LHS first.  If it is true, go to the success (cont) block.
-  CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock);
+  CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock,
+                           Cnt.getParentCount() - Cnt.getCount());

  // Any edges into the ContBlock are now from an (indeterminate number of)
  // edges from this first condition.  All of these values will be true.  Start
@@ -2999,7 +3015,9 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {

  // Emit the RHS condition as a bool value.
  CGF.EmitBlock(RHSBlock);
+  Cnt.beginRegion(Builder);
  Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
+  Cnt.adjustFallThroughCount();

  eval.end(CGF);

@@ -3010,6 +3028,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
  // into the phi node for the edge with the value of RHSCond.
  CGF.EmitBlock(ContBlock);
  PN->addIncoming(RHSCond, RHSBlock);
+  Cnt.applyAdjustmentsToRegion();

  // ZExt result to int.
  return Builder.CreateZExtOrBitCast(PN, ResTy, "lor.ext");
@@ -3049,6 +3068,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {

  // Bind the common expression if necessary.
  CodeGenFunction::OpaqueValueMapping binding(CGF, E);
+  RegionCounter Cnt = CGF.getPGORegionCounter(E);

  Expr *condExpr = E->getCond();
  Expr *lhsExpr = E->getTrueExpr();
@@ -3063,6 +3083,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {

    // If the dead side doesn't have labels we need, just emit the Live part.
    if (!CGF.ContainsLabel(dead)) {
+      if (CondExprBool)
+        Cnt.beginRegion(Builder);
      Value *Result = Visit(live);

      // If the live part is a throw expression, it acts like it has a void
@@ -3079,6 +3101,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
  // the select function.
  if (CGF.getLangOpts().OpenCL
      && condExpr->getType()->isVectorType()) {
+    Cnt.beginRegion(Builder);
+
    llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
    llvm::Value *LHS = Visit(lhsExpr);
    llvm::Value *RHS = Visit(rhsExpr);
@@ -3122,6 +3146,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
  // safe to evaluate the LHS and RHS unconditionally.
  if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) &&
      isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) {
+    Cnt.beginRegion(Builder);
+
    llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr);
    llvm::Value *LHS = Visit(lhsExpr);
    llvm::Value *RHS = Visit(rhsExpr);
@@ -3138,23 +3164,28 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");

  CodeGenFunction::ConditionalEvaluation eval(CGF);
-  CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock);
+  CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, Cnt.getCount());

  CGF.EmitBlock(LHSBlock);
+  Cnt.beginRegion(Builder);
  eval.begin(CGF);
  Value *LHS = Visit(lhsExpr);
  eval.end(CGF);
+  Cnt.adjustFallThroughCount();

  LHSBlock = Builder.GetInsertBlock();
  Builder.CreateBr(ContBlock);

  CGF.EmitBlock(RHSBlock);
+  Cnt.beginElseRegion();
  eval.begin(CGF);
  Value *RHS = Visit(rhsExpr);
  eval.end(CGF);
+  Cnt.adjustFallThroughCount();

  RHSBlock = Builder.GetInsertBlock();
  CGF.EmitBlock(ContBlock);
+  Cnt.applyAdjustmentsToRegion();

  // If the LHS or RHS is a throw expression, it will be legitimately null.
  if (!LHS)
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
@@ -1523,10 +1523,13 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
  llvm::Value *initialMutations =
    Builder.CreateLoad(StateMutationsPtr, "forcoll.initial-mutations");

+  RegionCounter Cnt = getPGORegionCounter(&S);
+
  // Start looping.  This is the point we return to whenever we have a
  // fresh, non-empty batch of objects.
  llvm::BasicBlock *LoopBodyBB = createBasicBlock("forcoll.loopbody");
  EmitBlock(LoopBodyBB);
+  Cnt.beginRegion(Builder);

  // The current index into the buffer.
  llvm::PHINode *index = Builder.CreatePHI(UnsignedLongLTy, 3, "forcoll.index");
@@ -1623,7 +1626,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
    EmitAutoVarCleanups(variable);

  // Perform the loop body, setting up break and continue labels.
-  BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody));
+  BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody, &Cnt));
  {
    RunCleanupsScope Scope(*this);
    EmitStmt(S.getBody());
@@ -1642,6 +1645,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
  llvm::Value *indexPlusOne
    = Builder.CreateAdd(index, llvm::ConstantInt::get(UnsignedLongLTy, 1));

+  // TODO: We should probably model this as a "continue" for PGO
  // If we haven't overrun the buffer yet, we can continue.
  Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count),
                       LoopBodyBB, FetchMoreBB);
@@ -1665,6 +1669,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
  index->addIncoming(zero, Builder.GetInsertBlock());
  count->addIncoming(refetchCount, Builder.GetInsertBlock());

+  // TODO: We should be applying PGO weights here, but this needs to handle the
+  // branch before FetchMoreBB or we risk getting the numbers wrong.
  Builder.CreateCondBr(Builder.CreateICmpEQ(refetchCount, zero),
                       EmptyBB, LoopBodyBB);

@@ -1687,6 +1693,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
    PopCleanupBlock();

  EmitBlock(LoopEnd.getBlock());
+  // TODO: Once we calculate PGO weights above, set the region count here
 }

 void CodeGenFunction::EmitObjCAtTryStmt(const ObjCAtTryStmt &S) {